diff --git a/.github/actions/setup-python/action.yaml b/.github/actions/setup-python/action.yaml index 923755939..f18e01968 100644 --- a/.github/actions/setup-python/action.yaml +++ b/.github/actions/setup-python/action.yaml @@ -15,7 +15,7 @@ runs: steps: - name: Set up Python ${{ inputs.python-version }} with actions/setup-python if: ${{ inputs.python-version != '2.7' }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ inputs.python-version }} diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..07123dde9 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,21 @@ +# Set update schedule for GitHub Actions +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + # Check for updates to GitHub Actions every week + interval: "weekly" + + # https://github.com/dependabot/dependabot-core/issues/6704 + - package-ecosystem: "github-actions" + directory: "/.github/actions/setup-python" + schedule: + # Check for updates to GitHub Actions every week + interval: "weekly" + + - package-ecosystem: "pip" + directory: "/docs" + schedule: + # Check for updates to GitHub Actions every week + interval: "weekly" diff --git a/.github/docker/rez-win-base/Dockerfile b/.github/docker/rez-win-base/Dockerfile deleted file mode 100644 index 3d97e70a0..000000000 --- a/.github/docker/rez-win-base/Dockerfile +++ /dev/null @@ -1,34 +0,0 @@ -# escape=` - -# Please Note: Any " requires \" in the Dockerfile for windows. - -ARG WINDOWS_VERSION -FROM mcr.microsoft.com/windows/servercore:$WINDOWS_VERSION - -LABEL org.opencontainers.image.description="WARNING: This is an internal image and should not be used outside of the rez repository!" - -SHELL ["powershell.exe", "-NoLogo", "-NoProfile", "-ExecutionPolicy", "ByPass"] - -ARG GIT_VERSION=2.23.0 -ARG CMAKE_VERSION=3.15.4 -ARG PWSH_VERSION=6.2.2 - -# ------------------------------------------------------------------------------------------------------------ -# Install: -# - Chocolatey -# - Git -# - Cmake -# - PowerShellCore -# -ENV chocolateyUseWindowsCompression false -ENV chocolateyVersion=1.4.0 -RUN iex ((new-object net.webclient).DownloadString('https://chocolatey.org/install.ps1')); ` - choco feature disable --name showDownloadProgress; ` - choco install git.install --yes --version=${ENV:GIT_VERSION}; ` - choco install cmake --yes --version=${ENV:CMAKE_VERSION} --installargs 'ADD_CMAKE_TO_PATH=System'; ` - choco install pwsh --yes --version=${PWSH_VERSION}; ` - choco install --yes choco-cleaner; ` - C:\ProgramData\chocolatey\bin\choco-cleaner.bat; ` - choco uninstall --yes choco-cleaner - -ENTRYPOINT ["powershell.exe", "-NoLogo", "-ExecutionPolicy", "ByPass"] diff --git a/.github/docker/rez-win-py/Dockerfile b/.github/docker/rez-win-py/Dockerfile deleted file mode 100644 index ab14b55bc..000000000 --- a/.github/docker/rez-win-py/Dockerfile +++ /dev/null @@ -1,65 +0,0 @@ -# escape=` - -ARG BASE_IMAGE_NAME -FROM $BASE_IMAGE_NAME - -LABEL org.opencontainers.image.description="WARNING: This is an internal image and should not be used outside of the rez repository!" - -# Name of this image -ARG IMAGE_NAME -ENV _IMAGE_NAME=$IMAGE_NAME - -# NOTE: Any " requires \" in the Dockerfile for windows. -# NOTE: The order matters. ARG after the shell command will allow access via -# the PowerShell environment like ${ENV:PYTHON_VERSION}. -SHELL ["powershell.exe", "-NoLogo", "-NoProfile", "-ExecutionPolicy", "ByPass"] - -# Python version to install (full chocolatey compatible version required) -# For example: 3.7.5 -# -ARG PYTHON_VERSION - - -# ------------------------------------------------------------------------------------------------------------ -# Python -# -# Installs given Python version to C:\Python -# Python 2.x uses msi while 3.x has an exe with separate arguments -# Verifies the installation by running python explicitly and via `python` -# -RUN ${PYTHON_INSTALL_PATH} = 'C:\Python'; ` - ${PYTHON_MAJOR_VERSION} = ${ENV:PYTHON_VERSION}.Split('.')[0]; ` - if (${PYTHON_MAJOR_VERSION} -eq "2") { ` - ${INSTALLARGS} = \"'/qn /norestart ADDLOCAL=ALL ALLUSERS=1 TARGETDIR=`\"\" + ${PYTHON_INSTALL_PATH} + \"`\"'\"; ` - } else { ` - ${INSTALLARGS} = \"'/quiet InstallAllUsers=1 PrependPath=1 TargetDir=`\"\" + ${PYTHON_INSTALL_PATH} + \"`\"'\"; ` - } ` - choco install python${PYTHON_MAJOR_VERSION} --yes --version=\"${ENV:PYTHON_VERSION}\" --override --installargs=${INSTALLARGS}; ` - if (-not $?) {exit 1} - -# Verify -RUN $python_relative_ver = (& python --version 2>&1).ToString().Trim().Split(" ")[1]; ` - $python_explicit_ver = (& C:\python\python.exe --version 2>&1).ToString().Trim().Split(" ")[1]; ` - if (-not $?) {exit 1}; ` - $python_relative_ver = (& python --version 2>&1).ToString().Trim().Split(" ")[1]; ` - $python_explicit_ver = (& C:\python\python.exe --version 2>&1).ToString().Trim().Split(" ")[1]; ` - if (-not ($python_explicit_ver -eq $python_relative_ver -and $python_explicit_ver -eq ${ENV:PYTHON_VERSION})) {exit 1} - - -# ------------------------------------------------------------------------------------------------------------ -# Git Bash (git for windows) -# -RUN choco install git --yes --params "/GitAndUnixToolsOnPath" - - -# ------------------------------------------------------------------------------------------------------------ -# Cleanup -# -RUN choco install --yes choco-cleaner; ` - C:\ProgramData\chocolatey\bin\choco-cleaner.bat; ` - choco uninstall --yes choco-cleaner - - -COPY entrypoint.ps1 /entrypoint.ps1 - -ENTRYPOINT ["powershell.exe", "-NoLogo", "-ExecutionPolicy", "ByPass", "-File", "/entrypoint.ps1"] diff --git a/.github/docker/rez-win-py/entrypoint.ps1 b/.github/docker/rez-win-py/entrypoint.ps1 deleted file mode 100644 index 0ac074073..000000000 --- a/.github/docker/rez-win-py/entrypoint.ps1 +++ /dev/null @@ -1,55 +0,0 @@ -# -# Entrypoint that installs given python version and runs tests. -# - -# Stop on errors; .exe has to be checked manually -Set-StrictMode -Version latest -$ErrorActionPreference = "Stop" - -# Fixes encoding issue on Windows 10 local docker run. -# -${ENV:PYTHONIOENCODING} = "UTF-8" - -# Print name of image being run, for debugging purposes -Write-Output "Using docker image ${ENV:_IMAGE_NAME}" - -# Verify Python -Write-Output "python found at $((Get-Command python).Path)" -python --version -if (-not $?) {exit 1} - -# Verify cmake -Write-Output "cmake found at $((Get-Command cmake).Path)" -cmake.exe --version -if (-not $?) {exit 1} - -# Verify pwsh -Write-Output "pwsh found at $((Get-Command pwsh).Path)" -pwsh --version -if (-not $?) {exit 1} - -# Verify git -Write-Output "git found at $((Get-Command git).Path)" -git --version -if (-not $?) {exit 1} - -# Verify git-bash -Write-Output "bash (via Git for windows) found at $((Get-Command bash).Path)" -bash --version -if (-not $?) {exit 1} - -# Install rez -# Note that the workflow's checkout has been bind mounted to /checkout -mkdir installdir -python .\checkout\install.py installdir -if (-not $?) {exit 1} - -# Install pytest for better rez-selftest output -.\installdir\Scripts\rez\rez-python -m pip install pytest-cov -.\installdir\Scripts\rez\rez-python -m pip install parameterized - -# Run Rez Tests -.\installdir\Scripts\rez\rez-selftest.exe -v - -# Pass on exit code to runner -exit $LASTEXITCODE diff --git a/.github/scripts/store_benchmark.py b/.github/scripts/store_benchmark.py index 79fec9eab..d8cbb8c1e 100644 --- a/.github/scripts/store_benchmark.py +++ b/.github/scripts/store_benchmark.py @@ -59,7 +59,8 @@ def store_result(): destdir = '-'.join(( time.strftime("%Y.%m.%d"), "%d.%d" % sys.version_info[:2], - _rez_version + # TODO: We could read the version from summary.json... + _rez_version, )) destpath = os.path.join(artifacts_dir, destdir) diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml index 3f60adcf5..a91c4b641 100644 --- a/.github/workflows/benchmark.yaml +++ b/.github/workflows/benchmark.yaml @@ -5,27 +5,24 @@ name: benchmark on: release: types: [released] + pull_request: + types: [opened, synchronize, reopened, labeled] jobs: run_benchmark: name: run_benchmark runs-on: ubuntu-latest + if: ${{ github.event_name == 'release' || contains(github.event.pull_request.labels.*.name, 'run-benchmarks') }} + strategy: matrix: python-version: - - '2.7' - '3.7' - # without this, we're sometimes getting at the end of this job: - # '[error] The operation was canceled'. - # Do we hit a resource limit? - # - max-parallel: 1 - steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup python ${{ matrix.python-version }} uses: ./.github/actions/setup-python @@ -37,11 +34,6 @@ jobs: run: | mkdir ./installdir - if [[ "${{ matrix.python-version }}" == "2.7" ]]; then - eval "$(conda shell.bash hook)" - conda activate python - fi - python ./install.py ./installdir - name: Run Benchmark @@ -53,13 +45,9 @@ jobs: - name: Validate Result run: | - if [[ "${{ matrix.python-version }}" == "2.7" ]]; then - eval "$(conda shell.bash hook)" - conda activate python - fi python ./.github/scripts/validate_benchmark.py - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: name: "benchmark-result-${{ matrix.python-version }}" path: ./out @@ -72,26 +60,20 @@ jobs: strategy: matrix: python-version: - - '2.7' - '3.7' # so we don't have jobs trying to push to git at the same time max-parallel: 1 steps: - - name: Setup python ${{ matrix.python-version }} - uses: ./.github/actions/setup-python - with: - python-version: ${{ matrix.python-version }} - os: ubuntu - - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 with: name: "benchmark-result-${{ matrix.python-version }}" path: . - - name: Checkout - uses: actions/checkout@v3 + - name: Checkout (release) + uses: actions/checkout@v4 + if: ${{ github.event_name =='release' }} with: ref: main path: src @@ -100,7 +82,20 @@ jobs: # protected branch (main) from this workflow. # See https://github.community/t/how-to-push-to-protected-branches-in-a-github-action/16101/14 # - token: "${{ secrets.GH_ACTION_TOKEN }}" + # Disable for now until we find a better solution. + # token: "${{ secrets.GH_ACTION_TOKEN }}" + + - name: Checkout (pr) + uses: actions/checkout@v4 + if: ${{ github.event_name !='release' }} + with: + path: src + + - name: Setup python ${{ matrix.python-version }} + uses: ./src/.github/actions/setup-python + with: + python-version: ${{ matrix.python-version }} + os: ubuntu-latest # Note failing due to # https://github.com/actions/virtual-environments/issues/675 @@ -114,26 +109,32 @@ jobs: - name: Store Benchmark Result run: | - if [[ "${{ matrix.python-version }}" == "2.7" ]]; then - eval "$(conda shell.bash hook)" - conda activate python - fi python ./.github/scripts/store_benchmark.py working-directory: src - - name: Setup git config + - name: Create summary run: | - git config user.name 'github-actions[bot]' - git config user.email 'github-actions[bot]@users.noreply.github.com' + echo '
' >> $GITHUB_STEP_SUMMARY + echo 'Results' >> $GITHUB_STEP_SUMMARY + cat metrics/benchmarking/RESULTS.md >> $GITHUB_STEP_SUMMARY + echo '
' >> $GITHUB_STEP_SUMMARY working-directory: src - - name: Git commit and push - run: | - if [[ "$(git status --porcelain)" == "" ]]; then - echo "Nothing new to commit" - else - git add --all - git commit -m "Generated from GitHub "${{ github.workflow }}" Workflow" - git push origin main - fi - working-directory: src + # - name: Setup git config + # if: ${{ github.event_name == 'release' }} + # run: | + # git config user.name 'github-actions[bot]' + # git config user.email 'github-actions[bot]@users.noreply.github.com' + # working-directory: src + + # - name: Git commit and push + # if: ${{ github.event_name == 'release' }} + # run: | + # if [[ "$(git status --porcelain)" == "" ]]; then + # echo "Nothing new to commit" + # else + # git add --all + # git commit -m "Generated from GitHub "${{ github.workflow }}" Workflow" + # git push origin main + # fi + # working-directory: src diff --git a/.github/workflows/copyright.yaml b/.github/workflows/copyright.yaml index cbff3535f..2c4d48ae4 100644 --- a/.github/workflows/copyright.yaml +++ b/.github/workflows/copyright.yaml @@ -17,10 +17,10 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: 3 diff --git a/.github/workflows/core.yaml b/.github/workflows/core.yaml deleted file mode 100644 index f3a4719ea..000000000 --- a/.github/workflows/core.yaml +++ /dev/null @@ -1,84 +0,0 @@ -# Test core parts of the package, that are not: -# * Shell dependent; -# * Build-system dependent; -# * Platform dependent. -# -name: core -on: - pull_request: - paths: - - 'src/**' - - '.github/workflows/core.yaml' - - '!src/rez/utils/_version.py' - - '!**.md' - push: - paths: - - 'src/**' - - '.github/workflows/core.yaml' - - '!src/rez/utils/_version.py' - - '!**.md' - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - main: - name: main - runs-on: ubuntu-latest - - strategy: - matrix: - python-version: - - '2.7' - - '3.7' - fail-fast: false - - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Setup python ${{ matrix.python-version }} - uses: ./.github/actions/setup-python - with: - python-version: ${{ matrix.python-version }} - os: ubuntu-latest - - - name: Install Rez - shell: bash - run: | - set -ex - mkdir ./installdir - - if [[ "${{ matrix.python-version }}" == "2.7" ]]; then - eval "$(conda shell.bash hook)" - conda activate python - fi - - python --version - python ./install.py ./installdir - - - name: Install Rez test dependencies - run: | - ./installdir/bin/rez/rez-python -m pip install pytest-cov - - # TODO: Add a --core rez-selftest option. Some test suites (eg test_context) - # have some 'core' parts (eg not reliant on a shell). It would be good to just - # run those parts when --core is present, rather than skipping the entire - # test class. This will be easier once ported to pytest. - - name: Run Rez Tests - run: | - ./installdir/bin/rez/rez-selftest -v \ - --config \ - --copy_package \ - --formatter \ - --imports \ - --packages \ - --package_filter \ - --packages_order \ - --resources_ \ - --rex \ - --schema \ - --solver \ - --version diff --git a/.github/workflows/flake8.yaml b/.github/workflows/flake8.yaml index ccf04c336..14221cfcb 100644 --- a/.github/workflows/flake8.yaml +++ b/.github/workflows/flake8.yaml @@ -26,10 +26,10 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: 3.7 diff --git a/.github/workflows/installation.yaml b/.github/workflows/installation.yaml index 567a28355..4eb671132 100644 --- a/.github/workflows/installation.yaml +++ b/.github/workflows/installation.yaml @@ -21,16 +21,9 @@ jobs: strategy: fail-fast: false matrix: - os: - - ubuntu-latest - - macos-latest - - windows-2019 - python-version: - - '2.7' - - '3.7' - method: - - 'install' - - 'pip' + os: ['ubuntu-latest', 'macos-latest', 'windows-latest'] + python-version: ['2.7', '3.7', '3.8', '3.9', '3.10', '3.11'] + method: ['install' ,'pip'] include: # ubuntu @@ -83,7 +76,7 @@ jobs: fi pip install --target ~/rez . # windows - - os: windows-2019 + - os: windows-latest method: install REZ_SET_PATH_COMMAND: '$env:PATH="$env:PATH;C:\ProgramData\rez\Scripts\rez"' REZ_INSTALL_COMMAND: | @@ -92,7 +85,7 @@ jobs: conda activate python } python ./install.py C:\ProgramData\rez - - os: windows-2019 + - os: windows-latest method: pip REZ_SET_PATH_COMMAND: '[System.Environment]::SetEnvironmentVariable("PATH","$env:PATH;C:\ProgramData\rez\bin"); $env:PYTHONPATH="$env:PYTHONPATH;C:\ProgramData\rez"' REZ_INSTALL_COMMAND: | @@ -102,8 +95,12 @@ jobs: } pip install --target C:\ProgramData\rez . + exclude: + - method: install + python-version: '2.7' + steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Setup python ${{ matrix.python-version }} uses: ./.github/actions/setup-python diff --git a/.github/workflows/mac.yaml b/.github/workflows/mac.yaml deleted file mode 100644 index 97b2442d6..000000000 --- a/.github/workflows/mac.yaml +++ /dev/null @@ -1,72 +0,0 @@ -name: mac -on: - pull_request: - paths: - - 'src/**' - - '.github/workflows/mac.yaml' - - '!src/rez/utils/_version.py' - - '!**.md' - push: - paths: - - 'src/**' - - '.github/workflows/mac.yaml' - - '!src/rez/utils/_version.py' - - '!**.md' - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - main: - name: main - runs-on: macos-${{ matrix.os-version }} - - strategy: - matrix: - os-version: - - '11' - python-version: - - '2.7' - - '3.7' - fail-fast: false - - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Setup python ${{ matrix.python-version }} - uses: ./.github/actions/setup-python - with: - python-version: ${{ matrix.python-version }} - os: macos-latest - - - name: Verify cmake - run: | - cmake --version - - - name: Verify pwsh - run: | - pwsh --version - - - name: Install Rez - run: | - if [[ "${{ matrix.python-version }}" == "2.7" ]]; then - eval "$(conda shell.bash hook)" - conda activate python - fi - mkdir ./installdir - python --version - python ./install.py ./installdir - - - name: Install Rez test dependencies - run: | - ./installdir/bin/rez/rez-python -m pip install pytest-cov - ./installdir/bin/rez/rez-python -m pip install parameterized - - - name: Run Rez Tests - run: | - ./installdir/bin/rez/rez-selftest -v - env: - _REZ_ENSURE_TEST_SHELLS: sh,csh,bash,tcsh,zsh,pwsh diff --git a/.github/workflows/pypi.yaml b/.github/workflows/pypi.yaml index fdae3bca5..13566cc7b 100644 --- a/.github/workflows/pypi.yaml +++ b/.github/workflows/pypi.yaml @@ -5,28 +5,33 @@ on: jobs: publish: - name: Publish to PyPi + name: Publish to PyPI runs-on: ubuntu-latest + permissions: + # IMPORTANT: this permission is mandatory for trusted publishing + id-token: write + steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: - python-version: 3 + python-version: 3.11 - name: Install Dependencies run: | - pip install wheel + pip install build - name: Build rez run: | - python setup.py sdist bdist_wheel + python -m build --sdist --wheel --outdir dist . - - name: Upload to PyPi + # Note that we don't need credentials. + # We rely on https://docs.pypi.org/trusted-publishers/. + - name: Upload to PyPI uses: pypa/gh-action-pypi-publish@release/v1 with: - user: __token__ - password: '${{ secrets.PYPI_API_TOKEN }}' + packages-dir: dist diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml new file mode 100644 index 000000000..dc4fc0f0b --- /dev/null +++ b/.github/workflows/tests.yaml @@ -0,0 +1,80 @@ +name: tests +on: + pull_request: + paths: + - 'src/**' + - '.github/workflows/tests.yaml' + - '!src/rez/utils/_version.py' + - '!**.md' + push: + paths: + - 'src/**' + - '.github/workflows/tests.yaml' + - '!src/rez/utils/_version.py' + - '!**.md' + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + core: + name: Tests + + runs-on: ${{ matrix.os }} + + strategy: + matrix: + os: ['macos-latest', 'ubuntu-latest', 'windows-latest'] + python-version: ['3.7', '3.8', '3.9', '3.10', '3.11'] + include: + - os: macos-latest + shells: 'sh,csh,bash,tcsh,zsh,pwsh' + rez-path: /installdir/bin/rez + - os: ubuntu-latest + shells: 'sh,csh,bash,tcsh,zsh,pwsh' + rez-path: /installdir/bin/rez + - os: windows-latest + shells: 'cmd,pwsh,gitbash' + rez-path: \installdir\Scripts\rez + + fail-fast: false + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install test system dependencies (linux) + if: ${{ startsWith(matrix.os, 'ubuntu-') }} + run: | + set -ex + sudo apt-get update + + cmake --version + pwsh --version + + sudo apt-get install -y csh tcsh zsh + + - name: Install rez + run: python ./install.py ./installdir + + - name: Setup environment variables + shell: bash + env: + _GH_REZ_INSTALL_PATH: ${{ github.workspace }}${{ matrix.rez-path }} + run: | + echo "$_GH_REZ_INSTALL_PATH" >> $GITHUB_PATH + + - name: Install test dependencies + run: rez-python -m pip install pytest-cov parameterized + + - name: Run tests + run: rez-selftest -v + env: + _REZ_ENSURE_TEST_SHELLS: ${{ matrix.shells }} diff --git a/.github/workflows/ubuntu.yaml b/.github/workflows/ubuntu.yaml deleted file mode 100644 index 266059510..000000000 --- a/.github/workflows/ubuntu.yaml +++ /dev/null @@ -1,89 +0,0 @@ -name: ubuntu -on: - pull_request: - paths: - - 'src/**' - - '.github/workflows/ubuntu.yaml' - - '!src/rez/utils/_version.py' - - '!**.md' - push: - paths: - - 'src/**' - - '.github/workflows/ubuntu.yaml' - - '!src/rez/utils/_version.py' - - '!**.md' - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - main: - name: main - runs-on: ubuntu-${{ matrix.os-version }} - - strategy: - matrix: - os-version: - - '20.04' - - '22.04' - python-version: - - '2.7' - - '3.7' - fail-fast: false - - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Setup python ${{ matrix.python-version }} - uses: ./.github/actions/setup-python - with: - python-version: ${{ matrix.python-version }} - os: ubuntu - - - name: apt-get update - run: | - sudo apt-get update - - - name: Verify cmake - run: | - cmake --version - - - name: Verify pwsh - run: | - pwsh --version - - - name: Install csh - run: | - sudo apt-get install -y csh - - - name: Install tcsh - run: | - sudo apt-get install -y tcsh - - - name: Install zsh - run: | - sudo apt-get install -y zsh - - - name: Install Rez - run: | - if [[ "${{ matrix.python-version }}" == "2.7" ]]; then - eval "$(conda shell.bash hook)" - conda activate python - fi - mkdir ./installdir - python --version - python ./install.py ./installdir - - - name: Install Rez test dependencies - run: | - ./installdir/bin/rez/rez-python -m pip install pytest-cov - ./installdir/bin/rez/rez-python -m pip install parameterized - - - name: Run Rez Tests - run: | - ./installdir/bin/rez/rez-selftest -v - env: - _REZ_ENSURE_TEST_SHELLS: sh,csh,bash,tcsh,zsh,pwsh diff --git a/.github/workflows/wiki.yaml b/.github/workflows/wiki.yaml deleted file mode 100644 index 99016cf3d..000000000 --- a/.github/workflows/wiki.yaml +++ /dev/null @@ -1,63 +0,0 @@ -name: wiki -on: - release: - types: [released] - push: - paths: - - '.github/workflows/wiki.yaml' - - 'wiki/**' - - src/rez/rezconfig.py - -jobs: - build: - name: Build Wiki Artifact - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 # required to generate credits list - - - uses: actions/setup-python@v4 - with: - python-version: 3.7 - - - name: Build Wiki - working-directory: wiki - run: | - python generate-wiki.py \ - --github-repo="${{ github.repository }}" \ - --out="${{ github.workspace }}/out" - - - uses: actions/upload-artifact@v3 - with: - name: wiki-markdown - path: out - - publish: - name: Publish to GitHub Wiki - runs-on: ubuntu-latest - needs: build - if: github.event_name == 'release' - - steps: - - name: Setup git config - run: | - git config --global user.name "github.com/${{ github.actor }}" - git config --global user.email "${{ github.actor }}@${{ github.sha }}" - - - name: Clone latest wiki repo - run: | - git clone https://${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }}.wiki.git . - - - uses: actions/download-artifact@v3 - with: - name: wiki-markdown - path: . - - - name: Push wiki updates to git - run: | - git add --all - ( git commit -m "Generated from GitHub ${{ github.workflow }} Workflow" \ - && git push origin main \ - ) || echo "Nothing new to commit" diff --git a/.github/workflows/windows.yaml b/.github/workflows/windows.yaml deleted file mode 100644 index 3f58ce5a6..000000000 --- a/.github/workflows/windows.yaml +++ /dev/null @@ -1,356 +0,0 @@ -# -# This workflow builds base and python-version-specific docker images if -# required, then runs the rez tests in the python image. -# -# The images are tagged with a hash of the relevant sourcefile contents (eg -# Dockerfile). Images from the 'AcademySoftwareFoundation/rez' GitHub container registry will be used -# if they are up-to-date. -# -# If images require updating, the necessary images will be -# created and will be stored in the GitHub Container registry. The images -# will always be stored in the current's repo namespace (ghcr.io//). -# -# This approach ensures that image rebuilds are avoided when possible, but are -# supported from forks who may have made changes to docker-related -# source (such as Dockerfile) and need this workflow to run. -# ---- -name: windows - -on: - pull_request: - paths: - - 'src/**' - - '.github/workflows/windows.yaml' - - '.github/docker/rez-win-base/**' - - '.github/docker/rez-win-py/**' - - '!src/rez/utils/_version.py' - - '!**.md' - push: - paths: - - 'src/**' - - '.github/workflows/windows.yaml' - - '.github/docker/rez-win-base/**' - - '.github/docker/rez-win-py/**' - - '!src/rez/utils/_version.py' - - '!**.md' - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - # The default namespace that the jobs will look at or use. - PUB_NAMESPACE: ghcr.io/academysoftwarefoundation/rez - - # We don't take changes to this workfile yaml into account when determining - # image tags, because changes here very rarely cause changes to the images, - # and this causes lots of unnecessary image rebuilds. On the offchance a - # change is made here that _does_ affect the images, increment this value - IMAGE_TAG_SALT: 3 - -jobs: - - # image tags are based on sourcefile contents - image_tags: - name: Calculate image tags - runs-on: ubuntu-latest - - outputs: - base: ${{ steps.base.outputs.tag }} - py: ${{ steps.py.outputs.tag }} - - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Determine base image tag - id: base - run: | - tag=$( \ - echo "${{ hashFiles('.github/docker/rez-win-base/*') }}${IMAGE_TAG_SALT}" \ - | md5sum - \ - | awk '{print $1}' \ - ) - echo "base tag is ${tag}" - echo "::set-output name=tag::${tag}" - - - name: Determine python image tag - id: py - run: | - tag=$( \ - echo "${{ hashFiles('.github/docker/rez-win-base/*', '.github/docker/rez-win-py/*') }}${IMAGE_TAG_SALT}" \ - | md5sum - \ - | awk '{print $1}' \ - ) - echo "py tag is ${tag}" - echo "::set-output name=tag::${tag}" - - # note that we don't track staleness on a per-matrix-entry basis. Github actions - # job outputs wrt matrix is problematic and frankly not worth the hassle. - public_py_image: - name: Check for up-to-date public py image - runs-on: windows-${{ matrix.os-version }} - needs: image_tags - - strategy: - fail-fast: false - matrix: - os-version: - - '2019' - py-version: - - '2.7.17' - - '3.7.5' - - outputs: - namespace: ${{ steps.inspect.outputs.namespace }} - needs_rebuild: ${{ steps.inspect.outputs.needs_rebuild }} - - steps: - - name: Inspect public py image - id: inspect - run: | - # Try to get the image from the pub namepsace first. - $pub_namespace = "${Env:PUB_NAMESPACE}" - $docker_image = "${pub_namespace}/rez-win-${{ matrix.os-version }}-py-${{ matrix.py-version }}:${{ needs.image_tags.outputs.py }}".ToLower() - - Write-Output "Inspecting image ${docker_image}..." - $ErrorActionPreference = "Continue" - docker manifest inspect $docker_image *>$null || Write-Output "(no such image)" - $ErrorActionPreference = "Stop" - - if ($LastExitCode -eq 0) { - Write-Output "Found ${docker_image}" - Write-Output "::set-output name=namespace::${pub_namespace}" - Write-Output "::set-output name=needs_rebuild::false" - } - else { - - Write-Output "${docker_image} not found" - - # Image not found in pub namespace, look into the current's - # repo registry or in the originating repo when the workflow is - # triggered from a PR. - if ('${{ github.event_name }}' -eq 'pull_request') { - # This is quite important since workflows don't have write - # permissions when the source branch is from a fork. - $github_namespace = "ghcr.io/${{ github.event.pull_request.head.repo.full_name }}" - } - else { - $github_namespace = "ghcr.io/${{ github.repository }}" - } - $docker_image = "${github_namespace}/rez-win-${{ matrix.os-version }}-py-${{ matrix.py-version }}:${{ needs.image_tags.outputs.py }}".ToLower() - - Write-Output "Inspecting image ${docker_image}..." - $ErrorActionPreference = "Continue" - docker manifest inspect $docker_image *>$null || Write-Output "(no such image)" - $ErrorActionPreference = "Stop" - - # Inform the next jobs that they need to use the "private" - # registry. - Write-Output "::set-output name=namespace::${github_namespace}" - - if ($LastExitCode -ne 0) { - # Well, no images found at all! We will need to build the images. - Write-Output "${docker_image} not found" - Write-Output "::set-output name=needs_rebuild::true" - } else { - Write-Output "Found ${docker_image}" - Write-Output "::set-output name=needs_rebuild::false" - } - } - - exit 0 - - base_image: - name: Build base docker image if required - runs-on: windows-${{ matrix.os-version }} - needs: - - image_tags - - public_py_image - - if: needs.public_py_image.outputs.needs_rebuild == 'true' - - strategy: - fail-fast: false - matrix: - # The windows version has to match the host system. - # 1809 -> 10.0.17763.805 -> windows-2019 - # Compare: https://hub.docker.com/_/microsoft-windows-servercore - include: - - os-version: '2019' - windows-version: '1809-amd64' - - steps: - - name: Fail with summary - if: (github.event_name == 'pull_request') && (github.event.pull_request.head.repo.full_name != github.repository) - shell: bash - run: | - echo '# Action required! - - This branch is coming from a fork and the appropriate docker images were - not found in `${{ needs.public_py_image.outputs.namespace }}`. - - Please ensure that you run the workflow in your fork. Once this is done, - please let the reviewers know so that they can re-run the workflow in - the context of the PR.' > $GITHUB_STEP_SUMMARY - - exit 1 - - - name: Set job vars - id: vars - run: | - $docker_image = "${{ needs.public_py_image.outputs.namespace }}/rez-win-${{ matrix.os-version }}-base:${{ needs.image_tags.outputs.base }}".ToLower() - - Write-Output "::set-output name=docker_image::${docker_image}" - - - name: Login to docker repository - run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u $ --password-stdin - - - name: Inspect base image - id: inspect - run: | - Write-Output "Inspecting image ${{ steps.vars.outputs.docker_image }}..." - $ErrorActionPreference = "Continue" - docker manifest inspect ${{ steps.vars.outputs.docker_image }} *>$null || Write-Output "(no such image)" - $ErrorActionPreference = "Stop" - - if ($LastExitCode -ne 0) { - Write-Output "::set-output name=image_exists::false" - } - exit 0 - - - name: Checkout - if: steps.inspect.outputs.image_exists == 'false' - uses: actions/checkout@v2 - - - name: Build base image - if: steps.inspect.outputs.image_exists == 'false' - run: | - Write-Output "Building image ${{ steps.vars.outputs.docker_image }}..." - cd .github\docker\rez-win-base - docker build ` - --tag ${{ steps.vars.outputs.docker_image }} ` - --build-arg WINDOWS_VERSION="${{ matrix.windows-version }}" ` - . - - - name: Push base image - if: steps.inspect.outputs.image_exists == 'false' - run: | - Write-Output "Pushing image ${{ steps.vars.outputs.docker_image }}..." - docker push ${{ steps.vars.outputs.docker_image }} - - py_image: - name: Build py docker image if required - runs-on: windows-${{ matrix.os-version }} - needs: - - image_tags - - base_image - - public_py_image - - strategy: - fail-fast: false - matrix: - os-version: - - '2019' - py-version: - - '2.7.17' - - '3.7.5' - - steps: - - name: Set job vars - id: vars - run: | - # When publishing the images, we always publish in the current repo's package registry - $base_docker_image = "ghcr.io/${{ github.repository }}/rez-win-${{ matrix.os-version }}-base:${{ needs.image_tags.outputs.base }}".ToLower() - $docker_image = "ghcr.io/${{ github.repository }}/rez-win-${{ matrix.os-version }}-py-${{ matrix.py-version }}:${{ needs.image_tags.outputs.py }}".ToLower() - - Write-Output "::set-output name=base_docker_image::${base_docker_image}" - Write-Output "::set-output name=docker_image::${docker_image}" - - - name: Login to docker repository - run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u $ --password-stdin - - - name: Inspect py image - id: inspect - run: | - Write-Output "Inspecting image ${{ steps.vars.outputs.docker_image }}..." - $ErrorActionPreference = "Continue" - docker manifest inspect ${{ steps.vars.outputs.docker_image }} *>$null || Write-Output "(no such image)" - $ErrorActionPreference = "Stop" - - if ($LastExitCode -ne 0) { - Write-Output "::set-output name=image_exists::false" - } - exit 0 - - - name: Checkout - if: steps.inspect.outputs.image_exists == 'false' - uses: actions/checkout@v2 - - - name: Pull base image - if: steps.inspect.outputs.image_exists == 'false' - run: | - Write-Output "Pulling base image ${{ steps.vars.outputs.base_docker_image }}..." - docker pull ${{ steps.vars.outputs.base_docker_image }} - - - name: Build py image - if: steps.inspect.outputs.image_exists == 'false' - run: | - Write-Output "Building image ${{ steps.vars.outputs.docker_image }}..." - cd .github\docker\rez-win-py - - docker build ` - --tag ${{ steps.vars.outputs.docker_image }} ` - --build-arg BASE_IMAGE_NAME="${{ steps.vars.outputs.base_docker_image }}" ` - --build-arg IMAGE_NAME="${{ steps.vars.outputs.docker_image }}" ` - --build-arg PYTHON_VERSION="${{ matrix.py-version }}" ` - . - - - name: Push py image - if: steps.inspect.outputs.image_exists == 'false' - run: | - Write-Output "Pushing image ${{ steps.vars.outputs.docker_image }}..." - docker push ${{ steps.vars.outputs.docker_image }} - - main: - name: Run rez tests - runs-on: windows-${{ matrix.os-version }} - needs: - - image_tags - - public_py_image - - py_image - - # Forces this job to run even if needed jobs are skipped but not when the workflow is cancelled or failed. - if: (success() || needs.py_image.result == 'skipped') && !cancelled() && !failure() - - strategy: - fail-fast: false - matrix: - os-version: - - '2019' - py-version: - - '2.7.17' - - '3.7.5' - - steps: - - name: Set job vars - id: vars - run: | - $docker_image = "${{ needs.public_py_image.outputs.namespace }}/rez-win-${{ matrix.os-version }}-py-${{ matrix.py-version }}:${{ needs.image_tags.outputs.py }}".ToLower() - - Write-Output "::set-output name=docker_image::${docker_image}" - Write-Output "Using image ${docker_image}..." - - - name: Checkout - uses: actions/checkout@v2 - - - name: Pull py image - run: | - docker pull ${{ steps.vars.outputs.docker_image }} - - - name: Run Docker image (installs and tests rez) - run: docker run --mount type=bind,src=$pwd,dst=C:\checkout,readonly ${{ steps.vars.outputs.docker_image }} - env: - _REZ_ENSURE_TEST_SHELLS: cmd,pwsh,gitbash diff --git a/.gitignore b/.gitignore index 3995ffa13..1cfc09222 100644 --- a/.gitignore +++ b/.gitignore @@ -18,4 +18,5 @@ __pycache__ .vscode/ .venv/ docs/source/api/ -docs/source/commands/ \ No newline at end of file +docs/source/commands/ +__tests_pkg_repo/ diff --git a/.mailmap b/.mailmap new file mode 100644 index 000000000..fd16d38b8 --- /dev/null +++ b/.mailmap @@ -0,0 +1,176 @@ +# This file establishes email equivalences so we can resolve what look like +# multiple authors, but actually are the same author who has used multiple +# emails over the course of their involvement with the project. +# +# The format is any of the following: +# +# CANONICAL-NAME +# CANONICAL-NAME alternate-name +# +# You can check for duplicates with this command: +# git shortlog -sne --all +# That command (and others) will use this file to collapse the duplicates. +# +# If you see any duplicates we don't account for here, or if you look at your +# own entry here and want a different name or email to be your canonical one +# (we may not have guessed correctly and matched your preferences), please +# file a PR with the edits to this file. + +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns +Allan Johns <(no author)@e6eb1edf-3a88-c135-55be-1f5b7bf7ed03> +Allan Johns RACHEL JOHNS +Allan Johns +Allan Johns ajohns +Allan Johns allan johns +Allan Johns nerdvegas + +Alexandra Lefève-Gourmelon Alexandra Lefeve + +Ben Dickson dbr + +Blazej Floch +Blazej Floch + +Brendan Abel <007brendan@gmail.com> +Brendan Abel <007brendan@gmail.com> +Brendan Abel <007brendan@gmail.com> + +David Lai davidlatwe + +Fabio Piparo +Fabio Piparo + +Hal <13111745+loonghao@users.noreply.github.com> longhao + +Jason Scott + +Jean-Christophe Morin <38703886+JeanChristopheMorinPerso@users.noreply.github.com> +Jean-Christophe Morin + +Joseph Yu +Joseph Yu + +Mark Streatfield +Mark Streatfield + +Stephen Mackenzie maxnbk +Stephen Mackenzie +Stephen Mackenzie +Stephen Mackenzie + +Michael Morehouse + +Morné Chamberlain + +Paul Molodowitch + +Sylvain Maziere predat +Sylvain Maziere + +Renaud Lessard Larouche +Renaud Lessard Larouche + +Sebastian Kral +Sebastian Kral skral + +Thomas Mansencal + +Thorsten Kaufmann diff --git a/ASWF/TSC/meeting-notes/2023-07-20.md b/ASWF/TSC/meeting-notes/2023-07-20/notes.md similarity index 99% rename from ASWF/TSC/meeting-notes/2023-07-20.md rename to ASWF/TSC/meeting-notes/2023-07-20/notes.md index 4d56ef50c..9304b6368 100644 --- a/ASWF/TSC/meeting-notes/2023-07-20.md +++ b/ASWF/TSC/meeting-notes/2023-07-20/notes.md @@ -134,7 +134,7 @@ * Several people got interested by the openmoonray rez package and some of what it had inside it, so dreamworks graciously agreed to talk about some of what they do, how they do it, etc. * (JB): * I have some slides, manage our team, and we have a few people here to answer questions and so forth. - * (Demo) + * Presentation: [Slides](presentation.pdf). * (Q&A with Joel Pollock, Matthew Low, Ibrahim Sani Kache) ### Ending remarks diff --git a/ASWF/TSC/meeting-notes/2023-07-20/presentation.pdf b/ASWF/TSC/meeting-notes/2023-07-20/presentation.pdf new file mode 100644 index 000000000..292133cb1 Binary files /dev/null and b/ASWF/TSC/meeting-notes/2023-07-20/presentation.pdf differ diff --git a/ASWF/TSC/meeting-notes/2023-08-17/notes.md b/ASWF/TSC/meeting-notes/2023-08-17/notes.md new file mode 100644 index 000000000..dcb6eebc1 --- /dev/null +++ b/ASWF/TSC/meeting-notes/2023-08-17/notes.md @@ -0,0 +1,91 @@ +# Rez TSC Meeting Notes - 2023-08-17 + +:movie_camera::scroll: Recording: https://zoom.us/rec/share/2OLhyMTtN6ybQaodW1CUMPCvCRLGUNxsUlb3WcqBCCwlAWV3ELw8xjf53WzkVx-a.nRR2z27w-uJ1rJ5B + +## Attendance + +* Host: Jean-Christophe Morin +* Secretary: Jean-Christophe Morin +* TSC Attendees: + * [x] Brendan Abel - Walt-Disney Imagineering + * [x] Jean-Christophe Morin - Freelance + * [x] Stephen Mackenzie - NVIDIA + * [x] Thorsten Kaufmann - Mackevision / Accenture +* Other Attendees: + * Deke Kincaid (Digital Domain) + * Dhruv Govil (Apple) + * Erwan Leroy (Carfty Apes) + * Ibrahim Sani Kache (Dreamworks) + * Jason Scott (Pitch Black) + * Jeff Bradley (Dreamworks) + * Jonas Avrin + * John Riddle (Crafty Apes) + * Junko V. Igarashi (Crafty Apes) + +## Agenda +* Agenda Issue: https://github.com/AcademySoftwareFoundation/rez/issues/1513 +* ASWF: + * [x] Take ownership of https://rez.readthedocs.io [#1517](https://github.com/AcademySoftwareFoundation/rez/issues/1517) + * [x] TSC chair transition after Allan stepped down [#1519](https://github.com/AcademySoftwareFoundation/rez/issues/1519) +* [x] @herronelou presentation on the usage of rez at https://www.craftyapes.com/. +* [x] Named Variants proposal: Named Variants [#1503](https://github.com/AcademySoftwareFoundation/rez/discussions/1503) + +## Short Version / Decisions / Action Items / Important Links + +* Action Items: + * @AcademySoftwareFoundation/rez-tsc: Find a plan to unblock [#1503](https://github.com/AcademySoftwareFoundation/rez/discussions/1503). + +## Details + +### ASWF + +#### Take ownership of https://rez.readthedocs.io + +https://github.com/AcademySoftwareFoundation/rez/issues/1517 + +* JC: + * We discovered that https://rez.readthedocs.io was pointing at the source of an old fork of rez. + * Contacted the owner and he gave us admin access. + * We now have the ability to update the documentation. + * The current page is our own. We pushed an update using tip of the main branch. + +#### TSC chair transition after Allan stepped down + +https://github.com/AcademySoftwareFoundation/rez/issues/1519 + +* JC: + * Mosty administrative tasks. Should be done now. + * See the issue for more details. + +## Crafty Apes presentation + +* Erwan: + * Working at https://www.craftyapes.com/ as the Global Head of 2D. + * Want to present what we did at Crafty Apes and how we implemented rez in our pipeline. +* Presentation: [Slides](presentation.pdf). +* Q&A: See the recording (attached at the top of this page). +* Thanks a lot for this great presentation! + +## Named Variants proposal + +https://github.com/AcademySoftwareFoundation/rez/discussions/1503 + +* Dhruv: + * I made this proposal. + * What would the next steps be? + * The biggest contention point is how to store named variants in a way + that preserves the order and python 2.7 support. + * We could use a standard dict, but that won't isn't compatible with Python 2.7. + * And it would also only be compatible with 3.7+. +* Stephen: We could use this as an opportunity to drop support for Python 2.7. +* Jeff: As a studio that still supports Python 2.7. So we'll stick to the older rez version for a while until we can drop 2.7. +* JC: It's that it's always an option. Our users are not forced to upgrade to newer version of rez. And rez is failrly stable, so they can stay on an older version if they want to still use Python 2.7. +* Jason: Agree that studios can just pick an older version of rez if they want. +* Stephen: Studios can also install multiple versions of rez in parallel. +* Erwan: When they do so, they'll have to be carefull to not release packages that use features + that older versions of rez won't know how to deal with. +* JC: We'll need to solve that. THough, we have added new package definition fields in the past and we've never heard of failrues due to that. +* Brendan: rez will ignore attributes it doesn't know. But we still need to find a solution to + evolve the package definiton format. +* Dhruv: What are the next steps? +* JC: We should revive the discussion on GH. And the TSC will have to decide what to do to move things forward. diff --git a/ASWF/TSC/meeting-notes/2023-08-17/presentation.pdf b/ASWF/TSC/meeting-notes/2023-08-17/presentation.pdf new file mode 100644 index 000000000..5931c40f6 Binary files /dev/null and b/ASWF/TSC/meeting-notes/2023-08-17/presentation.pdf differ diff --git a/CHANGELOG.md b/CHANGELOG.md index 5325cdfff..c1c714408 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,98 @@ -# Change Log +# Change log + + + +## 2.114.1 (2023-12-09) +[Source](https://github.com/AcademySoftwareFoundation/rez/tree/2.114.1) | [Diff](https://github.com/AcademySoftwareFoundation/rez/compare/2.114.0...2.114.1) + +**Merged pull requests:** + +- Fix exception when deprecated settings appear in a user config [\#1595](https://github.com/AcademySoftwareFoundation/rez/pull/1595) ([JeanChristopheMorinPerso](https://github.com/JeanChristopheMorinPerso)) +- Add cross reference for package_cache_during_build in docs [\#1598](https://github.com/AcademySoftwareFoundation/rez/pull/1598) ([brycegbrazen](https://github.com/brycegbrazen)) + +## 2.114.0 (2023-11-23) +[Source](https://github.com/AcademySoftwareFoundation/rez/tree/2.114.0) | [Diff](https://github.com/AcademySoftwareFoundation/rez/compare/2.113.0...2.114.0) + +Probably the last release before 3.0.0 + +Rez installations now only support Python 3.7+. It is important to note that the API +can still be used with Python 2.7+ but this will be dropped in 3.0.0 + +### Features + +* New environment variable `REZ_LOG_DEPRECATION_WARNINGS` that will force + all deprecation warnings to be printed, ignoring `PYTHONWARNINGS` and custom + warning filters. Note that enabling this will forcefully load every + configuration file instead of loading them lazilly. +* This PR adds a new config variable called `error_on_missing_variant_requires` that controls what happens when + a variant lists missing packages in its requirements. + + By default, it is True and will continue the existing behaviour of erroring when it encounters + a variant with missing packages in its list of requirements. This means that if the first variant + encounters a missing package in its request, it will not continue even if the second variant can resolve. + + If it is disabled, it will print to stderr, treat the current phase as failed and continue on to the next phase. + If all variants fail, it will be the same as if no variants could resolve. + + This feature was added by [dgovil](https://github.com/dgovil) in [\#1550](https://github.com/AcademySoftwareFoundation/rez/pull/1550). + +### Deprecations + +We decided to deprecated some things that have been "deprecated" for a while but were never +officially marked as deprecated. We have prepared +[a guide](https://rez.readthedocs.io/en/stable/guides/prepare_for_3.html) +to help you navigate these deprecations. + +What follows is everything that is marked as deprecated with information on when +removal will happen or when defaults will change. + +- Configuration settings: + - [disable_rez_1_compatibility](https://rez.readthedocs.io/en/stable/configuring_rez.html#disable_rez_1_compatibility): + Will be removed in a yet to be determined future release. + - [debug_old_commands](https://rez.readthedocs.io/en/stable/configuring_rez.html#debug_old_commands): + Will be removed in 3.0.0. This setting is currently a no-op. + - [warn_old_commands](https://rez.readthedocs.io/en/stable/configuring_rez.html#warn_old_commands): + Will be removed in a yet to be determined future release. + - [error_old_commands](https://rez.readthedocs.io/en/stable/configuring_rez.html#error_old_commands): + Will be removed in a yet to be determined future release. + - [warn_commands2](https://rez.readthedocs.io/en/stable/configuring_rez.html#warn_commands2): + Will be removed in 3.0.0. This setting is currently a no-op. + - [error_commands2](https://rez.readthedocs.io/en/stable/configuring_rez.html#error_commands2): + Will be removed in 3.0.0. This setting is currently a no-op. + - [rez_1_cmake_variables](https://rez.readthedocs.io/en/stable/configuring_rez.html#rez_1_cmake_variables): + Will be removed in 3.0.0. + - [rez_1_environment_variables](https://rez.readthedocs.io/en/stable/configuring_rez.html#rez_1_environment_variables): + Will be removed in a yet to be determined future release. + - [rxt_as_yaml](https://rez.readthedocs.io/en/stable/configuring_rez.html#rxt_as_yaml): + Will be removed in 3.0.0. +- Modules: + - `rez.vendor.version`. Use `rez.version` instead. Will be removed in 3.0.0. + - `rez.packages_maker__`. Use `rez.packages_maker` instead. Will be removed in 3.0.0. + - `rez.package_resources_`. Use `rez.package_resources` instead. Will be removed in 3.0.0. + - `rez.packages_`. Use `rez.packages` instead. Will be removed in 3.0.0. +- CLI: + - rez-pip: The `--pip-version` argument is deprecated. Will be removed in 3.0.0. + - rez-search: The `--sort` argument is deprecated and has been a no-op for a while now. Will be removed in 3.0.0. +- API: + - The `isolate` keyword argument of the `rez.rex.RexExecutor.execute_code` method is now officially deprecated and will be removed in 3.0.0. + Instead of `executor.execute_code(..., isolate=True)`, use + ```python + with executor.reset_globals(): + executor.execute_code(...) + ``` +- Python: rez 3.0.0 will completely drop support for installing and using the rez API with Python 2. + +### Change of default values due to deprecations + +Some default values have changed: + +- [rez_1_cmake_variables](https://rez.readthedocs.io/en/stable/configuring_rez.html#rez_1_cmake_variables) + is now disabled by default. + +Some default values will change in 3.0.0: + +- [disable_rez_1_compatibility](https://rez.readthedocs.io/en/stable/configuring_rez.html#disable_rez_1_compatibility) will become enabled by default. +- [rez_1_environment_variables](https://rez.readthedocs.io/en/stable/configuring_rez.html#rez_1_environment_variables) will become disabled by default. ## 2.113.0 (2023-09-11) [Source](https://github.com/AcademySoftwareFoundation/rez/tree/2.113.0) | [Diff](https://github.com/AcademySoftwareFoundation/rez/compare/2.112.0...2.113.0) @@ -2257,42 +2351,42 @@ be on the lookout for unintended side effects and report them if they arise. ## 2.26.4 [[#562](https://github.com/AcademySoftwareFoundation/rez/pull/562)] Fixed Regression in 2.24.0 -#### Addressed Issues +### Addressed Issues * [#561](https://github.com/AcademySoftwareFoundation/rez/issues/561) timestamp not written to installed package ## 2.26.3 [[#560](https://github.com/AcademySoftwareFoundation/rez/pull/560)] Package.py permissions issue -#### Addressed Issues +### Addressed Issues * [#559](https://github.com/AcademySoftwareFoundation/rez/issues/559) package.py permissions issue -#### Notes +### Notes Fixes issue where installed `package.py` can be set to r/w for only the current user. ## 2.26.2 [[#557](https://github.com/AcademySoftwareFoundation/rez/pull/557)] Package Copy Fixes For Non-Varianted Packages -#### Addressed Issues +### Addressed Issues * [#556](https://github.com/AcademySoftwareFoundation/rez/issues/556) rez-cp briefly copies original package definition in non-varianted packages * [#555](https://github.com/AcademySoftwareFoundation/rez/issues/555) rez-cp inconsistent symlinking when --shallow=true * [#554](https://github.com/AcademySoftwareFoundation/rez/issues/554) rez-cp doesn't keep file metadata in some cases -#### Notes +### Notes There were various minor issues related to copying non-varianted packages. ## 2.26.1 [[#552](https://github.com/AcademySoftwareFoundation/rez/pull/552)] Bugfix in Package Copy -#### Addressed Issues +### Addressed Issues * [#551](https://github.com/AcademySoftwareFoundation/rez/issues/551) package copy fails if symlinks in root dir -#### Notes +### Notes This was failing when symlinks were present within a non-varianted package being copied. Now, these symlinks are retained in the target package, unless `--follow-symlinks` is specified. @@ -2300,19 +2394,19 @@ symlinks are retained in the target package, unless `--follow-symlinks` is speci ## 2.26.0 [[#550](https://github.com/AcademySoftwareFoundation/rez/pull/550)] Build System Detection Fixes -#### Addressed Issues +### Addressed Issues * [#549](https://github.com/AcademySoftwareFoundation/rez/issues/549) '--build-system' rez-build option not always available -#### Notes +### Notes To fix this issue: * The '--build-system' rez-build option is now always present. * To provide further control over the build system type, the package itself can now specify its build system - see https://github.com/AcademySoftwareFoundation/rez/wiki/Package-Definition-Guide#build_system -#### COMPATIBILITY ISSUE! +### COMPATIBILITY ISSUE! Unfortunately, the 'cmake' build system had its own '--build-system' commandline option also. This was possible because previous rez versions suppressed the standard '--build-system' option if only @@ -2322,7 +2416,7 @@ changed to '--cmake-build-system'**. ## 2.25.0 [[#548](https://github.com/AcademySoftwareFoundation/rez/pull/548)] Various Build-related issues -#### Addressed Issues +### Addressed Issues * [#433](https://github.com/AcademySoftwareFoundation/rez/issues/433): "package_definition_build_python_paths" defined paths are not available from top level in package.py @@ -2330,7 +2424,7 @@ changed to '--cmake-build-system'**. * [#416](https://github.com/AcademySoftwareFoundation/rez/issues/416): Need currently-building-variant build variables * [#547](https://github.com/AcademySoftwareFoundation/rez/issues/547): rez-cp follows symlinks within package payload -#### Notes +### Notes The biggest update in this release is the introduction of new variables accessible at early-bind time: building, build_variant_index and build_variant_requires. This allows you to do things like define @@ -2361,13 +2455,13 @@ intact - but hte previous behavior can still be accessed with the rez-cp --follo This release adds a new tool, rez-cp, for copying packages/variants from one package repository to another, with optional renaming/reversioning. The associated API can be found in src/package_copy.py. -#### Addressed Issues +### Addressed Issues * #541 * #510 * #477 -#### Notes +### Notes * Package definition file writes are now atomic; * private_build_requires is kept in installed/released packages; @@ -2377,22 +2471,22 @@ another, with optional renaming/reversioning. The associated API can be found in ## 2.23.1: Fixed Regression in 2.20.0 -#### Addressed Issues +### Addressed Issues * #532 -#### Notes +### Notes Bug was introduced in: https://github.com/AcademySoftwareFoundation/rez/releases/tag/2.20.0 ## 2.23.0: Package Usage Tracking, Better Config Overrides -#### Addressed Issues +### Addressed Issues * #528 -#### Notes +### Notes Two new features are added in this release: @@ -2411,7 +2505,7 @@ The embedded simplejson lib was removed. The native json lib is used instead, an ## 2.22.1: Stdin-related fixes -#### Addressed Issues +### Addressed Issues * #512 * #526 @@ -2421,7 +2515,7 @@ The embedded simplejson lib was removed. The native json lib is used instead, an PR: #213 -#### Notes +### Notes Package/variant/family search API is now available in package_search.py. This gives the same functionality as provided by the rez-search CLI tool. @@ -2434,13 +2528,13 @@ PR: #501 ## 2.20.1: Windows Fixes -#### Merged PRs +### Merged PRs * #490: Fix alias command in Windows when PATH is modified * #489: Fix cmd.exe not escaping special characters * #482: Fix selftest getting stuck on Windows -#### Addressed Issues +### Addressed Issues * #389 * #343 @@ -2452,11 +2546,11 @@ PR: #501 PR: #523 -#### Addressed Issues +### Addressed Issues * #492 -#### Notes +### Notes The rez-python command now supports all native python args and passes those through to its python subprocess - so you can now shebang with rez-python if that is useful. @@ -2468,15 +2562,15 @@ extraneous args after -- tokens. ## 2.19.1: Fixed bug with rez-build and package preprocess -#### Merged PRs +### Merged PRs * #522 -#### Addressed Issues +### Addressed Issues * #514 -#### Notes +### Notes The problem occurred because the preprocess function was attempting to be serialized when the package definition is cached to memcache. However, this function is stripped in installed packages; diff --git a/CODEOWNERS b/CODEOWNERS index 92e3280dd..f2bf703f3 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -18,7 +18,7 @@ NOTICE @AcademySoftwareFoundation/rez-tsc /src/rez/data/ @AcademySoftwareFoundation/rez-tsc # Docs -/wiki/ @AcademySoftwareFoundation/rez-tsc +/docs/ @AcademySoftwareFoundation/rez-tsc # Core /src/rez/backport @AcademySoftwareFoundation/rez-tsc diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c1df2e49a..fc1dbd542 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -15,10 +15,10 @@ sending a pull request. Please follow these guidelines: 7. Use [this format](https://help.github.com/articles/closing-issues-using-keywords/) to mention the issue(s) your PR closes; 8. Add relevant tests to demonstrate that your changes work; -9. Add relevant documentation [here](wiki/pages) to document your changes, if applicable. Those +9. Add relevant documentation [here](docs/source) to document your changes, if applicable. Those markdown files prefixed with `_` are internal and should not be changed. 10. If your changes add a new rez config setting, update [rezconfig.py](src/rez/rezconfig.py) and - document the setting. The comments in this file are extracted and turned into Wiki content. Pay + document the setting. The comments in this file are extracted and turned into documentation. Pay attention to the comment formatting and follow the existing style closely. ## CLA diff --git a/INSTALL.md b/INSTALL.md index d7e0796cb..59ce8c35e 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -1,82 +1 @@ -## Installation Script - -To install rez, download the source. Then from the root directory, run: - -``` -]$ python ./install.py -``` - -This installs rez to `/opt/rez`. See `install.py -h` for how to install to a -different location. - -Once the installation is complete, a message tells you how to run it: - -``` -SUCCESS! To activate Rez, add the following path to $PATH: -/opt/rez/bin/rez - -You may also want to source the completion script (for bash): -source /opt/rez/completion/complete.sh -``` - -> [[media/icons/warning.png]] Do _not_ move the installation - re-install to a new -> location if you want to change the install path. If you want to install rez for -> multiple operating systems, perform separate installs for each of those systems. - - -## Installation Via Pip - -It is possible to install rez with pip, like so: - -``` -]$ pip install rez -``` - -However, this comes with a caveat - rez command line tools _are not guaranteed -to work correctly_ once inside a rez environment (ie after using the `rez-env` -command). The reasons are given in the next section. - -Pip installation is adequate however, if all you require is the rez API, or you -don't require its command line tools to be available within a resolved environment. - -Note that running pip-installed rez command line tools will print a warning like so: - -``` -Pip-based rez installation detected. Please be aware that rez command line tools -are not guaranteed to function correctly in this case. See -https://github.com/AcademySoftwareFoundation/rez/wiki/Installation#why-not-pip-for-production -for futher details. -``` - - -## Why Not Pip For Production? - -Rez is not a normal python package. Although it can successfully be installed -using standard mechanisms such as pip, this comes with a number of caveats. -Specifically: - -* When within a rez environment (ie after using the `rez-env` command), the rez - command line tools are not guaranteed to function correctly; -* When within a rez environment, other packages' tools (that were also installed - with pip) remain visible, but are not guaranteed to work. - -When you enter a rez environment, the rez packages in the resolve configure -that environment as they see fit. For example, it is not uncommon for a python -package to append to PYTHONPATH. Environment variables such as PYTHONPATH -affect the behaviour of tools, including rez itself, and this can cause it to -crash or behave abnormally. - -When you use the `install.py` script to install rez, some extra steps are taken -to avoid this problem. Specifically: - -* Rez is installed into a virtualenv so that it operates standalone; -* The rez tools are shebanged with `python -E`, in order to protect them from - environment variables that affect python's behaviour; -* The rez tools are stored in their own directory, so that other unrelated tools - are not visible. - -Due to the way standard wheel-based python installations work, it simply is not -possible to perform these extra steps without using a custom installation script. -Wheels do not give the opportunity to run post-installation code; neither do -they provide functionality for specifying interpreter arguments to be added for -any given entry point. +See https://rez.readthedocs.io/en/stable/installation.html for installation instructions. diff --git a/README.md b/README.md index 5b5977fb2..d32bac666 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,14 @@ +[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://github.com/AcademySoftwareFoundation/rez/blob/main/LICENSE)
[![Release](https://shields.io/github/v/release/AcademySoftwareFoundation/rez)](https://github.com/AcademySoftwareFoundation/rez/releases) -[![Pypy Release](https://shields.io/pypi/v/rez)](https://pypi.org/project/rez)
-[![Core](https://github.com/AcademySoftwareFoundation/rez/workflows/core/badge.svg?branch=main)](https://github.com/AcademySoftwareFoundation/rez/actions?query=workflow%3Acore+branch%3Amain) -[![Ubuntu](https://github.com/AcademySoftwareFoundation/rez/workflows/ubuntu/badge.svg?branch=main)](https://github.com/AcademySoftwareFoundation/rez/actions?query=workflow%3Aubuntu+branch%3Amain) -[![Mac](https://github.com/AcademySoftwareFoundation/rez/workflows/mac/badge.svg?branch=main)](https://github.com/AcademySoftwareFoundation/rez/actions?query=workflow%3Amac+branch%3Amain) -[![Windows](https://github.com/AcademySoftwareFoundation/rez/workflows/windows/badge.svg?branch=main)](https://github.com/AcademySoftwareFoundation/rez/actions?query=workflow%3AWindows+branch%3Amain)
+[![PyPI Release](https://shields.io/pypi/v/rez)](https://pypi.org/project/rez)
+[![Mailing List](https://img.shields.io/badge/rez--discussion-lists.aswf.io-003366?style=flat-square&logo=linuxfoundation)](https://lists.aswf.io/g/rez-discussion) +[![Slack](https://img.shields.io/badge/Slack-ASWF_%23rez-7a6800?style=flat-square&logo=slack)](https://slack.aswf.io/) +[![Contributing Guidelines](https://img.shields.io/badge/rez-Contributing%20Guidelines-0b610e?style=flat-square&logo=github)](https://github.com/AcademySoftwareFoundation/rez/blob/main/CONTRIBUTING.md)
+[![Tests](https://github.com/AcademySoftwareFoundation/rez/workflows/tests/badge.svg?branch=main)](https://github.com/AcademySoftwareFoundation/rez/actions?query=workflow%3Atests+branch%3Amain) [![Installation](https://github.com/AcademySoftwareFoundation/rez/workflows/installation/badge.svg?branch=main)](https://github.com/AcademySoftwareFoundation/rez/actions?query=workflow%3Ainstallation+branch%3Amain) [![Flake8](https://github.com/AcademySoftwareFoundation/rez/workflows/flake8/badge.svg?branch=main)](https://github.com/AcademySoftwareFoundation/rez/actions?query=workflow%3Aflake8+branch%3Amain) -[![Wiki](https://github.com/AcademySoftwareFoundation/rez/workflows/wiki/badge.svg)](https://github.com/AcademySoftwareFoundation/rez/actions?query=workflow%3Awiki+event%3Arelease) -[![Pypi](https://github.com/AcademySoftwareFoundation/rez/workflows/pypi/badge.svg)](https://github.com/AcademySoftwareFoundation/rez/actions?query=workflow%3Apypi+event%3Arelease) +[![Docs](https://readthedocs.org/projects/rez/badge/?version=stable)](https://rez.readthedocs.io/en/stable) +[![PyPI](https://github.com/AcademySoftwareFoundation/rez/workflows/pypi/badge.svg)](https://github.com/AcademySoftwareFoundation/rez/actions?query=workflow%3Apypi+event%3Arelease) [![Benchmark](https://github.com/AcademySoftwareFoundation/rez/workflows/benchmark/badge.svg)](https://github.com/AcademySoftwareFoundation/rez/actions?query=workflow%3Abenchmark+event%3Arelease)
[![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=AcademySoftwareFoundation_rez&metric=alert_status)](https://sonarcloud.io/summary/new_code?id=AcademySoftwareFoundation_rez) [![Bugs](https://sonarcloud.io/api/project_badges/measure?project=AcademySoftwareFoundation_rez&metric=bugs)](https://sonarcloud.io/summary/new_code?id=AcademySoftwareFoundation_rez) @@ -35,18 +36,18 @@ environments reference these existing packages. This means that configured envir are lightweight, and very fast to create, often taking just a few seconds to configure despite containing hundreds of packages. -See [the wiki](https://github.com/AcademySoftwareFoundation/rez/wiki) for full documentation. +See [the wiki](https://rez.readthedocs.io) for full documentation.

- - + +
Typical package managers install packages into an environment


- - + +
Rez installs packages once, and configures environments dynamically

@@ -90,7 +91,7 @@ and when re-evaluated later will reconstruct the same environment once more. ## Examples This example places the user into a resolved shell containing the requested packages, -using the [rez-env](https://github.com/AcademySoftwareFoundation/rez/wiki/Command-Line-Tools#rez-env) tool: +using the [rez-env](https://rez.readthedocs.io/en/stable/commands/rez-env.html) tool: ]$ rez-env requests-2.2+ python-2.6 'pymongo-0+<2.7' @@ -136,10 +137,10 @@ Resolved environments can also be created via the API: ## Quickstart -First, install Rez. Download the source, and from the source directory, run +First, install Rez using Python 3.7+. Download the source, and from the source directory, run (with DEST_DIR replaced with your install location): - ]$ python ./install.py -v DEST_DIR + ]$ python3 ./install.py -v DEST_DIR This installs the Rez command line tools. It will print a message at the end telling you how to use Rez when the installation has completed. Rez is not a diff --git a/docs/NOTES.md b/docs/NOTES.md deleted file mode 100644 index 357efc0da..000000000 --- a/docs/NOTES.md +++ /dev/null @@ -1,15 +0,0 @@ -============ -page section -============ - -Section 1 -========= - -Section 1.1 ------------ - -Section 1.1.1 -+++++++++++++ - -Section 1.1.1.1 -*************** diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 000000000..5c94e70ae --- /dev/null +++ b/docs/README.md @@ -0,0 +1,31 @@ +# Build instructions + +To build the docs you must use Python 3.11. + +To create a build environment run the following commands: +```python +python -m venv .venv +source .venv/bin/activate +pip install -r docs/requirements.txt + +cd docs +make html +``` + +# Example Headers + +============ +page section +============ + +Section 1 +========= + +Section 1.1 +----------- + +Section 1.1.1 ++++++++++++++ + +Section 1.1.1.1 +*************** diff --git a/docs/requirements.txt b/docs/requirements.txt index 470e82a19..523f5d7f3 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,3 +1,4 @@ -sphinx +sphinx >=7.0.0,<8.0.0 furo sphinx-autobuild +myst-parser diff --git a/docs/source/api.rst b/docs/source/api.rst index ad4c76fca..c58f7b36a 100644 --- a/docs/source/api.rst +++ b/docs/source/api.rst @@ -43,5 +43,5 @@ Python API rez.system rez.util rez.utils - rez.vendor.version + rez.version rez.wrapper diff --git a/docs/source/basic_concepts.rst b/docs/source/basic_concepts.rst index 34d2d01fd..01f53bfef 100644 --- a/docs/source/basic_concepts.rst +++ b/docs/source/basic_concepts.rst @@ -78,8 +78,13 @@ version than ``1.0``. .. note:: No special importance is given to specific characters or letters in Rez version numbers. The terms ``alpha`` and ``beta`` for example have no special meaning. Similarly, the number of tokens in - a version number doesn't matter, you can have as many as you like. While you are encouraged to use - semantic versioning (see ``_), it is not enforced. + a version number doesn't matter, you can have as many as you like. + + While you are encouraged to use semantic versioning (see ``_), it is not enforced. Please note + that if you are using semantic versioning, version ordering will NOT behave like described in the semantic versioning + 2.0.0 spec. + + Ex. foo-1.0.0 < foo-1.0.0-beta.1 .. _packages-concept: diff --git a/docs/source/changelog.md b/docs/source/changelog.md new file mode 100644 index 000000000..6b4af7804 --- /dev/null +++ b/docs/source/changelog.md @@ -0,0 +1,5 @@ +# Changelog + +```{include} ../../CHANGELOG.md +:start-after: +``` diff --git a/docs/source/conf.py b/docs/source/conf.py index ff4a4623d..ed9fad1cd 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -6,6 +6,10 @@ import os import sys +import sphinx.domains +import sphinx.addnodes +import sphinx.application + # Add path to rez's source. sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', 'src'))) @@ -33,12 +37,26 @@ "sphinx.ext.intersphinx", "sphinx.ext.napoleon", "sphinx.ext.todo", + "myst_parser", # Rez custom extension 'rez_sphinxext' ] templates_path = ['_templates'] +nitpick_ignore = [ + # TODO: Remove once we unvendor enum. + ("py:class", "rez.solver._Common"), + ("py:class", "_thread._local"), + ("py:class", "rez.utils.platform_._UnixPlatform"), + ("py:class", "rez.version._util._Common"), + ("py:class", "rez.version._version._Comparable"), +] + +nitpick_ignore_regex = [ + ("py:class", r"rez\.vendor\..*"), +] + # -- Options for HTML output ------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output @@ -70,6 +88,12 @@ # autoclass_content = 'both' autodoc_class_signature = 'separated' autodoc_member_order = 'bysource' +autodoc_inherit_docstrings = True +autodoc_default_options = { + "show-inheritance": True, + "undoc-members": True, + "inherited-members": True, +} # -- Options for extlinks extension ----------------------------------------- @@ -82,4 +106,40 @@ # -- Options for todo extension --------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/extensions/todo.html -todo_emit_warnings = True +todo_emit_warnings = False + + +# -- Custom ----------------------------------------------------------------- + +def handle_ref_warning( + app: sphinx.application.Sphinx, + domain: sphinx.domains.Domain, + node: sphinx.addnodes.pending_xref, +) -> bool | None: + """ + Emitted when a cross-reference to an object cannot be resolved even + after missing-reference. If the event handler can emit warnings for the + missing reference, it should return True. The configuration variables + nitpick_ignore and nitpick_ignore_regex prevent the event from being + emitted for the corresponding nodes. + """ + if domain and domain.name != 'py': + return None + + from docutils.utils import get_source_line + + source, line = get_source_line(node) + if 'docstring of collections.abc.' in source: + # Silence warnings that come from collections.abc + return True + + return False + + +def setup(app: sphinx.application.Sphinx) -> dict[str, bool | str]: + app.connect('warn-missing-reference', handle_ref_warning) + + return { + 'parallel_read_safe': True, + 'parallel_write_safe': True, + } diff --git a/docs/source/configuring_rez.rst b/docs/source/configuring_rez.rst index 3555510ab..effcd12bd 100644 --- a/docs/source/configuring_rez.rst +++ b/docs/source/configuring_rez.rst @@ -26,6 +26,14 @@ variable :envvar:`REZ_CONFIG_FILE` is then set to for all your users. You do not need to provide a copy of all settings in this file. Just provide those that are changed from the defaults. +Supported Configuration File Formats +==================================== + +Rez supports both YAML configuration files and Python configuration files. + +You may prefer a Python based configuration file if you need to vary your configuration settings based on your +current platform. + .. _configuring-rez-settings-merge-rules: Settings Merge Rules diff --git a/docs/source/environment.rst b/docs/source/environment.rst index 279db3db8..2030aa500 100644 --- a/docs/source/environment.rst +++ b/docs/source/environment.rst @@ -243,11 +243,43 @@ operation of rez. If set to a non-empty string, this prevents rez from cleaning up any temporary directories. This is for debugging purposes. +.. envvar:: REZ_ENV_PROMPT + + See the :data:`set_prompt` and :data:`prefix_prompt` settings. + +.. envvar:: REZ_LOGGING_CONF + + Path to a file that will be consumed by :func:`logging.config.fileConfig` to configure + the logger. + + +Development Environment Variables +================================= + +.. envvar:: REZ_LOG_DEPRECATION_WARNINGS + + Enable all deprecation warnings to be logged regardless of how you have configured + your python interpreter. This is usefull to help upgrading to newer versions of rez. + Prior to updating, you should set this environment variable to see if you need to + change some things to be compatible with newer versions. + + .. warning:: + + Enabling this will forcefully load every configuration file instead of loading them + lazilly. This can have an impact on startup time. + .. envvar:: REZ_SIGUSR1_ACTION If you set this to ``print_stack``, rez will prints its - current stacktrace to stdout if sent a USR1 signal. This is for debugging purposes. + current stacktrace to stdout if sent a USR1 signal. This is for debugging purposes only. -.. envvar:: REZ_ENV_PROMPT +.. envvar:: _REZ_NO_KILLPG - See the :data:`set_prompt` and :data:`prefix_prompt` settings. + By default, rez will try to kill its process group when it receives a :data:`SIGINT ` + or :data:`SIGTERM ` signal. Setting ``_REZ_NO_KILLPG`` to either "1", "true", "on" + or "yes" disables this behavior. This is handy when developing rez itself. + +.. envvar:: _REZ_QUIET_ON_SIG + + Print a message if rez receives a :data:`SIGINT ` + or :data:`SIGTERM ` signal. diff --git a/docs/source/getting_started.rst b/docs/source/getting_started.rst new file mode 100644 index 000000000..2872fe4fc --- /dev/null +++ b/docs/source/getting_started.rst @@ -0,0 +1,182 @@ +=============== +Getting started +=============== + +Essential packages +================== + +After installation, you need to create some essential Rez packages. The :ref:`rez-bind` +tool creates Rez packages that reference software already installed on your system. +Use the :option:`--quickstart ` argument to bind a set of standard packages: + +.. note:: + You may require administrative privileges for some of them + +.. code-block:: text + + ]$ rez-bind --quickstart + Binding platform into /home/ajohns/packages... + Binding arch into /home/ajohns/packages... + Binding os into /home/ajohns/packages... + Binding python into /home/ajohns/packages... + Binding rez into /home/ajohns/packages... + Binding rezgui into /home/ajohns/packages... + Binding setuptools into /home/ajohns/packages... + Binding pip into /home/ajohns/packages... + + Successfully converted the following software found on the current system into Rez packages: + + PACKAGE URI + ------- --- + arch /home/ajohns/packages/arch/x86_64/package.py + os /home/ajohns/packages/os/osx-10.11.5/package.py + pip /home/ajohns/packages/pip/8.0.2/package.py + platform /home/ajohns/packages/platform/osx/package.py + python /home/ajohns/packages/python/2.7.11/package.py + rez /home/ajohns/packages/rez/2.0.rc1.44/package.py + rezgui /home/ajohns/packages/rezgui/2.0.rc1.44/package.py + setuptools /home/ajohns/packages/setuptools/19.4/package.py + +Now you should be able to create an environment containing Python. Try this: + +.. code-block:: text + + ]$ rez-env python -- which python + /home/ajohns/packages/python-2.7.8/platform-linux/arch-x86_64/os-Ubuntu-12.04/bin/python + +Building packages +================= + +Before building your first rez package, ensure that: + +* The directory ``$HOME/packages`` exists and is writable; +* The `cmake `_ tool is available. + + .. note:: + CMake is not a hard requirement to build packages in rez. This example just + happens to use CMake, but you can use anything you want in your packages. + +The :ref:`rez-build` tool is used to build packages and install them locally (typically to ``$HOME/packages``). + +Once you've done that, you can use them via :ref:`rez-env`, just like any other package: + +.. code-block:: text + + ]$ cd example_packages/hello_world + ]$ rez-build --install + + -------------------------------------------------------------------------------- + Building hello_world-1.0.0... + -------------------------------------------------------------------------------- + Resolving build environment: python + resolved by ajohns@workstation.local, on Sun Jul 31 14:39:33 2016, using Rez v2.0.rc1.44 + + requested packages: + python + ~platform==osx (implicit) + ~arch==x86_64 (implicit) + ~os==osx-10.11.5 (implicit) + + resolved packages: + arch-x86_64 /home/ajohns/packages/arch/x86_64 (local) + os-osx-10.11.5 /home/ajohns/packages/os/osx-10.11.5 (local) + platform-osx /home/ajohns/packages/platform/osx (local) + python-2.7.11 /home/ajohns/packages/python/2.7.11/platform-osx/arch-x86_64/os-osx-10.11.5 (local) + + Invoking cmake build system... + Executing: /usr/local/bin/cmake -d /home/ajohns/workspace/rez/example_packages/hello_world -Wno-dev -DCMAKE_ECLIPSE_GENERATE_SOURCE_PROJECT=TRUE -D_ECLIPSE_VERSION=4.3 --no-warn-unused-cli -DCMAKE_INSTALL_PREFIX=/home/ajohns/packages/hello_world/1.0.0 -DCMAKE_MODULE_PATH=${CMAKE_MODULE_PATH} -DCMAKE_BUILD_TYPE=Release -DREZ_BUILD_TYPE=local -DREZ_BUILD_INSTALL=1 -G Unix Makefiles + Not searching for unused variables given on the command line. + -- Could NOT find PkgConfig (missing: PKG_CONFIG_EXECUTABLE) + -- Configuring done + -- Generating done + -- Build files have been written to: /home/ajohns/workspace/rez/example_packages/hello_world/build + + Executing: make -j4 + [100%] Built target py + + Executing: make -j4 install + [100%] Built target py + Install the project... + -- Install configuration: "Release" + -- Installing: /home/ajohns/packages/hello_world/1.0.0/./python/hello_world.py + -- Installing: /home/ajohns/packages/hello_world/1.0.0/./python/hello_world.pyc + -- Installing: /home/ajohns/packages/hello_world/1.0.0/./bin/hello + + All 1 build(s) were successful. + +You have just built your first package, and installed it to the :data:`local package path `, +which defaults to (and is usually kept as) ``$HOME/packages``. + +For more information, see :doc:`building_packages`. + +Testing your package +==================== + +You can use the :ref:`rez-env` tool to request a configured environment containing your package: + +.. code-block:: text + + ]$ rez-env hello_world + + You are now in a rez-configured environment. + + resolved by ajohns@workstation.local, on Sun Jul 31 14:43:54 2016, using Rez v2.0.rc1.44 + + requested packages: + hello_world + ~platform==osx (implicit) + ~arch==x86_64 (implicit) + ~os==osx-10.11.5 (implicit) + + resolved packages: + arch-x86_64 /home/ajohns/packages/arch/x86_64 (local) + hello_world-1.0.0 /home/ajohns/packages/hello_world/1.0.0 (local) + os-osx-10.11.5 /home/ajohns/packages/os/osx-10.11.5 (local) + platform-osx /home/ajohns/packages/platform/osx (local) + python-2.7.11 /home/ajohns/packages/python/2.7.11/platform-osx/arch-x86_64/os-osx-10.11.5 (local) + + > ]$ █ + +Now you are within the configured environment. The caret (``>``) prefixed to your prompt is a visual +cue telling you that you're within a rez-configured subshell. Rez does not update the currect environment, +instead it configures a subshell and puts you within it. + +Now you can run the ``hello`` tool in our ``hello_world`` package: + +.. code-block:: text + + > ]$ hello + Hello world! + +If you're within a rez shell, and you forget what packages are currently available or want to see +the list again, you can use the :ref:`rez-context` tool. It prints the same information you see when you +initially created the environment: + +.. code-block:: text + + > ]$ rez-context + resolved by ajohns@workstation.local, on Sun Jul 31 14:43:54 2016, using Rez v2.0.rc1.44 + + requested packages: + hello_world + ~platform==osx (implicit) + ~arch==x86_64 (implicit) + ~os==osx-10.11.5 (implicit) + + resolved packages: + arch-x86_64 /home/ajohns/packages/arch/x86_64 (local) + hello_world-1.0.0 /home/ajohns/packages/hello_world/1.0.0 (local) + os-osx-10.11.5 /home/ajohns/packages/os/osx-10.11.5 (local) + platform-osx /home/ajohns/packages/platform/osx (local) + python-2.7.11 /home/ajohns/packages/python/2.7.11/platform-osx/arch-x86_64/os-osx-10.11.5 (local) + +To exit the configured environment, simply exist the shell using the ``exit`` command or :kbd:`Control-d`: + +You can also create a configured environment and run a command inside of it, with a single command. +When you use this form, the shell is immediately exited after the command runs: + +.. code-block:: text + + ]$ rez-env hello_world -- hello + Hello world! + ]$ █ diff --git a/docs/source/guides/index.rst b/docs/source/guides/index.rst new file mode 100644 index 000000000..0cd0e639d --- /dev/null +++ b/docs/source/guides/index.rst @@ -0,0 +1,10 @@ +=========== +User guides +=========== + +This section contains various user guides. + +.. toctree:: + :maxdepth: 1 + + prepare_for_3 diff --git a/docs/source/guides/prepare_for_3.rst b/docs/source/guides/prepare_for_3.rst new file mode 100644 index 000000000..dc35a1d94 --- /dev/null +++ b/docs/source/guides/prepare_for_3.rst @@ -0,0 +1,85 @@ +===================== +Prepare for rez 3.0.0 +===================== + +The rez TSC started to prepare to release rez ``3.0.0``. Rez ``3.0.0`` will not be +a major breaking change but will still remove some things. + +This guide will show you how to prepare in advance before ``3.0.0`` is released so +that you are not caught by surprise by the changes made in ``3.0.0`` once you start +using ``3.0.0``. We hope that this guide will help make the upgrade process smoother. + +Steps for smooth upgrade +======================== + +#. Read the `release notes <../CHANGELOG.html>`_ for ``2.114.0`` and ``3.0.0`` (once released). The release + notes will contain a list of everything that was deprecated in ``2.114.0`` and will + be removed or changed in ``3.0.0``. + +#. Upgrade to ``2.114.0``. + + To upgrade to (or before upgrading to) rez ``3.0.0``, we suggest that you first + upgrade to ``2.114.0``. This is not mandatory and you can jump straight to ``3.0.0`` + if you wish. + + .. warning:: + + If you skip this step, you won't be able to see deprecation warnings before + things are removed frm the rez code base. + +#. Set the ``REZ_LOG_DEPRECATION_WARNINGS`` environment variable. + + ``2.114.1`` adds a new environment variable called :envvar:`REZ_LOG_DEPRECATION_WARNINGS` + that will force all rez related deprecation warnings to be printed out to stderr. + This will let you catch if you are using something deprecated that will be removed + in future versions. + +#. Run rez in you production workflows if possible and watch out for deprecation warnings + coming from rez. + +#. Address each warning one by one. + +#. Once you think you have addressed all warnings, upgrade to 3.0.0 (or wait for ``3.0.0`` to + be released if it's not yet available). + +Optional +======== + +Since some default configuration default values will change in ``3.0.0``, we highly suggest +that you run some anslysis scripts to see if you will be impacted by these changes. + +Detect old-style commands in your repositories +---------------------------------------------- + +Verify that your package repositories don't contain packages that +use old-style commands. + +You can use this python snippet to discover all your packages and variants +that contain old style commands. It will print a colored warning for every +package/variant that use old-style commands. + +.. code-block:: python + + from rez.config import config + from rez.packages import iter_packages, iter_package_families + + config.warn_old_commands = True + config.error_old_commands = False + config.disable_rez_1_compatibility = False + + for family_name in iter_package_families(): + packages = iter_packages(family_name.name) + + for package in packages: + package.validate_data() + + for variant in package.iter_variants(): + variant.validate_data() + +.. hint:: + + Remember to run it over all your repositories! + +If you see any warnings, we suggest that you move or remove the packages/variants +from your repositories. This might require some work but it should hopefully not +be too difficult. diff --git a/docs/source/index.rst b/docs/source/index.rst index da4e133f5..81cdaf7e0 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -12,10 +12,12 @@ Welcome to rez's documentation! :hidden: installation + getting_started basic_concepts building_packages context variants + guides/index .. toctree:: :maxdepth: 2 @@ -45,6 +47,7 @@ Welcome to rez's documentation! commands_index environment api + changelog.md Rez is a cross-platform package manager with a difference. Using Rez you can create standalone environments configured for a given set of packages. However, unlike many diff --git a/docs/source/installation.rst b/docs/source/installation.rst index 87c085a2b..a9ef7e622 100644 --- a/docs/source/installation.rst +++ b/docs/source/installation.rst @@ -5,6 +5,9 @@ Installation Installation Script =================== +.. warning:: + The install script only supports Python 3.7+. + To install rez, download the source. Then from the root directory, run:: ]$ python ./install.py @@ -34,6 +37,11 @@ It is possible to install rez with pip, like so:: ]$ pip install rez +.. note:: + While the ``install.py`` script only supports Python 3.7+, + rez is still usable with Python 2.7 when installed via pip. In other words, the API + is fully compatible with Python 2 and 3. + However, this comes with a caveat. Rez command line tools **are not guaranteed to work correctly** once inside a rez environment (ie after using the :ref:`rez-env` command). The reasons are given in the next section. diff --git a/docs/source/managing_packages.rst b/docs/source/managing_packages.rst index 1e6e1b995..69d3548ff 100644 --- a/docs/source/managing_packages.rst +++ b/docs/source/managing_packages.rst @@ -222,6 +222,12 @@ the cache is to copy that content locally instead, and avoid the network cost. Please note: Package caching does **NOT** cache package definitions. Only their payloads (ie, the package root directory). +Build behavior +-------------- + +Package caching during a package build is disabled by default. To enable caching during +a package build, you can set :data:`package_cache_during_build` to True. + .. _enabling-package-caching: Enabling Package Caching diff --git a/docs/source/package_commands.rst b/docs/source/package_commands.rst index 24b357d19..7e54f2b89 100644 --- a/docs/source/package_commands.rst +++ b/docs/source/package_commands.rst @@ -477,7 +477,7 @@ Following is a list of the objects and functions available. if "foo.cli" in ephemerals: info("Foo cli option is being specified!") -.. py:function:: ephemerals.get_range(name: str, range_: str) -> ~rez.vendor.version.version.VersionRange +.. py:function:: ephemerals.get_range(name: str, range_: str) -> ~rez.version.VersionRange Use ``get_range`` to test with the :func:`intersects` function. Here, we enable ``foo``'s commandline tools by default, unless explicitly disabled via @@ -539,7 +539,7 @@ Following is a list of the objects and functions available. info("floob version is %s" % resolve.floob.version) -.. py:function:: intersects(range1: str | ~rez.vendor.version.version.VersionRange | ~rez.rex_bindings.VariantBinding | ~rez.rex_bindings.VersionBinding, range2: str) -> bool +.. py:function:: intersects(range1: str | ~rez.version.VersionRange | ~rez.rex_bindings.VariantBinding | ~rez.rex_bindings.VersionBinding, range2: str) -> bool A boolean function that returns True if the version or version range of the given object, intersects with the given version range. Valid objects to query include: diff --git a/docs/source/package_definition.rst b/docs/source/package_definition.rst index 566ff4278..8223ccdbf 100644 --- a/docs/source/package_definition.rst +++ b/docs/source/package_definition.rst @@ -719,7 +719,7 @@ the data type, and includes a code snippet. .. code-block:: python - help = "https://github.com/__GITHUB_REPO__/wiki" + help = "https://example.com" .. code-block:: diff --git a/install.py b/install.py index 7b57a8691..f63464a7f 100644 --- a/install.py +++ b/install.py @@ -12,9 +12,21 @@ import os import sys import shutil -import os.path import subprocess +if sys.version_info[:2] < (3, 7): + sys.stderr.write( + 'Error: rez can only be installed with python greater or equal to 3.7. You are trying to install using Python {0}.{1}.{2} ({3}).\n'.format( + sys.version_info[0], + sys.version_info[1], + sys.version_info[2], + sys.executable, + ) + ) + exit(1) + +import venv + source_path = os.path.dirname(os.path.realpath(__file__)) src_path = os.path.join(source_path, "src") @@ -28,36 +40,16 @@ from rez.cli._entry_points import get_specifications # noqa: E402 from rez.vendor.distlib.scripts import ScriptMaker # noqa: E402 -# switch to builtin venv in python 3.7+ -# -# Note: There are issues using venv with 3.6. Installed pip version is 18.2 -# (which isn't high enough for rez-pip to behave correctly). 3.7 installs pip -# version 20.1. -# -use_venv = (sys.version_info[:2] >= (3, 7)) - -if use_venv: - import venv -else: - from build_utils.virtualenv.virtualenv import create_environment, path_locations - def create_virtual_environment(dest_dir): - if use_venv: - builder = venv.EnvBuilder(with_pip=True) - builder.create(dest_dir) - else: - create_environment(dest_dir) + builder = venv.EnvBuilder(with_pip=True) + builder.create(dest_dir) def get_virtualenv_bin_dir(dest_dir): - if use_venv: - builder = venv.EnvBuilder() - context = builder.ensure_directories(dest_dir) - return context.bin_path - else: - _, _, _, bin_dir = path_locations(dest_dir) - return bin_dir + builder = venv.EnvBuilder() + context = builder.ensure_directories(dest_dir) + return context.bin_path def get_virtualenv_py_executable(dest_dir): diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..fed528d4a --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" diff --git a/release-rez.py b/release-rez.py index 049085460..166083914 100644 --- a/release-rez.py +++ b/release-rez.py @@ -11,7 +11,7 @@ import argparse import os from datetime import date -from pipes import quote +from shlex import quote import subprocess import sys diff --git a/setup.py b/setup.py index fc936c293..132782a82 100644 --- a/setup.py +++ b/setup.py @@ -61,14 +61,16 @@ def find_files(pattern, path=None, root="rez"): long_description=long_description, long_description_content_type='text/markdown', url="https://github.com/AcademySoftwareFoundation/rez", - author="Contributors to the rez project", - author_email="rez-discussion@lists.aswf.io", + author="Allan Johns", + author_email="nerdvegas@gmail.com", + maintainer="Contributors to the rez project", + maintainer_email="rez-discussion@lists.aswf.io", license="Apache-2.0", license_files=["LICENSE"], entry_points={ "console_scripts": get_specifications().values() }, - include_package_data=True, + include_package_data=False, zip_safe=False, package_dir={'': 'src'}, packages=find_packages('src', exclude=["build_utils", diff --git a/src/build_utils/README.md b/src/build_utils/README.md index 43fdbc091..c1ed5342f 100644 --- a/src/build_utils/README.md +++ b/src/build_utils/README.md @@ -1,7 +1,3 @@ Source within this directory is used to perform build-related tasks, but is not installed as part of rez. - -Note: The embedded virtualenv found in this dir is only used up to python v3.6. -From v3.7 onwards, python's native `venv` module is used instead. See -https://github.com/AcademySoftwareFoundation/rez/releases/tag/2.72.0. diff --git a/src/build_utils/license/apply_copyright b/src/build_utils/license/apply_copyright index 6d5d1edbd..880cfa583 100755 --- a/src/build_utils/license/apply_copyright +++ b/src/build_utils/license/apply_copyright @@ -15,7 +15,6 @@ find ./src/rez/bind -name '*.py' > .lic.tmp find ./src/rez/cli -name '*.py' >> .lic.tmp find ./src/rez/tests -name '*.py' >> .lic.tmp find ./src/rez/utils -name '*.py' >> .lic.tmp -find ./src/rez/vendor/version -name '*.py' >> .lic.tmp find ./src/rez/bind -name '*.py' >> .lic.tmp find ./src/rezgui -name '*.py' >> .lic.tmp find ./src/rezplugins -name '*.py' >> .lic.tmp diff --git a/src/build_utils/virtualenv/LICENSE.txt b/src/build_utils/virtualenv/LICENSE.txt deleted file mode 100644 index ab145001f..000000000 --- a/src/build_utils/virtualenv/LICENSE.txt +++ /dev/null @@ -1,22 +0,0 @@ -Copyright (c) 2007 Ian Bicking and Contributors -Copyright (c) 2009 Ian Bicking, The Open Planning Project -Copyright (c) 2011-2016 The virtualenv developers - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/src/build_utils/virtualenv/README.md b/src/build_utils/virtualenv/README.md deleted file mode 100644 index 4680964fd..000000000 --- a/src/build_utils/virtualenv/README.md +++ /dev/null @@ -1,4 +0,0 @@ - -Note: This can be removed when python < 3.7 support is dropped. From 3.7 -onwards, the rez installation uses python's builtin `venv` module to create -the installation virtualenv. diff --git a/src/build_utils/virtualenv/__init__.py b/src/build_utils/virtualenv/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/build_utils/virtualenv/virtualenv.py b/src/build_utils/virtualenv/virtualenv.py deleted file mode 100755 index 3ccbed6df..000000000 --- a/src/build_utils/virtualenv/virtualenv.py +++ /dev/null @@ -1,2611 +0,0 @@ -#!/usr/bin/env python -"""Create a "virtual" Python installation""" - -# fmt: off -import os # isort:skip -import sys # isort:skip - -# If we are running in a new interpreter to create a virtualenv, -# we do NOT want paths from our existing location interfering with anything, -# So we remove this file's directory from sys.path - most likely to be -# the previous interpreter's site-packages. Solves #705, #763, #779 -if os.environ.get("VIRTUALENV_INTERPRETER_RUNNING"): - for path in sys.path[:]: - if os.path.realpath(os.path.dirname(__file__)) == os.path.realpath(path): - sys.path.remove(path) -# fmt: on - -import ast -import base64 -import codecs -import contextlib -import distutils.spawn -import distutils.sysconfig -import errno -import glob -import logging -import optparse -import os -import re -import shutil -import struct -import subprocess -import sys -import tempfile -import textwrap -import zipfile -import zlib -from distutils.util import strtobool -from os.path import join - -try: - import ConfigParser -except ImportError: - # noinspection PyPep8Naming - import configparser as ConfigParser - -__version__ = "16.6.1" -virtualenv_version = __version__ # legacy -DEBUG = os.environ.get("_VIRTUALENV_DEBUG", None) == "1" -if sys.version_info < (2, 7): - print("ERROR: {}".format(sys.exc_info()[1])) - print("ERROR: this script requires Python 2.7 or greater.") - sys.exit(101) - -HERE = os.path.dirname(os.path.abspath(__file__)) -IS_ZIPAPP = os.path.isfile(HERE) - -try: - # noinspection PyUnresolvedReferences,PyUnboundLocalVariable - basestring -except NameError: - basestring = str - -PY_VERSION = "python{}.{}".format(sys.version_info[0], sys.version_info[1]) - -IS_PYPY = hasattr(sys, "pypy_version_info") -IS_WIN = sys.platform == "win32" -IS_CYGWIN = sys.platform == "cygwin" -IS_DARWIN = sys.platform == "darwin" -ABI_FLAGS = getattr(sys, "abiflags", "") - -USER_DIR = os.path.expanduser("~") -if IS_WIN: - DEFAULT_STORAGE_DIR = os.path.join(USER_DIR, "virtualenv") -else: - DEFAULT_STORAGE_DIR = os.path.join(USER_DIR, ".virtualenv") -DEFAULT_CONFIG_FILE = os.path.join(DEFAULT_STORAGE_DIR, "virtualenv.ini") - -if IS_PYPY: - EXPECTED_EXE = "pypy" -else: - EXPECTED_EXE = "python" - -# Return a mapping of version -> Python executable -# Only provided for Windows, where the information in the registry is used -if not IS_WIN: - - def get_installed_pythons(): - return {} - - -else: - try: - import winreg - except ImportError: - # noinspection PyUnresolvedReferences - import _winreg as winreg - - def get_installed_pythons(): - final_exes = dict() - - # Grab exes from 32-bit registry view - exes = _get_installed_pythons_for_view("-32", winreg.KEY_WOW64_32KEY) - # Grab exes from 64-bit registry view - exes_64 = _get_installed_pythons_for_view("-64", winreg.KEY_WOW64_64KEY) - # Check if exes are unique - if set(exes.values()) != set(exes_64.values()): - exes.update(exes_64) - - # Create dict with all versions found - for version, bitness in sorted(exes): - exe = exes[(version, bitness)] - # Add minor version (X.Y-32 or X.Y-64) - final_exes[version + bitness] = exe - # Add minor extensionless version (X.Y); 3.2-64 wins over 3.2-32 - final_exes[version] = exe - # Add major version (X-32 or X-64) - final_exes[version[0] + bitness] = exe - # Add major extensionless version (X); 3.3-32 wins over 3.2-64 - final_exes[version[0]] = exe - - return final_exes - - def _get_installed_pythons_for_view(bitness, view): - exes = dict() - # If both system and current user installations are found for a - # particular Python version, the current user one is used - for key in (winreg.HKEY_LOCAL_MACHINE, winreg.HKEY_CURRENT_USER): - try: - python_core = winreg.OpenKey(key, "Software\\Python\\PythonCore", 0, view | winreg.KEY_READ) - except WindowsError: - # No registered Python installations - continue - i = 0 - while True: - try: - version = winreg.EnumKey(python_core, i) - i += 1 - try: - at_path = winreg.QueryValue(python_core, "{}\\InstallPath".format(version)) - except WindowsError: - continue - # Remove bitness from version - if version.endswith(bitness): - version = version[: -len(bitness)] - exes[(version, bitness)] = join(at_path, "python.exe") - except WindowsError: - break - winreg.CloseKey(python_core) - - return exes - - -REQUIRED_MODULES = [ - "os", - "posix", - "posixpath", - "nt", - "ntpath", - "genericpath", - "fnmatch", - "locale", - "encodings", - "codecs", - "stat", - "UserDict", - "readline", - "copy_reg", - "types", - "re", - "sre", - "sre_parse", - "sre_constants", - "sre_compile", - "zlib", -] - -REQUIRED_FILES = ["lib-dynload", "config"] - -MAJOR, MINOR = sys.version_info[:2] -if MAJOR == 2: - if MINOR >= 6: - REQUIRED_MODULES.extend(["warnings", "linecache", "_abcoll", "abc"]) - if MINOR >= 7: - REQUIRED_MODULES.extend(["_weakrefset"]) -elif MAJOR == 3: - # Some extra modules are needed for Python 3, but different ones - # for different versions. - REQUIRED_MODULES.extend( - [ - "_abcoll", - "warnings", - "linecache", - "abc", - "io", - "_weakrefset", - "copyreg", - "tempfile", - "random", - "__future__", - "collections", - "keyword", - "tarfile", - "shutil", - "struct", - "copy", - "tokenize", - "token", - "functools", - "heapq", - "bisect", - "weakref", - "reprlib", - ] - ) - if MINOR >= 2: - REQUIRED_FILES[-1] = "config-{}".format(MAJOR) - if MINOR >= 3: - import sysconfig - - platform_dir = sysconfig.get_config_var("PLATDIR") - REQUIRED_FILES.append(platform_dir) - REQUIRED_MODULES.extend(["base64", "_dummy_thread", "hashlib", "hmac", "imp", "importlib", "rlcompleter"]) - if MINOR >= 4: - REQUIRED_MODULES.extend(["operator", "_collections_abc", "_bootlocale"]) - if MINOR >= 6: - REQUIRED_MODULES.extend(["enum"]) - -if IS_PYPY: - # these are needed to correctly display the exceptions that may happen - # during the bootstrap - REQUIRED_MODULES.extend(["traceback", "linecache"]) - - if MAJOR == 3: - # _functools is needed to import locale during stdio initialization and - # needs to be copied on PyPy because it's not built in - REQUIRED_MODULES.append("_functools") - - -class Logger(object): - - """ - Logging object for use in command-line script. Allows ranges of - levels, to avoid some redundancy of displayed information. - """ - - DEBUG = logging.DEBUG - INFO = logging.INFO - NOTIFY = (logging.INFO + logging.WARN) / 2 - WARN = WARNING = logging.WARN - ERROR = logging.ERROR - FATAL = logging.FATAL - - LEVELS = [DEBUG, INFO, NOTIFY, WARN, ERROR, FATAL] - - def __init__(self, consumers): - self.consumers = consumers - self.indent = 0 - self.in_progress = None - self.in_progress_hanging = False - - def debug(self, msg, *args, **kw): - self.log(self.DEBUG, msg, *args, **kw) - - def info(self, msg, *args, **kw): - self.log(self.INFO, msg, *args, **kw) - - def notify(self, msg, *args, **kw): - self.log(self.NOTIFY, msg, *args, **kw) - - def warn(self, msg, *args, **kw): - self.log(self.WARN, msg, *args, **kw) - - def error(self, msg, *args, **kw): - self.log(self.ERROR, msg, *args, **kw) - - def fatal(self, msg, *args, **kw): - self.log(self.FATAL, msg, *args, **kw) - - def log(self, level, msg, *args, **kw): - if args: - if kw: - raise TypeError("You may give positional or keyword arguments, not both") - args = args or kw - rendered = None - for consumer_level, consumer in self.consumers: - if self.level_matches(level, consumer_level): - if self.in_progress_hanging and consumer in (sys.stdout, sys.stderr): - self.in_progress_hanging = False - print("") - sys.stdout.flush() - if rendered is None: - if args: - rendered = msg % args - else: - rendered = msg - rendered = " " * self.indent + rendered - if hasattr(consumer, "write"): - consumer.write(rendered + "\n") - else: - consumer(rendered) - - def start_progress(self, msg): - assert not self.in_progress, "Tried to start_progress({!r}) while in_progress {!r}".format( - msg, self.in_progress - ) - if self.level_matches(self.NOTIFY, self._stdout_level()): - print(msg) - sys.stdout.flush() - self.in_progress_hanging = True - else: - self.in_progress_hanging = False - self.in_progress = msg - - def end_progress(self, msg="done."): - assert self.in_progress, "Tried to end_progress without start_progress" - if self.stdout_level_matches(self.NOTIFY): - if not self.in_progress_hanging: - # Some message has been printed out since start_progress - print("...{}{}".format(self.in_progress, msg)) - sys.stdout.flush() - else: - print(msg) - sys.stdout.flush() - self.in_progress = None - self.in_progress_hanging = False - - def show_progress(self): - """If we are in a progress scope, and no log messages have been - shown, write out another '.'""" - if self.in_progress_hanging: - print(".") - sys.stdout.flush() - - def stdout_level_matches(self, level): - """Returns true if a message at this level will go to stdout""" - return self.level_matches(level, self._stdout_level()) - - def _stdout_level(self): - """Returns the level that stdout runs at""" - for level, consumer in self.consumers: - if consumer is sys.stdout: - return level - return self.FATAL - - @staticmethod - def level_matches(level, consumer_level): - """ - >>> l = Logger([]) - >>> l.level_matches(3, 4) - False - >>> l.level_matches(3, 2) - True - >>> l.level_matches(slice(None, 3), 3) - False - >>> l.level_matches(slice(None, 3), 2) - True - >>> l.level_matches(slice(1, 3), 1) - True - >>> l.level_matches(slice(2, 3), 1) - False - """ - if isinstance(level, slice): - start, stop = level.start, level.stop - if start is not None and start > consumer_level: - return False - if stop is not None and stop <= consumer_level: - return False - return True - else: - return level >= consumer_level - - @classmethod - def level_for_integer(cls, level): - levels = cls.LEVELS - if level < 0: - return levels[0] - if level >= len(levels): - return levels[-1] - return levels[level] - - -# create a silent logger just to prevent this from being undefined -# will be overridden with requested verbosity main() is called. -logger = Logger([(Logger.LEVELS[-1], sys.stdout)]) - - -def mkdir(at_path): - if not os.path.exists(at_path): - logger.info("Creating %s", at_path) - os.makedirs(at_path) - else: - logger.info("Directory %s already exists", at_path) - - -def copy_file_or_folder(src, dest, symlink=True): - if os.path.isdir(src): - shutil.copytree(src, dest, symlink) - else: - shutil.copy2(src, dest) - - -def copyfile(src, dest, symlink=True): - if not os.path.exists(src): - # Some bad symlink in the src - logger.warn("Cannot find file %s (bad symlink)", src) - return - if os.path.exists(dest): - logger.debug("File %s already exists", dest) - return - if not os.path.exists(os.path.dirname(dest)): - logger.info("Creating parent directories for %s", os.path.dirname(dest)) - os.makedirs(os.path.dirname(dest)) - if symlink and hasattr(os, "symlink") and not IS_WIN: - logger.info("Symlinking %s", dest) - try: - os.symlink(os.path.realpath(src), dest) - except (OSError, NotImplementedError): - logger.info("Symlinking failed, copying to %s", dest) - copy_file_or_folder(src, dest, symlink) - else: - logger.info("Copying to %s", dest) - copy_file_or_folder(src, dest, symlink) - - -def writefile(dest, content, overwrite=True): - if not os.path.exists(dest): - logger.info("Writing %s", dest) - with open(dest, "wb") as f: - f.write(content.encode("utf-8")) - return - else: - with open(dest, "rb") as f: - c = f.read() - if c != content.encode("utf-8"): - if not overwrite: - logger.notify("File %s exists with different content; not overwriting", dest) - return - logger.notify("Overwriting %s with new content", dest) - with open(dest, "wb") as f: - f.write(content.encode("utf-8")) - else: - logger.info("Content %s already in place", dest) - - -def rm_tree(folder): - if os.path.exists(folder): - logger.notify("Deleting tree %s", folder) - shutil.rmtree(folder) - else: - logger.info("Do not need to delete %s; already gone", folder) - - -def make_exe(fn): - if hasattr(os, "chmod"): - old_mode = os.stat(fn).st_mode & 0xFFF # 0o7777 - new_mode = (old_mode | 0x16D) & 0xFFF # 0o555, 0o7777 - os.chmod(fn, new_mode) - logger.info("Changed mode of %s to %s", fn, oct(new_mode)) - - -def _find_file(filename, folders): - for folder in reversed(folders): - files = glob.glob(os.path.join(folder, filename)) - if files and os.path.isfile(files[0]): - return True, files[0] - return False, filename - - -@contextlib.contextmanager -def virtualenv_support_dirs(): - """Context manager yielding either [virtualenv_support_dir] or []""" - - # normal filesystem installation - if os.path.isdir(join(HERE, "virtualenv_support")): - yield [join(HERE, "virtualenv_support")] - elif IS_ZIPAPP: - tmpdir = tempfile.mkdtemp() - try: - with zipfile.ZipFile(HERE) as zipf: - for member in zipf.namelist(): - if os.path.dirname(member) == "virtualenv_support": - zipf.extract(member, tmpdir) - yield [join(tmpdir, "virtualenv_support")] - finally: - shutil.rmtree(tmpdir) - # probably a bootstrap script - elif os.path.splitext(os.path.dirname(__file__))[0] != "virtualenv": - try: - # noinspection PyUnresolvedReferences - import virtualenv - except ImportError: - yield [] - else: - yield [join(os.path.dirname(virtualenv.__file__), "virtualenv_support")] - # we tried! - else: - yield [] - - -class UpdatingDefaultsHelpFormatter(optparse.IndentedHelpFormatter): - """ - Custom help formatter for use in ConfigOptionParser that updates - the defaults before expanding them, allowing them to show up correctly - in the help listing - """ - - def expand_default(self, option): - if self.parser is not None: - self.parser.update_defaults(self.parser.defaults) - return optparse.IndentedHelpFormatter.expand_default(self, option) - - -class ConfigOptionParser(optparse.OptionParser): - """ - Custom option parser which updates its defaults by checking the - configuration files and environmental variables - """ - - def __init__(self, *args, **kwargs): - self.config = ConfigParser.RawConfigParser() - self.files = self.get_config_files() - self.config.read(self.files) - optparse.OptionParser.__init__(self, *args, **kwargs) - - @staticmethod - def get_config_files(): - config_file = os.environ.get("VIRTUALENV_CONFIG_FILE", False) - if config_file and os.path.exists(config_file): - return [config_file] - return [DEFAULT_CONFIG_FILE] - - def update_defaults(self, defaults): - """ - Updates the given defaults with values from the config files and - the environ. Does a little special handling for certain types of - options (lists). - """ - # Then go and look for the other sources of configuration: - config = {} - # 1. config files - config.update(dict(self.get_config_section("virtualenv"))) - # 2. environmental variables - config.update(dict(self.get_environ_vars())) - # Then set the options with those values - for key, val in config.items(): - key = key.replace("_", "-") - if not key.startswith("--"): - key = "--{}".format(key) # only prefer long opts - option = self.get_option(key) - if option is not None: - # ignore empty values - if not val: - continue - # handle multiline configs - if option.action == "append": - val = val.split() - else: - option.nargs = 1 - if option.action == "store_false": - val = not strtobool(val) - elif option.action in ("store_true", "count"): - val = strtobool(val) - try: - val = option.convert_value(key, val) - except optparse.OptionValueError: - e = sys.exc_info()[1] - print("An error occurred during configuration: {!r}".format(e)) - sys.exit(3) - defaults[option.dest] = val - return defaults - - def get_config_section(self, name): - """ - Get a section of a configuration - """ - if self.config.has_section(name): - return self.config.items(name) - return [] - - def get_environ_vars(self, prefix="VIRTUALENV_"): - """ - Returns a generator with all environmental vars with prefix VIRTUALENV - """ - for key, val in os.environ.items(): - if key.startswith(prefix): - yield (key.replace(prefix, "").lower(), val) - - def get_default_values(self): - """ - Overriding to make updating the defaults after instantiation of - the option parser possible, update_defaults() does the dirty work. - """ - if not self.process_default_values: - # Old, pre-Optik 1.5 behaviour. - return optparse.Values(self.defaults) - - defaults = self.update_defaults(self.defaults.copy()) # ours - for option in self._get_all_options(): - default = defaults.get(option.dest) - if isinstance(default, basestring): - opt_str = option.get_opt_string() - defaults[option.dest] = option.check_value(opt_str, default) - return optparse.Values(defaults) - - -def main(): - parser = ConfigOptionParser( - version=virtualenv_version, usage="%prog [OPTIONS] DEST_DIR", formatter=UpdatingDefaultsHelpFormatter() - ) - - parser.add_option( - "-v", "--verbose", action="count", dest="verbose", default=5 if DEBUG else 0, help="Increase verbosity." - ) - - parser.add_option("-q", "--quiet", action="count", dest="quiet", default=0, help="Decrease verbosity.") - - parser.add_option( - "-p", - "--python", - dest="python", - metavar="PYTHON_EXE", - help="The Python interpreter to use, e.g., --python=python3.5 will use the python3.5 " - "interpreter to create the new environment. The default is the interpreter that " - "virtualenv was installed with ({})".format(sys.executable), - ) - - parser.add_option( - "--clear", dest="clear", action="store_true", help="Clear out the non-root install and start from scratch." - ) - - parser.set_defaults(system_site_packages=False) - parser.add_option( - "--no-site-packages", - dest="system_site_packages", - action="store_false", - help="DEPRECATED. Retained only for backward compatibility. " - "Not having access to global site-packages is now the default behavior.", - ) - - parser.add_option( - "--system-site-packages", - dest="system_site_packages", - action="store_true", - help="Give the virtual environment access to the global site-packages.", - ) - - parser.add_option( - "--always-copy", - dest="symlink", - action="store_false", - default=True, - help="Always copy files rather than symlinking.", - ) - - parser.add_option( - "--relocatable", - dest="relocatable", - action="store_true", - help="Make an EXISTING virtualenv environment relocatable. " - "This fixes up scripts and makes all .pth files relative.", - ) - - parser.add_option( - "--no-setuptools", - dest="no_setuptools", - action="store_true", - help="Do not install setuptools in the new virtualenv.", - ) - - parser.add_option("--no-pip", dest="no_pip", action="store_true", help="Do not install pip in the new virtualenv.") - - parser.add_option( - "--no-wheel", dest="no_wheel", action="store_true", help="Do not install wheel in the new virtualenv." - ) - - parser.add_option( - "--extra-search-dir", - dest="search_dirs", - action="append", - metavar="DIR", - default=[], - help="Directory to look for setuptools/pip distributions in. " "This option can be used multiple times.", - ) - - parser.add_option( - "--download", - dest="download", - default=True, - action="store_true", - help="Download pre-installed packages from PyPI.", - ) - - parser.add_option( - "--no-download", - "--never-download", - dest="download", - action="store_false", - help="Do not download pre-installed packages from PyPI.", - ) - - parser.add_option("--prompt", dest="prompt", help="Provides an alternative prompt prefix for this environment.") - - parser.add_option( - "--setuptools", - dest="setuptools", - action="store_true", - help="DEPRECATED. Retained only for backward compatibility. This option has no effect.", - ) - - parser.add_option( - "--distribute", - dest="distribute", - action="store_true", - help="DEPRECATED. Retained only for backward compatibility. This option has no effect.", - ) - - parser.add_option( - "--unzip-setuptools", - action="store_true", - help="DEPRECATED. Retained only for backward compatibility. This option has no effect.", - ) - - if "extend_parser" in globals(): - # noinspection PyUnresolvedReferences - extend_parser(parser) # noqa: F821 - - options, args = parser.parse_args() - - global logger - - if "adjust_options" in globals(): - # noinspection PyUnresolvedReferences - adjust_options(options, args) # noqa: F821 - - verbosity = options.verbose - options.quiet - logger = Logger([(Logger.level_for_integer(2 - verbosity), sys.stdout)]) - - def should_reinvoke(options): - """Do we need to reinvoke ourself?""" - # Did the user specify the --python option? - if options.python and not os.environ.get("VIRTUALENV_INTERPRETER_RUNNING"): - interpreter = resolve_interpreter(options.python) - if interpreter != sys.executable: - # The user specified a different interpreter, so we have to reinvoke. - return interpreter - - # At this point, we know the user wants to use sys.executable to create the - # virtual environment. But on Windows, sys.executable may be a venv redirector, - # in which case we still need to locate the underlying actual interpreter, and - # reinvoke using that. - if IS_WIN: - # OK. Now things get really fun... - # - # If we are running from a venv, with a redirector, then what happens is as - # follows: - # - # 1. The redirector sets __PYVENV_LAUNCHER__ in the environment to point - # to the redirector executable. - # 2. The redirector launches the "base" Python (from the home value in - # pyvenv.cfg). - # 3. The base Python executable sees __PYVENV_LAUNCHER__ in the environment - # and sets sys.executable to that value. - # 4. If site.py gets run, it sees __PYVENV_LAUNCHER__, and sets - # sys._base_executable to _winapi.GetModuleFileName(0) and removes - # __PYVENV_LAUNCHER__. - # - # Unfortunately, that final step (site.py) may not happen. There are 2 key - # times when that is the case: - # - # 1. Python 3.7.2, which had the redirector but not the site.py code. - # 2. Running a venv from a virtualenv, which uses virtualenv's custom - # site.py. - # - # So, we check for sys._base_executable, but if it's not present and yet we - # hand __PYVENV_LAUNCHER__, we do what site.py would have done and get our - # interpreter from GetModuleFileName(0). We also remove __PYVENV_LAUNCHER__ - # from the environment, to avoid loops (actually, mainly because site.py - # does so, and my head hurts enough buy now that I just want to be safe!) - - # Phew. - - if hasattr(sys, "_base_executable"): - return sys._base_executable - - if "__PYVENV_LAUNCHER__" in os.environ: - import _winapi - - del os.environ["__PYVENV_LAUNCHER__"] - return _winapi.GetModuleFileName(0) - - # We don't need to reinvoke - return None - - interpreter = should_reinvoke(options) - if interpreter is None: - # We don't need to reinvoke - if the user asked us to, tell them why we - # aren't. - if options.python: - logger.warn("Already using interpreter {}".format(sys.executable)) - else: - env = os.environ.copy() - logger.notify("Running virtualenv with interpreter {}".format(interpreter)) - env["VIRTUALENV_INTERPRETER_RUNNING"] = "true" - # Remove the variable __PYVENV_LAUNCHER__ if it's present, as it causes the - # interpreter to redirect back to the virtual environment. - if "__PYVENV_LAUNCHER__" in env: - del env["__PYVENV_LAUNCHER__"] - file = __file__ - if file.endswith(".pyc"): - file = file[:-1] - elif IS_ZIPAPP: - file = HERE - sub_process_call = subprocess.Popen([interpreter, file] + sys.argv[1:], env=env) - raise SystemExit(sub_process_call.wait()) - - if not args: - print("You must provide a DEST_DIR") - parser.print_help() - sys.exit(2) - if len(args) > 1: - print("There must be only one argument: DEST_DIR (you gave {})".format(" ".join(args))) - parser.print_help() - sys.exit(2) - - home_dir = args[0] - - if os.path.exists(home_dir) and os.path.isfile(home_dir): - logger.fatal("ERROR: File already exists and is not a directory.") - logger.fatal("Please provide a different path or delete the file.") - sys.exit(3) - - if os.pathsep in home_dir: - logger.fatal("ERROR: target path contains the operating system path separator '{}'".format(os.pathsep)) - logger.fatal("This is not allowed as would make the activation scripts unusable.".format(os.pathsep)) - sys.exit(3) - - if os.environ.get("WORKING_ENV"): - logger.fatal("ERROR: you cannot run virtualenv while in a working env") - logger.fatal("Please deactivate your working env, then re-run this script") - sys.exit(3) - - if "PYTHONHOME" in os.environ: - logger.warn("PYTHONHOME is set. You *must* activate the virtualenv before using it") - del os.environ["PYTHONHOME"] - - if options.relocatable: - make_environment_relocatable(home_dir) - return - - with virtualenv_support_dirs() as search_dirs: - create_environment( - home_dir, - site_packages=options.system_site_packages, - clear=options.clear, - prompt=options.prompt, - search_dirs=search_dirs + options.search_dirs, - download=options.download, - no_setuptools=options.no_setuptools, - no_pip=options.no_pip, - no_wheel=options.no_wheel, - symlink=options.symlink, - ) - if "after_install" in globals(): - # noinspection PyUnresolvedReferences - after_install(options, home_dir) # noqa: F821 - - -def call_subprocess( - cmd, - show_stdout=True, - filter_stdout=None, - cwd=None, - raise_on_return_code=True, - extra_env=None, - remove_from_env=None, - stdin=None, -): - cmd_parts = [] - for part in cmd: - if len(part) > 45: - part = part[:20] + "..." + part[-20:] - if " " in part or "\n" in part or '"' in part or "'" in part: - part = '"{}"'.format(part.replace('"', '\\"')) - if hasattr(part, "decode"): - try: - part = part.decode(sys.getdefaultencoding()) - except UnicodeDecodeError: - part = part.decode(sys.getfilesystemencoding()) - cmd_parts.append(part) - cmd_desc = " ".join(cmd_parts) - if show_stdout: - stdout = None - else: - stdout = subprocess.PIPE - logger.debug("Running command {}".format(cmd_desc)) - if extra_env or remove_from_env: - env = os.environ.copy() - if extra_env: - env.update(extra_env) - if remove_from_env: - for var_name in remove_from_env: - env.pop(var_name, None) - else: - env = None - try: - proc = subprocess.Popen( - cmd, - stderr=subprocess.STDOUT, - stdin=None if stdin is None else subprocess.PIPE, - stdout=stdout, - cwd=cwd, - env=env, - ) - except Exception: - e = sys.exc_info()[1] - logger.fatal("Error {} while executing command {}".format(e, cmd_desc)) - raise - all_output = [] - if stdout is not None: - if stdin is not None: - with proc.stdin: - proc.stdin.write(stdin) - - encoding = sys.getdefaultencoding() - fs_encoding = sys.getfilesystemencoding() - with proc.stdout as stdout: - while 1: - line = stdout.readline() - try: - line = line.decode(encoding) - except UnicodeDecodeError: - line = line.decode(fs_encoding) - if not line: - break - line = line.rstrip() - all_output.append(line) - if filter_stdout: - level = filter_stdout(line) - if isinstance(level, tuple): - level, line = level - logger.log(level, line) - if not logger.stdout_level_matches(level): - logger.show_progress() - else: - logger.info(line) - else: - proc.communicate(stdin) - proc.wait() - if proc.returncode: - if raise_on_return_code: - if all_output: - logger.notify("Complete output from command {}:".format(cmd_desc)) - logger.notify("\n".join(all_output) + "\n----------------------------------------") - raise OSError("Command {} failed with error code {}".format(cmd_desc, proc.returncode)) - else: - logger.warn("Command {} had error code {}".format(cmd_desc, proc.returncode)) - return all_output - - -def filter_install_output(line): - if line.strip().startswith("running"): - return Logger.INFO - return Logger.DEBUG - - -def find_wheels(projects, search_dirs): - """Find wheels from which we can import PROJECTS. - - Scan through SEARCH_DIRS for a wheel for each PROJECT in turn. Return - a list of the first wheel found for each PROJECT - """ - - wheels = [] - - # Look through SEARCH_DIRS for the first suitable wheel. Don't bother - # about version checking here, as this is simply to get something we can - # then use to install the correct version. - for project in projects: - for dirname in search_dirs: - # This relies on only having "universal" wheels available. - # The pattern could be tightened to require -py2.py3-none-any.whl. - files = glob.glob(os.path.join(dirname, project + "-*.whl")) - if files: - wheels.append(os.path.abspath(files[0])) - break - else: - # We're out of luck, so quit with a suitable error - logger.fatal("Cannot find a wheel for {}".format(project)) - - return wheels - - -def install_wheel(project_names, py_executable, search_dirs=None, download=False): - if search_dirs is None: - search_dirs_context = virtualenv_support_dirs - else: - - @contextlib.contextmanager - def search_dirs_context(): - yield search_dirs - - with search_dirs_context() as search_dirs: - _install_wheel_with_search_dir(download, project_names, py_executable, search_dirs) - - -def _install_wheel_with_search_dir(download, project_names, py_executable, search_dirs): - wheels = find_wheels(["setuptools", "pip"], search_dirs) - python_path = os.pathsep.join(wheels) - - # PIP_FIND_LINKS uses space as the path separator and thus cannot have paths - # with spaces in them. Convert any of those to local file:// URL form. - try: - from urlparse import urljoin - from urllib import pathname2url - except ImportError: - from urllib.parse import urljoin - from urllib.request import pathname2url - - def space_path2url(p): - if " " not in p: - return p - return urljoin("file:", pathname2url(os.path.abspath(p))) - - find_links = " ".join(space_path2url(d) for d in search_dirs) - - extra_args = ["--ignore-installed"] - if DEBUG: - extra_args.append("-v") - - config = _pip_config(py_executable, python_path) - defined_cert = bool(config.get("install.cert") or config.get(":env:.cert") or config.get("global.cert")) - - script = textwrap.dedent( - """ - import sys - import pkgutil - import tempfile - import os - - defined_cert = {defined_cert} - - try: - from pip._internal import main as _main - cert_data = pkgutil.get_data("pip._vendor.certifi", "cacert.pem") - except ImportError: - from pip import main as _main - cert_data = pkgutil.get_data("pip._vendor.requests", "cacert.pem") - except IOError: - cert_data = None - - if not defined_cert and cert_data is not None: - cert_file = tempfile.NamedTemporaryFile(delete=False) - cert_file.write(cert_data) - cert_file.close() - else: - cert_file = None - - try: - args = ["install"] + [{extra_args}] - if cert_file is not None: - args += ["--cert", cert_file.name] - args += sys.argv[1:] - - sys.exit(_main(args)) - finally: - if cert_file is not None: - os.remove(cert_file.name) - """.format( - defined_cert=defined_cert, extra_args=", ".join(repr(i) for i in extra_args) - ) - ).encode("utf8") - - cmd = [py_executable, "-"] + project_names - logger.start_progress("Installing {}...".format(", ".join(project_names))) - logger.indent += 2 - - env = { - "PYTHONPATH": python_path, - "PIP_FIND_LINKS": find_links, - "PIP_USE_WHEEL": "1", - "PIP_ONLY_BINARY": ":all:", - "PIP_USER": "0", - "PIP_NO_INPUT": "1", - } - - if not download: - env["PIP_NO_INDEX"] = "1" - - try: - call_subprocess(cmd, show_stdout=False, extra_env=env, stdin=script) - finally: - logger.indent -= 2 - logger.end_progress() - - -def _pip_config(py_executable, python_path): - cmd = [py_executable, "-m", "pip", "config", "list"] - config = {} - for line in call_subprocess( - cmd, - show_stdout=False, - extra_env={"PYTHONPATH": python_path}, - remove_from_env=["PIP_VERBOSE", "PIP_QUIET"], - raise_on_return_code=False, - ): - key, _, value = line.partition("=") - if value: - config[key] = ast.literal_eval(value) - return config - - -def create_environment( - home_dir, - site_packages=False, - clear=False, - prompt=None, - search_dirs=None, - download=False, - no_setuptools=False, - no_pip=False, - no_wheel=False, - symlink=True, -): - """ - Creates a new environment in ``home_dir``. - - If ``site_packages`` is true, then the global ``site-packages/`` - directory will be on the path. - - If ``clear`` is true (default False) then the environment will - first be cleared. - """ - home_dir, lib_dir, inc_dir, bin_dir = path_locations(home_dir) - - py_executable = os.path.abspath( - install_python(home_dir, lib_dir, inc_dir, bin_dir, site_packages=site_packages, clear=clear, symlink=symlink) - ) - - install_distutils(home_dir) - - to_install = [] - - if not no_setuptools: - to_install.append("setuptools") - - if not no_pip: - to_install.append("pip") - - if not no_wheel: - to_install.append("wheel") - - if to_install: - install_wheel(to_install, py_executable, search_dirs, download=download) - - install_activate(home_dir, bin_dir, prompt) - - install_python_config(home_dir, bin_dir, prompt) - - -def is_executable_file(fpath): - return os.path.isfile(fpath) and is_executable(fpath) - - -def path_locations(home_dir, dry_run=False): - """Return the path locations for the environment (where libraries are, - where scripts go, etc)""" - home_dir = os.path.abspath(home_dir) - lib_dir, inc_dir, bin_dir = None, None, None - # XXX: We'd use distutils.sysconfig.get_python_inc/lib but its - # prefix arg is broken: http://bugs.python.org/issue3386 - if IS_WIN: - # Windows has lots of problems with executables with spaces in - # the name; this function will remove them (using the ~1 - # format): - if not dry_run: - mkdir(home_dir) - if " " in home_dir: - import ctypes - - get_short_path_name = ctypes.windll.kernel32.GetShortPathNameW - size = max(len(home_dir) + 1, 256) - buf = ctypes.create_unicode_buffer(size) - try: - # noinspection PyUnresolvedReferences - u = unicode - except NameError: - u = str - ret = get_short_path_name(u(home_dir), buf, size) - if not ret: - print('Error: the path "{}" has a space in it'.format(home_dir)) - print("We could not determine the short pathname for it.") - print("Exiting.") - sys.exit(3) - home_dir = str(buf.value) - lib_dir = join(home_dir, "Lib") - inc_dir = join(home_dir, "Include") - bin_dir = join(home_dir, "Scripts") - elif IS_PYPY: - lib_dir = home_dir - inc_dir = join(home_dir, "include") - bin_dir = join(home_dir, "bin") - elif not IS_WIN: - lib_dir = join(home_dir, "lib", PY_VERSION) - inc_dir = join(home_dir, "include", PY_VERSION + ABI_FLAGS) - bin_dir = join(home_dir, "bin") - return home_dir, lib_dir, inc_dir, bin_dir - - -def change_prefix(filename, dst_prefix): - prefixes = [sys.prefix] - - if IS_DARWIN: - prefixes.extend( - ( - os.path.join("/Library/Python", sys.version[:3], "site-packages"), - os.path.join(sys.prefix, "Extras", "lib", "python"), - os.path.join("~", "Library", "Python", sys.version[:3], "site-packages"), - # Python 2.6 no-frameworks - os.path.join("~", ".local", "lib", "python", sys.version[:3], "site-packages"), - # System Python 2.7 on OSX Mountain Lion - os.path.join("~", "Library", "Python", sys.version[:3], "lib", "python", "site-packages"), - ) - ) - - if hasattr(sys, "real_prefix"): - prefixes.append(sys.real_prefix) - if hasattr(sys, "base_prefix"): - prefixes.append(sys.base_prefix) - prefixes = list(map(os.path.expanduser, prefixes)) - prefixes = list(map(os.path.abspath, prefixes)) - # Check longer prefixes first so we don't split in the middle of a filename - prefixes = sorted(prefixes, key=len, reverse=True) - filename = os.path.abspath(filename) - # On Windows, make sure drive letter is uppercase - if IS_WIN and filename[0] in "abcdefghijklmnopqrstuvwxyz": - filename = filename[0].upper() + filename[1:] - for i, prefix in enumerate(prefixes): - if IS_WIN and prefix[0] in "abcdefghijklmnopqrstuvwxyz": - prefixes[i] = prefix[0].upper() + prefix[1:] - for src_prefix in prefixes: - if filename.startswith(src_prefix): - _, relative_path = filename.split(src_prefix, 1) - if src_prefix != os.sep: # sys.prefix == "/" - assert relative_path[0] == os.sep - relative_path = relative_path[1:] - return join(dst_prefix, relative_path) - assert False, "Filename {} does not start with any of these prefixes: {}".format(filename, prefixes) - - -def find_module_filename(modname): - - if sys.version_info < (3, 4): - # noinspection PyDeprecation - import imp - - try: - file_handler, filepath, _ = imp.find_module(modname) - except ImportError: - return None - else: - if file_handler is not None: - file_handler.close() - return filepath - else: - import importlib.util - - if sys.version_info < (3, 5): - - def find_spec(modname): - # noinspection PyDeprecation - loader = importlib.find_loader(modname) - if loader is None: - return None - else: - return importlib.util.spec_from_loader(modname, loader) - - else: - find_spec = importlib.util.find_spec - - spec = find_spec(modname) - if spec is None: - return None - if not os.path.exists(spec.origin): - # https://bitbucket.org/pypy/pypy/issues/2944/origin-for-several-builtin-modules - # on pypy3, some builtin modules have a bogus build-time file path, ignore them - return None - filepath = spec.origin - # https://www.python.org/dev/peps/pep-3147/#file guarantee to be non-cached - if os.path.basename(filepath) == "__init__.py": - filepath = os.path.dirname(filepath) - return filepath - - -def copy_required_modules(dst_prefix, symlink): - for modname in REQUIRED_MODULES: - if modname in sys.builtin_module_names: - logger.info("Ignoring built-in bootstrap module: %s" % modname) - continue - filename = find_module_filename(modname) - if filename is None: - logger.info("Cannot import bootstrap module: %s" % modname) - else: - # special-case custom readline.so on OS X, but not for pypy: - if ( - modname == "readline" - and IS_DARWIN - and not (IS_PYPY or filename.endswith(join("lib-dynload", "readline.so"))) - ): - dst_filename = join(dst_prefix, "lib", "python{}".format(sys.version[:3]), "readline.so") - elif modname == "readline" and IS_WIN: - # special-case for Windows, where readline is not a standard module, though it may have been installed - # in site-packages by a third-party package - dst_filename = None - else: - dst_filename = change_prefix(filename, dst_prefix) - if dst_filename is not None: - copyfile(filename, dst_filename, symlink) - if filename.endswith(".pyc"): - py_file = filename[:-1] - if os.path.exists(py_file): - copyfile(py_file, dst_filename[:-1], symlink) - - -def copy_required_files(src_dir, lib_dir, symlink): - if not os.path.isdir(src_dir): - return - for fn in os.listdir(src_dir): - bn = os.path.splitext(fn)[0] - if fn != "site-packages" and bn in REQUIRED_FILES: - copyfile(join(src_dir, fn), join(lib_dir, fn), symlink) - - -def copy_license(prefix, dst_prefix, lib_dir, symlink): - """Copy the license file so `license()` builtin works""" - for license_path in ( - # posix cpython - os.path.join(prefix, os.path.relpath(lib_dir, dst_prefix), "LICENSE.txt"), - # windows cpython - os.path.join(prefix, "LICENSE.txt"), - # pypy - os.path.join(prefix, "LICENSE"), - ): - if os.path.exists(license_path): - dest = subst_path(license_path, prefix, dst_prefix) - copyfile(license_path, dest, symlink) - return - logger.warn("No LICENSE.txt / LICENSE found in source") - - -def copy_include_dir(include_src, include_dest, symlink): - """Copy headers from *include_src* to *include_dest* symlinking if required""" - if not os.path.isdir(include_src): - return - # PyPy headers are located in ``pypy-dir/include`` and following code - # avoids making ``venv-dir/include`` symlink to it - if IS_PYPY: - for fn in os.listdir(include_src): - copyfile(join(include_src, fn), join(include_dest, fn), symlink) - else: - copyfile(include_src, include_dest, symlink) - - -def copy_tcltk(src, dest, symlink): - """ copy tcl/tk libraries on Windows (issue #93) """ - for lib_version in "8.5", "8.6": - for libname in "tcl", "tk": - src_dir = join(src, "tcl", libname + lib_version) - dest_dir = join(dest, "tcl", libname + lib_version) - # Only copy the dirs from the above combinations that exist - if os.path.exists(src_dir) and not os.path.exists(dest_dir): - copy_file_or_folder(src_dir, dest_dir, symlink) - - -def subst_path(prefix_path, prefix, home_dir): - prefix_path = os.path.normpath(prefix_path) - prefix = os.path.normpath(prefix) - home_dir = os.path.normpath(home_dir) - if not prefix_path.startswith(prefix): - logger.warn("Path not in prefix %r %r", prefix_path, prefix) - return - return prefix_path.replace(prefix, home_dir, 1) - - -def install_python(home_dir, lib_dir, inc_dir, bin_dir, site_packages, clear, symlink=True): - """Install just the base environment, no distutils patches etc""" - if sys.executable.startswith(bin_dir): - print("Please use the *system* python to run this script") - return - - if clear: - rm_tree(lib_dir) - # FIXME: why not delete it? - # Maybe it should delete everything with #!/path/to/venv/python in it - logger.notify("Not deleting %s", bin_dir) - - if hasattr(sys, "real_prefix"): - logger.notify("Using real prefix %r", sys.real_prefix) - prefix = sys.real_prefix - elif hasattr(sys, "base_prefix"): - logger.notify("Using base prefix %r", sys.base_prefix) - prefix = sys.base_prefix - else: - prefix = sys.prefix - prefix = os.path.abspath(prefix) - mkdir(lib_dir) - fix_lib64(lib_dir, symlink) - stdlib_dirs = [os.path.dirname(os.__file__)] - if IS_WIN: - stdlib_dirs.append(join(os.path.dirname(stdlib_dirs[0]), "DLLs")) - elif IS_DARWIN: - stdlib_dirs.append(join(stdlib_dirs[0], "site-packages")) - if hasattr(os, "symlink"): - logger.info("Symlinking Python bootstrap modules") - else: - logger.info("Copying Python bootstrap modules") - logger.indent += 2 - try: - # copy required files... - for stdlib_dir in stdlib_dirs: - copy_required_files(stdlib_dir, lib_dir, symlink) - # ...and modules - copy_required_modules(home_dir, symlink) - copy_license(prefix, home_dir, lib_dir, symlink) - finally: - logger.indent -= 2 - # ...copy tcl/tk - if IS_WIN: - copy_tcltk(prefix, home_dir, symlink) - mkdir(join(lib_dir, "site-packages")) - import site - - site_filename = site.__file__ - if site_filename.endswith(".pyc") or site_filename.endswith(".pyo"): - site_filename = site_filename[:-1] - elif site_filename.endswith("$py.class"): - site_filename = site_filename.replace("$py.class", ".py") - site_filename_dst = change_prefix(site_filename, home_dir) - site_dir = os.path.dirname(site_filename_dst) - writefile(site_filename_dst, SITE_PY) - writefile(join(site_dir, "orig-prefix.txt"), prefix) - site_packages_filename = join(site_dir, "no-global-site-packages.txt") - if not site_packages: - writefile(site_packages_filename, "") - - if IS_PYPY or IS_WIN: - standard_lib_include_dir = join(prefix, "include") - else: - standard_lib_include_dir = join(prefix, "include", PY_VERSION + ABI_FLAGS) - if os.path.exists(standard_lib_include_dir): - copy_include_dir(standard_lib_include_dir, inc_dir, symlink) - else: - logger.debug("No include dir %s", standard_lib_include_dir) - - platform_include_dir = distutils.sysconfig.get_python_inc(plat_specific=1) - if platform_include_dir != standard_lib_include_dir: - platform_include_dest = distutils.sysconfig.get_python_inc(plat_specific=1, prefix=home_dir) - if platform_include_dir == platform_include_dest: - # Do platinc_dest manually due to a CPython bug; - # not http://bugs.python.org/issue3386 but a close cousin - platform_include_dest = subst_path(platform_include_dir, prefix, home_dir) - if platform_include_dest: - # PyPy's stdinc_dir and prefix are relative to the original binary - # (traversing virtualenvs), whereas the platinc_dir is relative to - # the inner virtualenv and ignores the prefix argument. - # This seems more evolved than designed. - copy_include_dir(platform_include_dir, platform_include_dest, symlink) - - # pypy never uses exec_prefix, just ignore it - if os.path.realpath(sys.exec_prefix) != os.path.realpath(prefix) and not IS_PYPY: - if IS_WIN: - exec_dir = join(sys.exec_prefix, "lib") - else: - exec_dir = join(sys.exec_prefix, "lib", PY_VERSION) - copy_required_files(exec_dir, lib_dir, symlink) - - mkdir(bin_dir) - py_executable = join(bin_dir, os.path.basename(sys.executable)) - if "Python.framework" in prefix: - # OS X framework builds cause validation to break - # https://github.com/pypa/virtualenv/issues/322 - if os.environ.get("__PYVENV_LAUNCHER__"): - del os.environ["__PYVENV_LAUNCHER__"] - if re.search(r"/Python(?:-32|-64)*$", py_executable): - # The name of the python executable is not quite what - # we want, rename it. - py_executable = os.path.join(os.path.dirname(py_executable), "python") - - logger.notify("New %s executable in %s", EXPECTED_EXE, py_executable) - pc_build_dir = os.path.dirname(sys.executable) - pyd_pth = os.path.join(lib_dir, "site-packages", "virtualenv_builddir_pyd.pth") - if IS_WIN and os.path.exists(os.path.join(pc_build_dir, "build.bat")): - logger.notify("Detected python running from build directory %s", pc_build_dir) - logger.notify("Writing .pth file linking to build directory for *.pyd files") - writefile(pyd_pth, pc_build_dir) - else: - if os.path.exists(pyd_pth): - logger.info("Deleting %s (not Windows env or not build directory python)", pyd_pth) - os.unlink(pyd_pth) - - if sys.executable != py_executable: - # FIXME: could I just hard link? - executable = sys.executable - shutil.copyfile(executable, py_executable) - make_exe(py_executable) - if IS_WIN or IS_CYGWIN: - python_w = os.path.join(os.path.dirname(sys.executable), "pythonw.exe") - if os.path.exists(python_w): - logger.info("Also created pythonw.exe") - shutil.copyfile(python_w, os.path.join(os.path.dirname(py_executable), "pythonw.exe")) - python_d = os.path.join(os.path.dirname(sys.executable), "python_d.exe") - python_d_dest = os.path.join(os.path.dirname(py_executable), "python_d.exe") - if os.path.exists(python_d): - logger.info("Also created python_d.exe") - shutil.copyfile(python_d, python_d_dest) - elif os.path.exists(python_d_dest): - logger.info("Removed python_d.exe as it is no longer at the source") - os.unlink(python_d_dest) - - # we need to copy the DLL to enforce that windows will load the correct one. - # may not exist if we are cygwin. - if IS_PYPY: - py_executable_dll_s = [("libpypy-c.dll", "libpypy_d-c.dll")] - else: - py_executable_dll_s = [ - ("python{}.dll".format(sys.version_info[0]), "python{}_d.dll".format(sys.version_info[0])), - ( - "python{}{}.dll".format(sys.version_info[0], sys.version_info[1]), - "python{}{}_d.dll".format(sys.version_info[0], sys.version_info[1]), - ), - ] - - for py_executable_dll, py_executable_dll_d in py_executable_dll_s: - python_dll = os.path.join(os.path.dirname(sys.executable), py_executable_dll) - python_dll_d = os.path.join(os.path.dirname(sys.executable), py_executable_dll_d) - python_dll_d_dest = os.path.join(os.path.dirname(py_executable), py_executable_dll_d) - if os.path.exists(python_dll): - logger.info("Also created %s", py_executable_dll) - shutil.copyfile(python_dll, os.path.join(os.path.dirname(py_executable), py_executable_dll)) - if os.path.exists(python_dll_d): - logger.info("Also created %s", py_executable_dll_d) - shutil.copyfile(python_dll_d, python_dll_d_dest) - elif os.path.exists(python_dll_d_dest): - logger.info("Removed %s as the source does not exist", python_dll_d_dest) - os.unlink(python_dll_d_dest) - if IS_PYPY: - # make a symlink python --> pypy-c - python_executable = os.path.join(os.path.dirname(py_executable), "python") - if IS_WIN or IS_CYGWIN: - python_executable += ".exe" - logger.info("Also created executable %s", python_executable) - copyfile(py_executable, python_executable, symlink) - - if IS_WIN: - for name in ["libexpat.dll", "libeay32.dll", "ssleay32.dll", "sqlite3.dll", "tcl85.dll", "tk85.dll"]: - src = join(prefix, name) - if os.path.exists(src): - copyfile(src, join(bin_dir, name), symlink) - - for d in sys.path: - if d.endswith("lib_pypy"): - break - else: - logger.fatal("Could not find lib_pypy in sys.path") - raise SystemExit(3) - logger.info("Copying lib_pypy") - copyfile(d, os.path.join(home_dir, "lib_pypy"), symlink) - - if os.path.splitext(os.path.basename(py_executable))[0] != EXPECTED_EXE: - secondary_exe = os.path.join(os.path.dirname(py_executable), EXPECTED_EXE) - py_executable_ext = os.path.splitext(py_executable)[1] - if py_executable_ext.lower() == ".exe": - # python2.4 gives an extension of '.4' :P - secondary_exe += py_executable_ext - if os.path.exists(secondary_exe): - logger.warn( - "Not overwriting existing {} script {} (you must use {})".format( - EXPECTED_EXE, secondary_exe, py_executable - ) - ) - else: - logger.notify("Also creating executable in %s", secondary_exe) - shutil.copyfile(sys.executable, secondary_exe) - make_exe(secondary_exe) - - if ".framework" in prefix: - original_python = None - if "Python.framework" in prefix: - logger.debug("MacOSX Python framework detected") - # Make sure we use the embedded interpreter inside - # the framework, even if sys.executable points to - # the stub executable in ${sys.prefix}/bin - # See http://groups.google.com/group/python-virtualenv/ - # browse_thread/thread/17cab2f85da75951 - original_python = os.path.join(prefix, "Resources/Python.app/Contents/MacOS/Python") - if "EPD" in prefix: - logger.debug("EPD framework detected") - original_python = os.path.join(prefix, "bin/python") - shutil.copy(original_python, py_executable) - - # Copy the framework's dylib into the virtual - # environment - virtual_lib = os.path.join(home_dir, ".Python") - - if os.path.exists(virtual_lib): - os.unlink(virtual_lib) - copyfile(os.path.join(prefix, "Python"), virtual_lib, symlink) - - # And then change the install_name of the copied python executable - # noinspection PyBroadException - try: - mach_o_change(py_executable, os.path.join(prefix, "Python"), "@executable_path/../.Python") - except Exception: - e = sys.exc_info()[1] - logger.warn("Could not call mach_o_change: %s. " "Trying to call install_name_tool instead.", e) - try: - call_subprocess( - [ - "install_name_tool", - "-change", - os.path.join(prefix, "Python"), - "@executable_path/../.Python", - py_executable, - ] - ) - except Exception: - logger.fatal("Could not call install_name_tool -- you must " "have Apple's development tools installed") - raise - - if not IS_WIN: - # Ensure that 'python', 'pythonX' and 'pythonX.Y' all exist - py_exe_version_major = "python{}".format(sys.version_info[0]) - py_exe_version_major_minor = "python{}.{}".format(sys.version_info[0], sys.version_info[1]) - py_exe_no_version = "python" - required_symlinks = [py_exe_no_version, py_exe_version_major, py_exe_version_major_minor] - - py_executable_base = os.path.basename(py_executable) - - if py_executable_base in required_symlinks: - # Don't try to symlink to yourself. - required_symlinks.remove(py_executable_base) - - for pth in required_symlinks: - full_pth = join(bin_dir, pth) - if os.path.exists(full_pth): - os.unlink(full_pth) - if symlink: - os.symlink(py_executable_base, full_pth) - else: - copyfile(py_executable, full_pth, symlink) - - cmd = [ - py_executable, - "-c", - "import sys;out=sys.stdout;" 'getattr(out, "buffer", out).write(sys.prefix.encode("utf-8"))', - ] - logger.info('Testing executable with %s %s "%s"', *cmd) - try: - proc = subprocess.Popen(cmd, stdout=subprocess.PIPE) - proc_stdout, proc_stderr = proc.communicate() - except OSError: - e = sys.exc_info()[1] - if e.errno == errno.EACCES: - logger.fatal("ERROR: The executable {} could not be run: {}".format(py_executable, e)) - sys.exit(100) - else: - raise e - - proc_stdout = proc_stdout.strip().decode("utf-8") - # normalize paths using realpath to ensure that a virtualenv correctly identifies itself even - # when addressed over a symlink - proc_stdout = os.path.normcase(os.path.realpath(proc_stdout)) - norm_home_dir = os.path.normcase(os.path.realpath(home_dir)) - if hasattr(norm_home_dir, "decode"): - norm_home_dir = norm_home_dir.decode(sys.getfilesystemencoding()) - if proc_stdout != norm_home_dir: - logger.fatal("ERROR: The executable %s is not functioning", py_executable) - logger.fatal("ERROR: It thinks sys.prefix is {!r} (should be {!r})".format(proc_stdout, norm_home_dir)) - logger.fatal("ERROR: virtualenv is not compatible with this system or executable") - if IS_WIN: - logger.fatal( - "Note: some Windows users have reported this error when they " - 'installed Python for "Only this user" or have multiple ' - "versions of Python installed. Copying the appropriate " - "PythonXX.dll to the virtualenv Scripts/ directory may fix " - "this problem." - ) - sys.exit(100) - else: - logger.info("Got sys.prefix result: %r", proc_stdout) - - pydistutils = os.path.expanduser("~/.pydistutils.cfg") - if os.path.exists(pydistutils): - logger.notify("Please make sure you remove any previous custom paths from " "your %s file.", pydistutils) - # FIXME: really this should be calculated earlier - - fix_local_scheme(home_dir, symlink) - - if site_packages: - if os.path.exists(site_packages_filename): - logger.info("Deleting %s", site_packages_filename) - os.unlink(site_packages_filename) - - return py_executable - - -def install_activate(home_dir, bin_dir, prompt=None): - if IS_WIN: - files = {"activate.bat": ACTIVATE_BAT, "deactivate.bat": DEACTIVATE_BAT, "activate.ps1": ACTIVATE_PS} - - # MSYS needs paths of the form /c/path/to/file - drive, tail = os.path.splitdrive(home_dir.replace(os.sep, "/")) - home_dir_msys = (drive and "/{}{}" or "{}{}").format(drive[:1], tail) - - # Run-time conditional enables (basic) Cygwin compatibility - home_dir_sh = """$(if [ "$OSTYPE" "==" "cygwin" ]; then cygpath -u '{}'; else echo '{}'; fi;)""".format( - home_dir, home_dir_msys - ) - files["activate"] = ACTIVATE_SH.replace("__VIRTUAL_ENV__", home_dir_sh) - - else: - files = { - "activate": ACTIVATE_SH, - "activate.fish": ACTIVATE_FISH, - "activate.csh": ACTIVATE_CSH, - "activate.ps1": ACTIVATE_PS, - } - files["activate_this.py"] = ACTIVATE_THIS - - if sys.version_info >= (3, 4): - # Add xonsh support - files["activate.xsh"] = ACTIVATE_XSH - - install_files(home_dir, bin_dir, prompt, files) - - -def install_files(home_dir, bin_dir, prompt, files): - if hasattr(home_dir, "decode"): - home_dir = home_dir.decode(sys.getfilesystemencoding()) - virtualenv_name = os.path.basename(home_dir) - for name, content in files.items(): - content = content.replace("__VIRTUAL_PROMPT__", prompt or "") - content = content.replace("__VIRTUAL_WINPROMPT__", prompt or "({}) ".format(virtualenv_name)) - content = content.replace("__VIRTUAL_ENV__", home_dir) - content = content.replace("__VIRTUAL_NAME__", virtualenv_name) - content = content.replace("__BIN_NAME__", os.path.basename(bin_dir)) - content = content.replace("__PATH_SEP__", os.pathsep) - writefile(os.path.join(bin_dir, name), content) - - -def install_python_config(home_dir, bin_dir, prompt=None): - if IS_WIN: - files = {} - else: - files = {"python-config": PYTHON_CONFIG} - install_files(home_dir, bin_dir, prompt, files) - for name, _ in files.items(): - make_exe(os.path.join(bin_dir, name)) - - -def install_distutils(home_dir): - distutils_path = change_prefix(distutils.__path__[0], home_dir) - mkdir(distutils_path) - # FIXME: maybe this prefix setting should only be put in place if - # there's a local distutils.cfg with a prefix setting? - # FIXME: this is breaking things, removing for now: - # distutils_cfg = DISTUTILS_CFG + "\n[install]\nprefix=%s\n" home_dir - writefile(os.path.join(distutils_path, "__init__.py"), DISTUTILS_INIT) - writefile(os.path.join(distutils_path, "distutils.cfg"), DISTUTILS_CFG, overwrite=False) - - -def fix_local_scheme(home_dir, symlink=True): - """ - Platforms that use the "posix_local" install scheme (like Ubuntu with - Python 2.7) need to be given an additional "local" location, sigh. - """ - try: - import sysconfig - except ImportError: - pass - else: - # noinspection PyProtectedMember - if sysconfig._get_default_scheme() == "posix_local": - local_path = os.path.join(home_dir, "local") - if not os.path.exists(local_path): - os.mkdir(local_path) - for subdir_name in os.listdir(home_dir): - if subdir_name == "local": - continue - copyfile( - os.path.abspath(os.path.join(home_dir, subdir_name)), - os.path.join(local_path, subdir_name), - symlink, - ) - - -def fix_lib64(lib_dir, symlink=True): - """ - Some platforms (particularly Gentoo on x64) put things in lib64/pythonX.Y - instead of lib/pythonX.Y. If this is such a platform we'll just create a - symlink so lib64 points to lib - """ - # PyPy's library path scheme is not affected by this. - # Return early or we will die on the following assert. - if IS_PYPY: - logger.debug("PyPy detected, skipping lib64 symlinking") - return - # Check we have a lib64 library path - if not [p for p in distutils.sysconfig.get_config_vars().values() if isinstance(p, basestring) and "lib64" in p]: - return - - logger.debug("This system uses lib64; symlinking lib64 to lib") - - assert os.path.basename(lib_dir) == "python{}".format(sys.version[:3]), "Unexpected python lib dir: {!r}".format( - lib_dir - ) - lib_parent = os.path.dirname(lib_dir) - top_level = os.path.dirname(lib_parent) - lib_dir = os.path.join(top_level, "lib") - lib64_link = os.path.join(top_level, "lib64") - assert os.path.basename(lib_parent) == "lib", "Unexpected parent dir: {!r}".format(lib_parent) - if os.path.lexists(lib64_link): - return - if symlink: - os.symlink("lib", lib64_link) - else: - copyfile(lib_dir, lib64_link, symlink=False) - - -def resolve_interpreter(exe): - """ - If the executable given isn't an absolute path, search $PATH for the interpreter - """ - # If the "executable" is a version number, get the installed executable for - # that version - orig_exe = exe - python_versions = get_installed_pythons() - if exe in python_versions: - exe = python_versions[exe] - - if os.path.abspath(exe) != exe: - exe = distutils.spawn.find_executable(exe) or exe - if not os.path.exists(exe): - logger.fatal("The path {} (from --python={}) does not exist".format(exe, orig_exe)) - raise SystemExit(3) - if not is_executable(exe): - logger.fatal("The path {} (from --python={}) is not an executable file".format(exe, orig_exe)) - raise SystemExit(3) - return exe - - -def is_executable(exe): - """Checks a file is executable""" - return os.path.isfile(exe) and os.access(exe, os.X_OK) - - -# Relocating the environment: -def make_environment_relocatable(home_dir): - """ - Makes the already-existing environment use relative paths, and takes out - the #!-based environment selection in scripts. - """ - home_dir, lib_dir, inc_dir, bin_dir = path_locations(home_dir) - activate_this = os.path.join(bin_dir, "activate_this.py") - if not os.path.exists(activate_this): - logger.fatal( - "The environment doesn't have a file %s -- please re-run virtualenv " "on this environment to update it", - activate_this, - ) - fixup_scripts(home_dir, bin_dir) - fixup_pth_and_egg_link(home_dir) - # FIXME: need to fix up distutils.cfg - - -OK_ABS_SCRIPTS = [ - "python", - "python{}".format(sys.version[:3]), - "activate", - "activate.bat", - "activate_this.py", - "activate.fish", - "activate.csh", - "activate.xsh", -] - - -def fixup_scripts(_, bin_dir): - if IS_WIN: - new_shebang_args = ("{} /c".format(os.path.normcase(os.environ.get("COMSPEC", "cmd.exe"))), "", ".exe") - else: - new_shebang_args = ("/usr/bin/env", sys.version[:3], "") - - # This is what we expect at the top of scripts: - shebang = "#!{}".format( - os.path.normcase(os.path.join(os.path.abspath(bin_dir), "python{}".format(new_shebang_args[2]))) - ) - # This is what we'll put: - new_shebang = "#!{} python{}{}".format(*new_shebang_args) - - for filename in os.listdir(bin_dir): - filename = os.path.join(bin_dir, filename) - if not os.path.isfile(filename): - # ignore child directories, e.g. .svn ones. - continue - with open(filename, "rb") as f: - try: - lines = f.read().decode("utf-8").splitlines() - except UnicodeDecodeError: - # This is probably a binary program instead - # of a script, so just ignore it. - continue - if not lines: - logger.warn("Script %s is an empty file", filename) - continue - - old_shebang = lines[0].strip() - old_shebang = old_shebang[0:2] + os.path.normcase(old_shebang[2:]) - - if not old_shebang.startswith(shebang): - if os.path.basename(filename) in OK_ABS_SCRIPTS: - logger.debug("Cannot make script %s relative", filename) - elif lines[0].strip() == new_shebang: - logger.info("Script %s has already been made relative", filename) - else: - logger.warn( - "Script %s cannot be made relative (it's not a normal script that starts with %s)", - filename, - shebang, - ) - continue - logger.notify("Making script %s relative", filename) - script = relative_script([new_shebang] + lines[1:]) - with open(filename, "wb") as f: - f.write("\n".join(script).encode("utf-8")) - - -def relative_script(lines): - """Return a script that'll work in a relocatable environment.""" - activate = ( - "import os; " - "activate_this=os.path.join(os.path.dirname(os.path.realpath(__file__)), 'activate_this.py'); " - "exec(compile(open(activate_this).read(), activate_this, 'exec'), { '__file__': activate_this}); " - "del os, activate_this" - ) - # Find the last future statement in the script. If we insert the activation - # line before a future statement, Python will raise a SyntaxError. - activate_at = None - for idx, line in reversed(list(enumerate(lines))): - if line.split()[:3] == ["from", "__future__", "import"]: - activate_at = idx + 1 - break - if activate_at is None: - # Activate after the shebang. - activate_at = 1 - return lines[:activate_at] + ["", activate, ""] + lines[activate_at:] - - -def fixup_pth_and_egg_link(home_dir, sys_path=None): - """Makes .pth and .egg-link files use relative paths""" - home_dir = os.path.normcase(os.path.abspath(home_dir)) - if sys_path is None: - sys_path = sys.path - for a_path in sys_path: - if not a_path: - a_path = "." - if not os.path.isdir(a_path): - continue - a_path = os.path.normcase(os.path.abspath(a_path)) - if not a_path.startswith(home_dir): - logger.debug("Skipping system (non-environment) directory %s", a_path) - continue - for filename in os.listdir(a_path): - filename = os.path.join(a_path, filename) - if filename.endswith(".pth"): - if not os.access(filename, os.W_OK): - logger.warn("Cannot write .pth file %s, skipping", filename) - else: - fixup_pth_file(filename) - if filename.endswith(".egg-link"): - if not os.access(filename, os.W_OK): - logger.warn("Cannot write .egg-link file %s, skipping", filename) - else: - fixup_egg_link(filename) - - -def fixup_pth_file(filename): - lines = [] - with open(filename) as f: - prev_lines = f.readlines() - for line in prev_lines: - line = line.strip() - if not line or line.startswith("#") or line.startswith("import ") or os.path.abspath(line) != line: - lines.append(line) - else: - new_value = make_relative_path(filename, line) - if line != new_value: - logger.debug("Rewriting path {} as {} (in {})".format(line, new_value, filename)) - lines.append(new_value) - if lines == prev_lines: - logger.info("No changes to .pth file %s", filename) - return - logger.notify("Making paths in .pth file %s relative", filename) - with open(filename, "w") as f: - f.write("\n".join(lines) + "\n") - - -def fixup_egg_link(filename): - with open(filename) as f: - link = f.readline().strip() - if os.path.abspath(link) != link: - logger.debug("Link in %s already relative", filename) - return - new_link = make_relative_path(filename, link) - logger.notify("Rewriting link {} in {} as {}".format(link, filename, new_link)) - with open(filename, "w") as f: - f.write(new_link) - - -def make_relative_path(source, dest, dest_is_directory=True): - """ - Make a filename relative, where the filename is dest, and it is - being referred to from the filename source. - - >>> make_relative_path('/usr/share/something/a-file.pth', - ... '/usr/share/another-place/src/Directory') - '../another-place/src/Directory' - >>> make_relative_path('/usr/share/something/a-file.pth', - ... '/home/user/src/Directory') - '../../../home/user/src/Directory' - >>> make_relative_path('/usr/share/a-file.pth', '/usr/share/') - './' - """ - source = os.path.dirname(source) - if not dest_is_directory: - dest_filename = os.path.basename(dest) - dest = os.path.dirname(dest) - else: - dest_filename = None - dest = os.path.normpath(os.path.abspath(dest)) - source = os.path.normpath(os.path.abspath(source)) - dest_parts = dest.strip(os.path.sep).split(os.path.sep) - source_parts = source.strip(os.path.sep).split(os.path.sep) - while dest_parts and source_parts and dest_parts[0] == source_parts[0]: - dest_parts.pop(0) - source_parts.pop(0) - full_parts = [".."] * len(source_parts) + dest_parts - if not dest_is_directory and dest_filename is not None: - full_parts.append(dest_filename) - if not full_parts: - # Special case for the current directory (otherwise it'd be '') - return "./" - return os.path.sep.join(full_parts) - - -FILE_PATH = __file__ if os.path.isabs(__file__) else os.path.join(os.getcwd(), __file__) - - -# Bootstrap script creation: -def create_bootstrap_script(extra_text, python_version=""): - """ - Creates a bootstrap script, which is like this script but with - extend_parser, adjust_options, and after_install hooks. - - This returns a string that (written to disk of course) can be used - as a bootstrap script with your own customizations. The script - will be the standard virtualenv.py script, with your extra text - added (your extra text should be Python code). - - If you include these functions, they will be called: - - ``extend_parser(optparse_parser)``: - You can add or remove options from the parser here. - - ``adjust_options(options, args)``: - You can change options here, or change the args (if you accept - different kinds of arguments, be sure you modify ``args`` so it is - only ``[DEST_DIR]``). - - ``after_install(options, home_dir)``: - - After everything is installed, this function is called. This - is probably the function you are most likely to use. An - example would be:: - - def after_install(options, home_dir): - subprocess.call([join(home_dir, 'bin', 'easy_install'), - 'MyPackage']) - subprocess.call([join(home_dir, 'bin', 'my-package-script'), - 'setup', home_dir]) - - This example immediately installs a package, and runs a setup - script from that package. - - If you provide something like ``python_version='2.5'`` then the - script will start with ``#!/usr/bin/env python2.5`` instead of - ``#!/usr/bin/env python``. You can use this when the script must - be run with a particular Python version. - """ - filename = FILE_PATH - if filename.endswith(".pyc"): - filename = filename[:-1] - with codecs.open(filename, "r", encoding="utf-8") as f: - content = f.read() - py_exe = "python{}".format(python_version) - content = "#!/usr/bin/env {}\n# WARNING: This file is generated\n{}".format(py_exe, content) - # we build the string as two, to avoid replacing here, but yes further done - return content.replace("# EXTEND - " "bootstrap here", extra_text) - - -# EXTEND - bootstrap here - - -def convert(s): - b = base64.b64decode(s.encode("ascii")) - return zlib.decompress(b).decode("utf-8") - - -# file site.py -SITE_PY = convert( - """ -eJy1Pf1z2zaWv+uvQOnJWEplOnHaXtepe+MkTus5N/HG6WxuU5+WEiGJNUWyBGlZ2+n+7fc+ABAg -Kdm+7Wk6rkQCDw8P7xsPSBAEp0Uhs1is8rhOpVAyKmdLUUTVUol5XopqmZTxQRGV1Qaezm6ihVSi -yoXaqBBbhYPB03/zM3gqPi4TZVCAb1Fd5auoSmZRmm5EsiryspKxiOsyyRYiyZIqidLkn9Aiz0Lx -9N/HYHCeCZh5mshS3MpSAVwl8rm43FTLPBPDusA5Pw+/jl6MxkLNyqSooEGpcQaKLKNqkEkZA5rQ -slZAyqSSB6qQs2SezGzDdV6nsSjSaCbFP/7BU6Om+/sDla/keilLKTJABmBKgFUgHvA1KcUsj2Uo -xCs5i3AAft4Qa8DQxrhmCsmY5SLNswXMKZMzqVRUbsRwWlcEiFAWcQ44JYBBlaTpYJ2XN2oES0rr -sYZHImL28CfD7AHzxPG7nAM4vs8GP2fJ3ZhhA/cguGrJbFPKeXInIgQLP+WdnE30s2EyF3EynwMN -smqETQaMgBJpMj0saDm+0yv0/SFhZbkygjEkosyN+SX1CAfnlYhSBWxbF0gjRZi/kdMkyoAa2S0M -BxCBpIO+ceJEVXYcmp3IAUCJ61iBlKyUGK6iJANm/SmaEdp/S7I4X6sRUQBWS4lfa1W58x/2EABa -OwQYD3CxzGrWWZrcyHQzAgQ+AvalVHVaoUDESSlnVV4mUhEAQG0j5B0gPRZRKTUJmTON3I6J/kST -JMOFRQFDgceXSJJ5sqhLkjAxT4BzgSvevv8g3py9Oj99p3nMAGOZXawAZ4BCC+3gBAOIw1qVh2kO -Ah0OLvB/IopjFLIFjg94NQ0O713pwRDmXoTtPs6CA9n14uphYI4VKBMaa0D9focuY7UE+vxxz3oP -BqfbqEIT52/rZQ4ymUUrKZYR8xdyxuA7Def7sKiWL4EbFMKpgFQKFwcRTBAekMSl2TDPpCiAxdIk -k6MBUGhKbf1VBFZ4l2cHtNYtTgAI5SCDl86zEY2YSZhoF9ZL1Bem8YZmppsM7Dqv8pIUB/B/NiNd -lEbZDeGoiKH421QukixDhJAXBvt7+zSwukmAE+NQXFAr0gumkdhn7cUtUSRq4CVkOuBJeRetilSO -WXxRt+5WIzSYrIRZ65Q5DlpWpF5p1Zqp9vLeUfipxXWEZrUsJQCvp57QzfN8LKagswmbIlqxeFXr -nDhn0CNP1Al5glpCX/wOFD1Vql5J+xJ5BTQLMdRgnqdpvgaSHQ8GQuxhI2OUfeaEt/AO/gJc/JvK -arYcDJyRLGANCpHfBgqBgEmQmeZqjYTHbZqV20omyVhT5GUsSxrqYcQ+ZMQf2BjnOniXV9qo8XRx -lfNVUqFKmmqTmbDFy/Yr1o8ved4wDbDcimhmmjZ0WuH00mIZTaVxSaZyjpKgF+mlXXYYc9AzJtni -SqzYysA7IItM2IL0KxZUOvNKkhMAMFj4oiwp6pQaKWQwEcFAq4LgryI06bl2loC92SwPUCGx+Z6B -/QHc/glitF4mQJ8ZQAANg1oKlm+aVCU6CI0+GvhG3/Tn8YFTz+faNvGQ8yhJtZWPssE5PTwrSxLf -mSyw11gTQ8EMswpdu0UGdEQxD4JgoB0YkSvzDRhpMKjKzTHwgjADTSbTOkHLN5mgrdc/1ICHEc7I -XjfbzsIDFWE6vQMyOV3mZb7C13ZyV6BNYCzsMdgTl6RmJHvIHiu+RPq5Or0wTdF4O/wsUBENLj+c -vT3/dHYlTsTnRqeN2wrtGsY8yyLgazIJwFWtYRtlBi1R8yWo/MRbsO/EFWDQqDfJr4yqGpgXUP9Y -1vQapjHzXg7O3p2+ujib/Hx19mFydf7xDBAEQyMHezRltI81eIwqBOkAtoxVqA3soNODHrw6vbIP -BpNETb75CvgNngwXsgIuLocw47EIVtGdAg4NxtR0hDh2GoAJD0Yj8b04Ek+fihdHBK/YFBsAB5bX -aYwPJ9qoT5JsngejwWAQyzlIy41EmRo+JVd2xKsORAQYuTatv+ZJZt73vI6miiDAU35dSqBchq3G -thFw92oWKcmteGzoOJmgjE8mQz0wMD8xGDgoLLb7wjRBGS8T8ANpoVDmpypP8ScOgFJD/IqhCGoV -ZBwdaoS3UVpLZcagZoC/oSeYqMAMYuhtG4ITOUdBxadNd/yg2UiyWtqHq9Ci2iXO3Ey6lKv8VsZg -vpGezrzFB3oDsV2RgnKFeYFmIIvCMmg8jwgjGdYusHhofYBdVwTFUMQQY4/DSZkp4GQOzYi3ddzI -yrQo89sEzdV0o1+CtgVpQ51rTKOGlqNf75EdlSowGPhRGZJqLfdBnsqaXR7CG0GiwokbyQwJ3AWK -+jV9vcnydTbhWOoEpXY4souJvKaXExs0a7An3oIaAyRzCA0aojEUcBoFstsBIA/Th+kCZck7BUBg -KhQ5/A4sEyzQFDnuwGERxuilIH4uJVqcWzMEOeuGGA4kehvaByQF8AchweSsvFlp0VyGZtE0g4Ed -kvhsdxFyEOMDaFExBL071NC4kaHf5+NrwOLClVOn3wB1+qdPn5hv1JJCdMRsirNGgzQndRsWG1DN -CXi+xqhzwE98AIF7BmBqpXlTHFyJvGCDDgvKmQSwmlfgPi6rqjg+PFyv16EOUPNycajmh19/+803 -3z5jNRHHxEAwH0dcdLYmPKR36BSF35kg9HuzdC2GTDKfHQnWUJJhJ28G8fuhTuJcHB+MrEpBNm4U -Pf41xhBUyMQMOmA6A3WDBqXf/zj4/Th88UcQYpOoGro9hiO2b1ox80IBK/i6G3pUOVgUsH+zvEaV -3zCEEl/CcBDqxXJaLwKLgae5zQ+YMIrr0LLCwfNrxMBnEMNeyiisCWoLYg80HM4KfGD2icjMak2B -VEbl3YnDNv3azNA4frjcW6fFER4zxUQhk6Bk+E0eLIi2cVeC8KN9pI+bwvWRzMezCMYIGlsXx9pF -GaIAETK4GmNX/BzuRt8JpGrNOho0jasjQVODuzFlcmtw5HnvI8h9QZ4mxY28tLoFEKqVhDKhplkG -aOGO07F7bV6wL8C8SPRintMTCc7WcefdM17bOk0pN9DiUo8qDNhbabTYOXDm0AAA0SgDbmj83fet -ReE16IGVczYBGWzeYSZ8EzKVkEjDYC/oYaeO+d/Wm0ncBwLXaMhZjQdBJ5RPeIRSgTQVQ7/nNha3 -tO0OttPyEE+ZVSKRVluEq609eoVqp4GaJxmqYGeRwlmag8No1SLxUfPedxrIocbHvUZNS6CmQ0MP -p9UJ+X2eAO7rdpiJWdQYs7ohPaK0ShRZOaTTEv6Af0GhMuU1gJgEzYJ5qJT5M/sTZM7OV3/Zwh+W -0sgRbtNt/kjHZmg46AYhAXVIDwKQAfG6ckjeIyuCFJjLWRpPruG13CrYBCNUIGKO8SAFA1jTy6Yb -Tg2ehJhGJ+FEyHeVkoX4EoIkMEct1n6Y1v5zGdREskOnBfkJOgQ+ccNjJzQ+aYXKPi/7QTJl3Isc -mHcKXo+bLnY53PCrjdzBYfd1tkUKlDEF88HIRfXaUMZNSX5x4rRoqGUGMbzkDeRtDJiRRgO73ho0 -rLgB4y16eyz9mOapfTHsOwxydSRXdwGalkTNctWntTUjdKlh8btIpgjCI3kwuvYgyTRB14rj9f/L -IPdD96Z3An5iHJXrJAtIW2mKnfiL0cHDEs8zaYdXFEQdwkQxU3b4tgSZoC2zQ6AYSX9RQOyutLPf -hdueY28D/HiTD+yIDDfw/OfPxy+uu4QZPwy0pevZXVVGCtcv5WVkwdgG6Lo7N9LESIQo2+jdMb17 -ikFEmSsIMsX7q08CKcaJw3W0uZdE/ej6SIIau5cgLYyJVQA2KkFkksNgNzf24vln4dZP453QHeX1 -uP670ALYB/EmS/Mo7gN63XkCHb75atKTNnPR/earR5KjTxxb7pgdeOQ5a6WMUjLtznu06qDldq+g -7ViMWLNSEKbXv0uILq9oNG1qs78FfgxUzJhKMN/Pxg0Zu1PHj+/jbIWnLcgOYJ0Y0nz2aFEg5M6n -v0LcqnTK6zZKUkobAzEODlAJmpibkwn9wutB2infDgn6giL1+RlyBwf5o+50tJ90ahKkPbGp+RSR -6qKyp/eImy0Ob9/X3d7arhx6Dbdvsw8foBG8kfsmqy2b6fXsmtTWUY/aegR2fwJe25nzT8Dh2YNQ -eNRADE0rzlHLJvRNxbUCBtgDVPMWF2WHg9Idmjdb5sbNYKFT4ilK51Oxpg1XSu2Bl5IBlJgdkB44 -yId62+51XZa8+UZCXsjyAPeTxgJrTYynQSUsXTCH72SFmNhmM0qVOpUJeR/jBjr5aWcSND5rP+8s -c5MqkdltUkJfUCnD4Mf3P5312Ac9DHZ6uJ701hC7sguL08JVfLir5eNjdgRNIk+P2sWrk78zMWD/ -FO6J470tKN5mo+hqtpSzm4mkvUNcZuzrJDNf42tExW4p+hUgKppTGQtMZZbWSAN2o7D+aF5nM0pv -VxJMsi4WxOIB2hHktM08jRZiSJ1jzBjo1aSkwm1UaiejKHMsTxN1Eh8ukljI3+ooxZBMzueAC24+ -6FchD0+JA/GGNzW5bEnJWV0m1QZoEKlc793Q/qfTcLrhiQ49JDlLzxTEHdFjcYXTxvdMuNiQywR2 -fqoaJwl8QB2c3cqQnsP7LJ/gqBMkLzAUITVqR/X8eNAeIde5cJi/HqH9RtIrd4OI1twlKmoZj5Ru -lJiTuUYowxFGqfybfvqc6DLXFiwX27Fc7MZy0cZy0YvlwsdysRtLVyZwYZuMgxGFvqxDOx3du+Pu -Jgx4nLNotuR2WP6FZV4AURQmBDJCxVWQXlqC92cICOk9Z8eQHjY76AnXlZU55y41SGR/3H3Q4Zap -X3U603687sxTMR7PtioCv+9hGFK1yJS6s8DFURWFnmAs0nwKcmvRHTcAxqJdUMCZrux2MuWsXFvV -X/73xx/fv8PmCMpuUFM3XERU2ziV4dOoXKiuODUBQwH8SC39jX3qpgHu9eZF9jkvsj8W+5wX2dfD -7PGfNznWESHniDXtMeeiABtKhR22mVv+sL/feq7rJPRzZnPeJQC3IKscl2ALlU4vL9+cfjwNqFIi -+Ffgiowhri8fLj6mhW3Q9YDc5pbk2KcxkY39c+fkEdthCdOr4+m1HhxtCwx9HP9f5wkEBSRCnXh7 -3DQfnwWwur9Te+OGuOwr2HcO4zsegiN3fRnb9gD+FhF6yBNAX3swQzOVJpZpqVxr3HbQwwH6uIij -7T37dLBD93hNDj1ss+0+UzcxrWT16uyH83cX568uTz/+6HhN6P28vzo8Emc/fRK0I44qn92ICHeB -Kyy+AFXsng4QcQ7/1RhOx3XFiS/o9ebiQiemV1gfjgWDqKVDeM6VGxYaZyY4s2Yf6pILxCjVPrlT -iE8FClSojy76iovAVa6LCqm+f4r+Xa29fX3AwhzEoC28EBgeGrukYBBcVQOvqPSzMoFIyTse+nBC -D1Laqtk98JQyL52NUSfdb1LLXn6KOsOTprPWjJ8DF9fgOlRFmoCufBlYAdDdsBSg4Rv90O7lMV59 -SsfpDiPrhjzrrVhggcTLgOem+48aRvutBgwbBnsD884kbYRT1SLW84h9bLTPG97yDr7apddroGDB -cF+jwkU0TJfA7COI58QyAZ8beHIJ5gpda4DQWgk/I3rsWB+Z4z508HoVH/w10ATxW//yS0/zqkwP -/i4KCBwEV00EPcR0G7+BWCGUoTh7/3YUMHKzNAL38K811tSCCaeskiPsVKrBe4WToZLpXO+k++oA -X2jDSq8Hrf6lLErdv9+dDFAEfv9jSHb29z8MAW19ix1gjPMZteFj0bDFDw/fuNuu5rMnrpYyTXWl -6PmbizPwubCOGeWItx7OYEwO1HHjUBcd8eGgFijcVoTXJTJzia4f7SrHodesNx+Igke9vY1ou1qU -c+v26iTYyihRLtpDnLYmjKnEDZGjYUnM6gb4W0upbYN0dtsQ3VF0mDMmlyWV7vmsASxNTyMOJiDK -wDJpk93kvbMkq0ypVprMQJ2C5gW9OgZZQeriWSBiwDzj/GJeKnOEAB4WmzJZLCtML0PnkMqXsflP -p58uzt9RRe/Ri8Zd7eHRMbnQY941P8GqKEwUwBe30gn5ajJxWbf1CmGgEoL/tV/xdvwJD9Dpxykt -L7+tX/EZjhMnhuIZgJ6qi7aQoOPsdOuTnkYYGFcbQeLHLXpqMPPBUBYMS8b1/rY7vy4r2pYti0Ku -h3n5iOT4vLA1OLqzU4PT/ugpzgvcUoiH/Y3gbZ9smc8Uut503myr9nE/HSnE02SAUbe1P4Yph+k0 -1dNxeG0bs1A4qC3tL25Q4LSbZSjJQMOh23nk8li/JtbNmQG9+tEOMPGdRtcIYr8+BxS1n+FhYond -CRBMRyyCE0/UkEsGpK6ar8GTeqJIfwTiiRgOHcEdj8RTceTN0rEH989SKy+wkT+CJtTlh1TwnZfA -ivDlN3Yg+RUhhor0WDThIE4vy21lDH7WS/Qun/uz7BUCyv+h3JVRtpBDhjU2ML/0yb0lfUnq1iP2 -5+S6z7SIc3BS77bweFcy+nP2BrUWK3Ta3chNWx/55MEGvWXwOwnmgy+jNSj/oq6GvJJbNvl6z6Fs -h3o/RKy3gqZ6/3MYoAf9W19phw+4lxoaFnp6v23ZcuK1sR6ttZCtMw779oV2ZWclhGiV2kcknZM9 -xim1fkBjck8aox/YpzpTb39bX79bbdKA5KF9gPyMtwAuN3y0KqEq3+awAL2J5a1MwS6AtTIF3TgO -lXL35ip2jeuRtEHCfxwEv2iXP8puKKx7/bfzsXj97gP8fSXfg43CAzlj8XdAS7zOS4jf+EgZnfXF -2vCKA7O8Vnhqh6BR9pyPRaM/ZIo2eHaYqddF6361utVAAuvz0AnGs/iAYoMz04AOXDam2JRjw29z -bqTl8BlnrI86gX7pjBKYgnrVragv1PxQ9zh0unwOLs5fn727OgurO2Qc8zO4dtq4Aai/VyRJ7eKj -EvdrxsI+mdX4RENxfNQfZVr0uKg6zDPF+RjmiX2IAQob2vHJ78g68lGJsbooNnE+C7GlGPKBO1Gt -wWcdORHdvUbVs2gIazjSWz2N44yPgT4igOfURE+BGtL40RQPovDjMNhi5MaC0q3wv6c369jN1uoT -BTShQRu1ZppDv3+jZpZMWQ3R8hChdmIJb49epUmkVtOZewDpfSb0WX3QIJR+l/OoTishMwhZKJKm -Q9OgRt0zQywmvNJsK+ggDeVC0nW0UU75RKREgKPStqfEfQLKpUGg+1N0w8oWDzOJmg/pAXRClKKT -3Omq6tmSxZgDDq3hOhvK6yR74VYGaCLzoByAzlRDbZgoumWM0kJWmgD8YDj6/LzZwqZ068yripsV -uvgNvj19+jQQ/3m/R8EYhGme34CrAxB7PYELer3FGuo52UXqKUgzr0JgxdlSfoYH15QNts/rjNJ+ -O7rSQkj7fwOD16RhQ9OhZew4JVbyzie34A0UbUd+zhK6igMTNxJVrb7RBJM6RpKIF0EJ7EdqliT7 -HP7DOmzyGg/5YBJPM4q8A15PEMwY3+KODUewS/TbqOTOso1FB+ZCgAOuh+Hh6OQgHXABRCeXG43n -5DxLqqYy/Zm7naePp1b2ig3NUCJao0yYibSo4ZwY85i0WU34vYs5vVggn31+3qrNcubJr+9DHpga -hCyfzw2q8NAs0yyX5cyYU1yzZJZUDhjTDuFwZ7pahLLb4aAHpQCUOdmA2AqzffuFXRkX0/e0tXlg -RtIVIZW9roXTLlHWqooKw2Z8SvBYQlrONV+ojP0d5Y21K+ANJr7QXI/Fyd6xcveMVJ3p4+JcU9Cc -IQc4dEWJVY6WIz1F4dyLY+Ez3245440fin4ZudukrOoonehTyRN04SZ2s1cjas/W7Dw3Zr2UMVrB -ZHGgK4PRdRg1h8iwgtDUekP8r8P/0D2v4h/tKHJ0AI/YRwJPAkkTT8xpS5Of+WwZ1p5f06VfDlsg -qC/NiYhel9evHqOz0OJ7MXwxhsC0dZqr2ODtIDDAkxhD2XY3GN6Jkg1e7YPW4vsTMXw+Fl/vgB5u -GeD4yB2hHd61IGwF8uLa4Qs+i9Cp4ncXjbcXiGxYzX9/U1NENtYoja7b1O5S5Tsiyl9aROGjQboW -8vnjxg5WILTzRMYHTxQSQuPS6MXtrEXD4sFYXWdGxorPhIp9rknb5+OkU4gE1o0SctSR8VDtKOaw -qVYdfB4P5Zztlj1MbMonWjVg9gIirA/T+LWIBQAmfRtGfN9IgO81LVy/aOs+oi4otmB74mN9dIu3 -hZqGA18EdjlgD+VA3Oe6n/lwy9DUFfoC8ghO37k57fDyn88/XKC8bc+vg+g95eOeB9BbJd6zqrsr -tT15tOXZzZqbT3/uyeOWvq7/ltQ4UBr5AYfRXGVGrt4x1obi1i9dGaFrRFd4WRhtwqJJSqXruHAA -y7fCoCdBMTtVGsFqgC9Tz3HP9JbuPjtVVBF1eXYpvjp6PjbH/RiQnsCL8MWX+oow6maONPt+7lgH -TvBS9zsK/8MBllR+b/uqE1PQtE5aV4X4g3F+aBROLB0aI3N/HXmzZpHAO7UaYrJ7a0/3vwVvYwp+ -hrkcLwePCaLBsA9dV200IrxNvz1ein1tiIPuEpZtGnCX9ns0K+te/WYA42/k3GXEb9CVqotjm6Qy -hgF354ZuqR7xxE3vORQ2Cmgyq5tg1/S5/6656xYDO3Oz796Zu38MzKWB7rvdFPbRgHMDmhAPKiBv -lLa1CN68PBq5en2saz+6Nc1uK5znFrvQ6WdqSTADTdyokTXMuYpm7nesLzrg+zV7aoCcQ1wuQ7Sm -29H1bU5/iJnn+gx8bo/7UtFNrDlPBxp0c9KEKx8ncrFQkwiv7ZpQfEolR52wwwQ8b+nSJRmpjQlT -8CoRAGHYSNdTujXTwBGg5Pj6UC7ZcS6LEDQ0lYI6taUqiVmX65gIwIWcWqT+JrnNxaFBKjFHrOqy -KCGQDPQlk1xL0lew2gA1aa9VpG4M6qbHWN85SBqRCpjM0T9OaLYKUYAIbH07On0yse+Aa541R5+T -seUGmdUrWUZVc+WHv12dQOTTjECnQnF1nZxWI44tNnERSyxzOEhhesl+/xIiMj8c3XHWeWT13MP5 -iXroXOljjlE7ZmTkRPNYJXdPNO8XDz8ymvfgPyaa5yr8MsUz9Sne0OQgp1M3Jr+EYTcV9ftX6sF/ -3XoQIMx0o2m4wHoaStlEek2d9ss8v5lM2Iafz41jxONo/aaro81VxWNqhP04LaN0MvBjNMWNNGZ9 -JfSEEEPcXkNeNKiIfx3qnYwJiCrVeUMH85WvCEQoZID16DnIWJnEMd5KxSrES6DgVrG3Bhp5tkqU -+YfZc+H11cfTDx9/vnQG8vYUDJYTQ4dhN6cPEniXVI3G7t5gw+0MiN53zZq33bVepmkEohl3T3yA -EUypj395NJ4EQT4Zca3lKtqYe59klteLJW/yw1wcaP6iGYONrfDK1jkYwcrUCWU0OS4RsIRkHHnO -k5jSm0bJmcek6eAV3cwW+H6L11VfLIl7tjQmejq4dUjbBW7LNon4DW5hKTmBnpNpAqYtwL/if86B -7hND+GB3VcQWQODuHAsHxHYusCDwC1dJIa2c1K5e8PdXvY75aWbe8FYjH0kj/2mFR9HpEk7FgRRf -V60qwTcpi/ZW9x4fkuBi1hJduCwXIW21lzNmkSGMEyKJzQMtaRTo6pKxBhzeBv3+SnwCW6BXZmQ6 -YLaeLl02Z/PwRs2MR59Rgf26jdw6YheAb83UUPTVmn6Ewcqzh2swhT6Z8RFAo1gmYL4XYPZG6Ew+ -a9P3HGuDrOJZ4+XaeRSjijM6FzBqqaqwBQPLkxc1HlCkS9PMXeQYnd7mScx10SihZhi8BBIUSAsM -bZ25F/VRbeDayBnpY0QQbw2L4o21ATJuAaqWJUl3W9uhwh5jVU+bCHp/f1ovlLu/nChVy6+//err -vZVaPP/Lt9+8+MbraKaz5Woy52xK8C9M/AQtOraKzHr30nz5MUtKIqR/9G6u9UpTl3Xwgwp/XaLr -oQG2z1xtxc3DzwOxA0EHyeFbaPUur96ik0j4jsWlLOn+oTyjB1vqWfZaR6n4onvUlRQEkLXj34gW -uhDbwOjKgt6lhyX/y/Pejl0qsjkMjeUceuRwQo0ezwOLiNoGF1wjL9NvTTNbTltIQ8kbd8HAjlP5 -ivHijnnX2xYgmDML+hpEvAKxlAf232DQtxpbcNQZ/KlFGa2UVmocsfKpA/9OUnTEyNGui9DmDhhy -E9nhTXmI4r4yty6GLvZtP6NnCxjPHFvNqPfOAQlDE0Zm0romyQVp+/ow/5kU+l6zR4O0Xe0NtKAI -9LLoaKn3iNw9O1kmJ9Cz5dPLB+x5ujfQDphdIU6YYLONIm7zb2s1o7jH04pcJXeBvX6dA1DnoBIe -uOpevEhdf2JmcBIW3mWX9PSHi/evTi+IDpPL09f/dfoD1VSjxLbC+Qfv3mX5AVP6wDt55e7k6ULY -vsEbZHvuRubDuxpC532ngrAHQv9B7b7FaUd67uttHTrnXbudAPHd025B3RbT9kLupKz0lVNu8Wvr -tNVAP+WjMeaXU1ZoHpkSIHes1o4lDvnCOXLSjSdZLpryIAO8KezQAWtn03vbqjvDdWNrnQfUdx9v -2ZEf2XNWtHwY5SFz2nIOc8rUlgxw/qf9z+OQ94Vn9M2dYSCXM+lcX0s31zKoyv93eEqI/NGf0v/i -zdjeaU/tuHRF2X+sAWvQZjrI6RyS787PLUWNZbqFCqAuWVWae1kZFaM6daGXLY18osTnA7rQ4gAV -0LX9haum04d/S7AAsrLXGyquTmZTBo3ndeoWNdo+nQ6UVaNSmXzunCAFLXgIlG6kW4F0YEKKbdx0 -I/afqH2d88KzMURJfTepgzw6KQ72hlrPxMG26xrc6wqEeL69Ydy6EUH3OOIe6p4eqjZn4h1Lh2c0 -tt3DIL4nyFxtJehWQy/hh8V9ekMEvt5+fn5syx6Q4/G1o4eolDywRudEfHb85J2X6Drd8evvX5R/ -jO0ZLkzuj9qjXAetw1zbc8idw6lb8sym7pIhBd77/i1F08P7pyuCQRtXy4HHNDMx/P2PkZ2dcxZX -T8E+GXWn3WixbaD4dLELio4jd0F19KJ4UuJ+VPu5c2Jkw6e+hs/seTG6mNythgkcOSFnzOcR7mGS -+c1M293pSo4HdWcUTX9u0OZJvfwg7uyMcKuRl77ZbibaU294YotPwbeb9Pd//oD+3VMNtvvRrqSP -bfWi90Q++7t4FAarlttcap6HYHlAlQ5JgT8RQyv7eOdTQ0iXoZrJIWNQ+EMONwZIJ5gxI994op0K -aygG/wu2AIfx -""" -) - -# file activate.sh -ACTIVATE_SH = convert( - """ -eJytVV1v2kAQfPevWAxKk7QU0cdWRCUKEkgJRJhStU3lHPYSn2rO6O5MQj7+e/dsY2wc6EPDA2Dv -3N3szuxeHSYBVzDnIcIiVhpmCLFCH+65DsBWUSw9hBkXLeZpvmIabTidy2gBM6aCU6sO6ygGjwkR -aZCxAK7B5xI9Ha4ty8fNKjg+gScL6BMLhRqac1iu/ciDs5aPq5aIwxA+nR21rQRTB4kGFYU+oFhx -GYkFCg0rJjmbhagyVA1+QfMRGk/T7vi9+wK/aZ2OpVCgZYzA50ABoPx89EImKS2mgYVhspyi2Xq7 -8eSOLi/c6WA8+da9dK+7kz5tZ9N+X0AHKBK8+ZhIx25U0HaOwIdlJHUCzN+lKVcWJfE5/xeZH5P+ -aNgfXfX2UMrjFWJ5pEovDx0kWUYR1azuiWdUEMWkj4+a1E7sAEz48KiCD3AfcC+AgK0QGP1QyIsW -CxPWAUlgnJZtRX7zSBGSRkdwRwzIQPRvHknzsGRkyWyp+gjwnVwZxToLay7usm1KQFMgaJgSgxcw -cYcK7snezDdfazBWpWPJYktijv5GACq/MOU/7zr9ZlLq5+f85U+n7057Y2cwGjZfkyFJsinJxLmh -S0U7ILDT3qOs065I6rSrWjrtgyJm4Q2RFLKJ9obTbfo1w61t0uuALSLho6I+Mh2MO/Tq4GA4hw2g -tkOgaUKb1t+c/mLRtEjjXEoMccVKLV0YFuWzLavAtmO7buHRdW0rq0MxJavSbFTJtFGzhwK65brn -g6E77F71XPdzBiv2cc572xCmYPTGKsl6qFX3NJahtdOmu0dZRrnUnskpxewvBk73/LLnXo9HV9eT -ijF3jdAxJB2j8FZ0+2Fb0HQbqinUOvCwx5FVeGlTDBWWFxzf0nBAwRYIN6XC39i3J1BanE3DgrNN -8nW4Yn8QVCzRzIZYsJAzlV0glATX7xSNdYnMXxvCEq0iotCSxevm6GhnJ+p2c21YVvqY31jLNQ0d -Ac1FhrMbX+3UzW8yB99gBv7n/Puf2ffa3CPN/gKu/HeT -""" -) - -# file activate.fish -ACTIVATE_FISH = convert( - """ -eJytVm1v2zYQ/q5fcZUdyClqGVuHfQgwDGnjIQYSO3DcAMM6yLREWxxo0iMpty7243ekLImy5RQY -lg+RJT73fvfwerDImYY14xS2hTawolBomuE/Jjaw1LJQKYUVEyOSGrYnhsZrpvMlvP3CTM4EEFCF -EBZsv8MAcmN2N6ORfdM55TxO5RauQVOtmRRv46AHdxKENFYQmIGMKZoafoiDYF0ItCIFJCuic7Y+ -JDtichhmEH6UYk+V0WjQGXIHRoKWW2od2YAVgZQIQHVyI9g3GgaAf5oaJ3JU1idqs68PrFB10ID+ -+OFPh1hL5QzhR2UAo/UxP8bx8Ijr0Bb2m5ebfq2kdImKrHymuQQPGNgDLwvW2qUsuHDPs+CS05GF -0pSNHf4BoyC6iSD6LKITkxmt6mztReOvWKA9U6YgnIo9bGVGgYgMtZtCCWva5BSrqbaEY1gIlWJL -hYkjZ7WHQJsYyTP/FPZEMbLiVDsUW4Oh2HxDgWlLZg93yctkvvh0+5A83S7uwzrFPddcGrtrg81X -rGxruUYbuk7zfzKtC6pHP73/GQg3VGFLW12Qo/Mc81TrrGwPygT9Nnm+T17G8+fJbFomKoxDCD+L -8BqbAobcwPtatir7cPO11D5oV+w8lutalnJNLys6l2wEj71Ty1DoBrvCfie9vy/uZ9P72eM4DM78 -qM9OvakPXvejDXvFG5fzp/ns8WmRzDD388nd2C/6M2rHhqbbnTkAlyl22tINYlK1rUv30nYj4Vx+ -cT2p6FbuESrXsHTgnZKoYVlRWyWr0fNl3A6Fw7n6wPNorIim3lxE+sRGOSLaSEWdM1KxDROEN3Z8 -8+DJdgFSSHCAEg/1PQl6JtFZq67Mt6t1RFdFHU9f2lUMHaXgaITw5heIhBQZflaFJREatYrI18Pq -7E23z7tDJtPuo4aXLoTrXxgXIP5s1lG6SHvwSdhImVKU0z3xGSoOPE5sxxcE1bB4+YEwSbzXJAmt -/v+PuP4jYVWennEFklbrsu2XPFXz02VBh3QJbHFX2PfCHyXJh8k0md4+jjETR5E638t+wxJL21P4 -MQ7KJwz/hhMO6XbF46kuPPW1tC+7pt92B5Pjh+G2/HZcEhy65qtv7ciSu8nz7YeH8XF+wuN991Hu -Dm7k0wKbCRupTQy1bYKUcTqjRxpqTb4/9Gcz3YJ3cgIOHtnTlkN9bYgp9Du33XgyGlHErmN6x8kB -N7MzUrTmS+FKiU+KT6WTEhcUxXBNQK17fGa/epjJ2m5+7+Avu2vuFN1hip1z/nIgyJY2tv37opms -I2klzT3hyqiYMGuIrvSVjjrhMMBYklRyjL3cWl65kht1gyt9DVGHMAxweKj1uN0doae24tIyBfOO -a6FOZy1jZzukdvvqN1kPccDLjbwGdtJ8m72rgeki+xOnXcf/CzFcuJM= -""" -) - -# file activate.csh -ACTIVATE_CSH = convert( - """ -eJx9VNtO4zAQffdXDKEiUEFhX8t22bJFWqRyEVuQVkKy3Hi6sZQ44Dit+sK379hJittG5KGqPZdz -fOZyCLNUlbBQGUJelRbmCFWJElbKphCVRWUShLnS5yKxaiksDpIyjaC/MEUO9Lc/YIfwt6ggEVoX -FkylQVmQymBis7Wz/jJIcRLma5iIpZIIEwXXmSgVfJf+Qs5//suFygZJkf8YMFaiBY2rTGkcxa8s -ZkxkSpQgsWUBsUVi27viD9MJf7l9mj2Pp/xxPPsNByO4gKMjoCSol+Dvot6e3/A9cl6VdmB71ksw -mIoyvYROnKeHu8dZiARvpMebHe0CeccvoLz9sjY5tq3h5v6lgY5eD4b9yGFFutCSrkzlRMAm554y -we3bWhYJqXcIzx5bGYMZLoW2sBRGiXmG5YAFsdsIvhA7rCDiPDhyHtXl2lOQpGhkZtuVCKKH7+ec -X9/e8/vx3Q3nw00EfWoBxwFWrRTBeSWiE7Apagb0OXRKz7XIEUbQFcMwK7HLOT6OtwlZQo9PIGao -pVrULKj64Ysnt3/G19ObtgkCJrXzF74jRz2MaCnJgtcN5B7wLfK2DedOp4vGydPcet5urq2XBEZv -DcnQpBZVJt0KUBqEa4YzpS0a3x7odFOm0Dlqe9oEkN8qVUlK01/iKfSa3LRRKmqkBc2vBKFpmyCs -XG4d2yYyEQZBzIvKOgLN+JDveiVoaXyqedVYOkTrmCRqutrfNVHr6xMFBhh9QD/qNQuGLvq72d03 -3Jy2CtGCf0rca/tp+N4BXqsflKquRr0L2sjmuClOu+/8/NKvTQsNZ3l9ZqxeTew//1a6EA== -""" -) - -# file activate.xsh -ACTIVATE_XSH = convert( - """ -eJyNU11PwjAUfe+vuNY9sIj7ASQ+YCSBRD6i02gIaSq7gyWjXdqyaIz/3XYwVmB+9GFZ78c57T2n -lNIXKfQa+NJkJTcIeqmywkAqFZSZMlueoygppSRVcgPvrjgyUuYask0hlYEVGqaxAK6B7f8JSTAF -lmCN2uFqpcMeAbuyFGjxkcglhUwAzzOuUe9SbiWY18H5vm5B6sbgM4qir8jSdCib3t+x59FD/NS/ -Z7N+PKRdoDRskAIXhBsIziqPyFrSf9O9xsPpZDgdD85JD6lz6kPqtwM0RYdx1bnB5Lka2u5cxzML -vKLWTjZ7mI5n8b8A9rUNjpAiQW3U1gmKFIQ0lXpW1gblEh4xT6EuvGjXtHGFE5ZcwlZotGhKYY4l -FwZKrjL+lqMmvoXmp4dYhKQV1M7d6yPEv5jNKcqYf1VGbcmZB5x4lRcCfzfvLXaBiCdJ5wj46uD+ -Tmg3luR2NGGT/nhgGbpgX48wN7HaYhcUFjlfYrULCTkxWru36jF59rJ9NlJlf7JQde5j11VS+yZr -0d22eUPaxdycLKMTvqWjR3610emDtgTu36ylcJe83rhv/di/AYN1UZY= -""" -) - -# file activate.bat -ACTIVATE_BAT = convert( - """ -eJyVk1FLhEAUhd8X/A8XWSkf28dCyMUpBR3FzAiCS+WYwq4TOdXfb0Z3dTJdyCfveO85n8frNXut -OPCyNFbGqmUCzDxIs3s3REJzB1GrEE3VVJdQsLJuWAEYh97QkaRxlGRwbqxAXp1Uf+RYM32W1LKB -7Vp2nJC6DReD9m+5qeQ6Wd+a/SN7dlzn9oI7dxsSXJCcoXOskfLgYXdv/j8LnXiM8iGg/RmiZmOr -bFMSgcebMwGfKhgbBIfnL14X8P7BX3Zs38J3LSoQFdtD3YCVuJlvnfgmj5kfUz+OCLxxqUWoF9zk -qtYAFyZkBsO9ArzUh/td0ZqP9IskElTFMsnwb4/GqeoLPUlZT5dJvf8Id5hQIONynUSa2G0Wc+m8 -Z+w2w4/Tt2hbYT0hbgOK1I0I4tUw/QOTZfLE -""" -) - -# file deactivate.bat -DEACTIVATE_BAT = convert( - """ -eJyFkN0KgkAUhO8X9h0GQapXCIQEDQX/EBO6kso1F9KN3Or1201Si6JzN+fMGT5mxQ61gKgqSijp -mETup9nGDgo3yi29S90QjmhnEteOYb6AFNjdBC9xvoj9iTUd7lzWkDVrwFuYiZ15JiW8QiskSlbx -lpUo4sApXtlJGodJhqNQWW7k+Ou831ACNZrC6BeW+eXPNEbfl7OiXr6H/oHZZl4ceXHoToG0nuIM -pk+k4fAba/wd0Pr4P2CqyLeOlJ4iKfkJo6v/iaH9YzfPMEoeMG2RUA== -""" -) - -# file activate.ps1 -ACTIVATE_PS = convert( - """ -eJytVcFu2zAMvfsrWNfYEmx2sWuGHVIkQAO0adBkvWyDoMh0I0CWDElOGwz598p2HDt20w3YdLP5 -SD2Sj9QlrDbcQMIFQpobC2uEWFkwKtcMY0i0SmGhlpuvsFM5MCqls+pcArcQc43Mil0EEwXWxRlB -BNEVZZZvqcUoM188LzBM88yOVjezJVmMVzfwDYJ0x+VWMWq5klG6YypNqYyjjNrN0eF6vJySyezB -4U0muA0LKww0GiW2WH35wTHsVRT5QwgXVKO0npfkkhXR4UmoNRWjGGtagx/mmVu2+QXBXMkJGqvz -woRD77cH7vAEBtb9rq7YUs3pWuCI3N9OyOPsYfV9fFveOCzRlU9xApTbUZ3hebcjXmOqXCI1Evwe -1IfQMJXhIYnSc++9QbLOdkSUiMmWa5tTQTLXvMz2aB7Blb1g+55/ly+3mDYX/jzn0eJZFqbOazp/ -7DEqQjjMaae7XhAKpMmbbHpI1FppWgnAOG1Lp1KmpOUyxy67i54MTtldwhJFAvEBcfF+PRqdvY/L -drFiByr7vlwrc0Ui29mNs4QplH8hoPrJFE6XkEuDFrjWKHBLpT2q1XgNEQilknGToJvJdrFc++tR -83o1d8bWp/M88OtJtUDW+vfak+B3Y14Rcj2bk/n4bkrcKWBkOV0Q4sOnVoyqNR1fR3I5vr6dksXD -/d1iVU9st3rnNNk01fcPtT+diDccT8ajFavg55OmBiWhIolQulH2uyrqNfh0thpgJblxHLuNisDc -TnYbONOY8BewqvyJL9w4KT9BJ0hxnrXTWbhRbpWfYackPgsu8cTtw1/ugH2rbCgM/nuWtJMcy3Wx -wQ+5fYZ17h4aJT8Wz41hVMfu5YnOpjwIBn/eITAcwn+rxN57BRHvOEk= -""" -) - -# file distutils-init.py -DISTUTILS_INIT = convert( - """ -eJytV21v5DQQ/p5fMaRCJLANcAcSqlghuBdUcRzo6BdUnSI3cXZNs3bO9m679+uZsbOJnWR7fKBS -u65nPC/PvK7YdUpbUCYR/mSOw/GBaSnkxiTJBaiuUjUHYUAqCwwOQts9a7k8wE7V+5avwCh44FAx -CXuDnBasgkbIGuyWg7F1K+5Q0LWTzaT9DG7wgdL3oCR0x+64QkaUv9sbC3ccdXjBeMssaG5EzQ0I -SeJQDkq77I52q+TXyCcawevLx+JYfIRaaF5ZpY8nP7ztSYIEyXYc1uhu0TG7LfobIhm7t6I1Jd0H -HP8oIbMJe+YFFmXZiJaXZb6CdBCQ5olohudS6R0dslhBDuuZEdnszSA/v0oAf07xKOiQpTcIaxCG -QQN0rLpnG0TQwucGWNdxpg1FA1H1+IEhHFpVMSsQfWb85dFYvhsF/YS+8NZwr710lpdlIaTh2mbf -rGDqFFxgdnxgV/D6h2ffukcIBUotDlwbVFQK2Sj4EbLnK/iud8px+TjhRzLcac7acvRpTdSiVawu -fVpkaTk6PzKmK3irJJ/atoIsRRL9kpw/f/u1fHn97tWLmz/e/Z3nTunoaWwSfmCuFTtWbYXkmFUD -z9NJMzUgLdF9YRHA7pjmgxByiWvv31RV8Zfa64q/xix449jOOz0JxejH2QB8HwQg8NgeO26SiDIL -heMpfndxuMFz5p0oKI1H1TGgi6CSwFiX6XgVgUEsBd2WjVa70msKFa56CPOnbZ5I9EnkZZL0jP5M -o1LwR9Tb51ssMfdmX8AL1R1d9Wje8gP2NSw7q8Xd3iKMxGL1cUShLDU/CBeKEo2KZRYh1efkY8U7 -Cz+fJL7SWulRWseM6WvzFOBFqQMxScjhoFX0EaGLFSVKpWQjNuSXMEi4MvcCa3Jw4Y4ZbtAWuUl6 -095iBAKrRga0Aw80OjAhqy3c7UVbl/zRwlgZUCtu5BcW7qV6gC3+YpPacOvwxFCZoJc7OVuaFQ84 -U9SDgUuaMVuma2rGvoMRC3Y8rfb92HG6ee1qoNO8EY8YuL4mupbZBnst9eIUhT5/lnonYoyKSu12 -TNbF6EGP2niBDVThcbjwyVG1GJ+RK4tYguqreUODkrXiIy9VRy3ZZIa3zbRC0W68LRAZzfQRQ4xt -HScmNbyY01XSjHUNt+8jNt6iSMw3aXAgVzybPVkFAc3/m4rZHRZvK+xpuhne5ZOKnz0YB0zUUClm -LrV9ILGjvsEUSfO48COQi2VYkyfCvBjc4Z++GXgB09sgQ9YQ5MJFoIVOfVaaqyQha2lHKn3huYFP -KBJb8VIYX/doeTHjSnBr8YkT34eZ07hCWMOimh6LPrMQar8cYTF0yojHdIw37nPavenXpxRHWABc -s0kXJujs0eKbKdcs4qdgR4yh1Y5dGCJlMdNoC5Y5NgvcbXD9adGIzAEzLy/iKbiszYPA/Wtm8UIJ -OEGYljt14Bk9z5OYROuXrLMF8zW3ey09W+JX0E+EHPFZSIMwvcYWHucYNtXSb8u4AtCAHRiLmNRn -1UCevMyoabqBiRt3tcYS9fFZUw/q4UEc/eW8N/X3Tn1YyyEec3NjpSeVWMXJOTNx5tWqcsNwLu5E -TM5hEMJTTuGZyMPGdQ5N+r7zBJpInqNJjbjGkUbUs+iGTEAt63+Ee2ZVbNMnwacF6yz4AXEZ/Ama -5RTNk7yefGB+5ESiAtoi/AE9+5LpjemBdfj0Ehf09Lzht5qzCwT9oL00zZZaWjzEWjfEwoU9mMiD -UbThVzZ34U7fXP+C315S91UcO9rAFLen4fr29OA9WnOyC1c8Zu5xNaLeyNo2WNvPmkCtc2ICqidc -zmg+LaPu/BXc9srfx9pJbJiSw5NZkgXxWMiyBWpyNjdmeRbmzb+31cHS -""" -) - -# file distutils.cfg -DISTUTILS_CFG = convert( - """ -eJxNj00KwkAMhfc9xYNuxe4Ft57AjYiUtDO1wXSmNJnK3N5pdSEEAu8nH6lxHVlRhtDHMPATA4uH -xJ4EFmGbvfJiicSHFRzUSISMY6hq3GLCRLnIvSTnEefN0FIjw5tF0Hkk9Q5dRunBsVoyFi24aaLg -9FDOlL0FPGluf4QjcInLlxd6f6rqkgPu/5nHLg0cXCscXoozRrP51DRT3j9QNl99AP53T2Q= -""" -) - -# file activate_this.py -ACTIVATE_THIS = convert( - """ -eJylVE1v2zAMvetXENqhNpZ5wHoL0EMOBdqh64Kt3RAEhaE4TMzOlgxJ+ULR/z7Sdpr041BsOUSS -9fj0SD5Jaz0qIq1NRFiTjytToV3DwnkoVt6jjUA2om888v9QqduAgFssEtegTWJJIV9QhWnm0cyT -dAAPJ3n7Jc9PhvC0/5hmSt3wCgpjYYawCjiHTYkWdm4F9SpE+QS8iVsKkewSxrtYOnt8/gCsi0z6 -TOuM7OemhWZKa62obpyP4MJ+Fiji03wXlIp+N1TAv71ShdsCmwjXpsZz753vtr0hljQKAX0kZ9ud -RE+O9f5TKVKdKvUBOCcOnEsCEB2MRzcX0NKAwIBHsoHm2CYsoDl5LKLzu1TxMuclnHGeWWNimfHK -svxkvzazIGOyl5Cmire4YOSdnWo5Td8d4gM22b0jm0x76jv4CIeAbIkx6YIGoHWahaaimByCmV7N -DFfktaKesM257xtI4zhBT8sygpm59YsMn2n9dfnj5nZ0lZ9f/2olyzlCZubzYzNAH1Cza0Pb9U+N -Kf6YJUp5BVg6blvT26ozRI1FaSyFWl3+zMeT8YT5SxNMjD5hs3Cyza7Z5Wv0gS2Qk1047h5jv05u -Lr5fM5pRWVOZyHemzkI0PoYNceH1vVkbxtICnuCdr0Ra3ksLRwVr6y/J8alXNJNKH2cRmAyrjk6U -vp/sNUvALpqpfl++zALOzkBvyJ5+0S2oO5JxXcx/piDhBwHvJas6sq55D486E6EmSo+yvjnT4eld -+saBii/aWlLEDi7cqRJUxg6SkW2XPBPB2wzke1zlHLyg7b5C7UIdpkdu/CYmFpcxKb9tTFeHvfEW -bEt+khbtQs4f8N0GrneByuKGWSp+9I7V9bPpUAw/pfZFJgkSODeE2qdQSDg5uatvYvb76i9zKfxE -""" -) - -# file python-config -PYTHON_CONFIG = convert( - """ -eJyNVV1P2zAUfc+v8ODBiSABxlulTipbO6p1LWqBgVhlhcZpPYUkctzSivHfd6+dpGloGH2Ja/ue -e+65Hz78xNhtf3x90xmw7vCWsRPGLvpDNuz87MKfdKMWSWxZ4ilNpCLZJiuWc66SVFUOZkkcirll -rfxIBAzOMtImDzSVPBRrekwoX/OZu/0r4lm0DHiG60g86u8sjPw5rCyy86NRkB8QuuBRSqfAKESn -3orLTCQxE3GYkC9tYp8fk89OSwNsmXgizrhUtnumeSgeo5GbLUMk49Rv+2nK48Cm/qMwfp333J2/ -dVcAGE0CIQHBsgIeEr4Wij0LtWDLzJ9ze5YEvH2WI6CHTAVcSu9ZCsXtgxu81CIvp6/k4eXsdfo7 -PvDCRD75yi41QitfzlcPp1OI7i/1/iQitqnr0iMgQ+A6wa+IKwwdxyk9IiXNAzgquTFU8NIxAVjM -osm1Zz526e+shQ4hKRVci69nPC3Kw4NQEmkQ65E7OodxorSvxjvpBjQHDmWFIQ1mlmzlS5vedseT -/mgIEsMJ7Lxz2bLAF9M5xeLEhdbHxpWOw0GdkJApMVBRF1y+a0z3c9WZPAXGFcFrJgCIB+024uad -0CrzmEoRa3Ub4swNIHPGf7QDV+2uj2OiFWsChgCwjKqN6rp5izpbH6Wc1O1TclQTP/XVwi6anTr1 -1sbubjZLI1+VptPSdCfwnFBrB1jvebrTA9uUhU2/9gad7xPqeFkaQcnnLbCViZK8d7R1kxzFrIJV -8EaLYmKYpvGVkig+3C5HCXbM1jGCGekiM2pRCVPyRyXYdPf6kcbWEQ36F5V4Gq9N7icNNw+JHwRE -LTgxRXACpvnQv/PuT0xCCAywY/K4hE6Now2qDwaSE5FB+1agsoUveYDepS83qFcF1NufvULD3fTl -g6Hgf7WBt6lzMeiyyWVn3P1WVbwaczHmTzE9A5SyItTVgFYyvs/L/fXlaNgbw8v3azT+0eikVlWD -/vBHbzQumP23uBCjsYdrL9OWARwxs/nuLOzeXbPJTa/Xv6sUmQir5pC1YRLz3eA+CD8Z0XpcW8v9 -MZWF36ryyXXf3yBIz6nzqz8Muyz0m5Qj7OexfYo/Ph3LqvkHUg7AuA== -""" -) - -MH_MAGIC = 0xFEEDFACE -MH_CIGAM = 0xCEFAEDFE -MH_MAGIC_64 = 0xFEEDFACF -MH_CIGAM_64 = 0xCFFAEDFE -FAT_MAGIC = 0xCAFEBABE -BIG_ENDIAN = ">" -LITTLE_ENDIAN = "<" -LC_LOAD_DYLIB = 0xC -maxint = MAJOR == 3 and getattr(sys, "maxsize") or getattr(sys, "maxint") - - -class FileView(object): - """ - A proxy for file-like objects that exposes a given view of a file. - Modified from macholib. - """ - - def __init__(self, file_obj, start=0, size=maxint): - if isinstance(file_obj, FileView): - self._file_obj = file_obj._file_obj - else: - self._file_obj = file_obj - self._start = start - self._end = start + size - self._pos = 0 - - def __repr__(self): - return "".format(self._start, self._end, self._file_obj) - - def tell(self): - return self._pos - - def _checkwindow(self, seek_to, op): - if not (self._start <= seek_to <= self._end): - raise IOError( - "{} to offset {:d} is outside window [{:d}, {:d}]".format(op, seek_to, self._start, self._end) - ) - - def seek(self, offset, whence=0): - seek_to = offset - if whence == os.SEEK_SET: - seek_to += self._start - elif whence == os.SEEK_CUR: - seek_to += self._start + self._pos - elif whence == os.SEEK_END: - seek_to += self._end - else: - raise IOError("Invalid whence argument to seek: {!r}".format(whence)) - self._checkwindow(seek_to, "seek") - self._file_obj.seek(seek_to) - self._pos = seek_to - self._start - - def write(self, content): - here = self._start + self._pos - self._checkwindow(here, "write") - self._checkwindow(here + len(content), "write") - self._file_obj.seek(here, os.SEEK_SET) - self._file_obj.write(content) - self._pos += len(content) - - def read(self, size=maxint): - assert size >= 0 - here = self._start + self._pos - self._checkwindow(here, "read") - size = min(size, self._end - here) - self._file_obj.seek(here, os.SEEK_SET) - read_bytes = self._file_obj.read(size) - self._pos += len(read_bytes) - return read_bytes - - -def read_data(file, endian, num=1): - """ - Read a given number of 32-bits unsigned integers from the given file - with the given endianness. - """ - res = struct.unpack(endian + "L" * num, file.read(num * 4)) - if len(res) == 1: - return res[0] - return res - - -def mach_o_change(at_path, what, value): - """ - Replace a given name (what) in any LC_LOAD_DYLIB command found in - the given binary with a new name (value), provided it's shorter. - """ - - def do_macho(file, bits, endian): - # Read Mach-O header (the magic number is assumed read by the caller) - cpu_type, cpu_sub_type, file_type, n_commands, size_of_commands, flags = read_data(file, endian, 6) - # 64-bits header has one more field. - if bits == 64: - read_data(file, endian) - # The header is followed by n commands - for _ in range(n_commands): - where = file.tell() - # Read command header - cmd, cmd_size = read_data(file, endian, 2) - if cmd == LC_LOAD_DYLIB: - # The first data field in LC_LOAD_DYLIB commands is the - # offset of the name, starting from the beginning of the - # command. - name_offset = read_data(file, endian) - file.seek(where + name_offset, os.SEEK_SET) - # Read the NUL terminated string - load = file.read(cmd_size - name_offset).decode() - load = load[: load.index("\0")] - # If the string is what is being replaced, overwrite it. - if load == what: - file.seek(where + name_offset, os.SEEK_SET) - file.write(value.encode() + "\0".encode()) - # Seek to the next command - file.seek(where + cmd_size, os.SEEK_SET) - - def do_file(file, offset=0, size=maxint): - file = FileView(file, offset, size) - # Read magic number - magic = read_data(file, BIG_ENDIAN) - if magic == FAT_MAGIC: - # Fat binaries contain nfat_arch Mach-O binaries - n_fat_arch = read_data(file, BIG_ENDIAN) - for _ in range(n_fat_arch): - # Read arch header - cpu_type, cpu_sub_type, offset, size, align = read_data(file, BIG_ENDIAN, 5) - do_file(file, offset, size) - elif magic == MH_MAGIC: - do_macho(file, 32, BIG_ENDIAN) - elif magic == MH_CIGAM: - do_macho(file, 32, LITTLE_ENDIAN) - elif magic == MH_MAGIC_64: - do_macho(file, 64, BIG_ENDIAN) - elif magic == MH_CIGAM_64: - do_macho(file, 64, LITTLE_ENDIAN) - - assert len(what) >= len(value) - - with open(at_path, "r+b") as f: - do_file(f) - - -if __name__ == "__main__": - main() diff --git a/src/build_utils/virtualenv/virtualenv_support/__init__.py b/src/build_utils/virtualenv/virtualenv_support/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/build_utils/virtualenv/virtualenv_support/pip-19.1.1-py2.py3-none-any.whl b/src/build_utils/virtualenv/virtualenv_support/pip-19.1.1-py2.py3-none-any.whl deleted file mode 100644 index 8476c1193..000000000 Binary files a/src/build_utils/virtualenv/virtualenv_support/pip-19.1.1-py2.py3-none-any.whl and /dev/null differ diff --git a/src/build_utils/virtualenv/virtualenv_support/setuptools-41.0.1-py2.py3-none-any.whl b/src/build_utils/virtualenv/virtualenv_support/setuptools-41.0.1-py2.py3-none-any.whl deleted file mode 100644 index 92836e984..000000000 Binary files a/src/build_utils/virtualenv/virtualenv_support/setuptools-41.0.1-py2.py3-none-any.whl and /dev/null differ diff --git a/src/build_utils/virtualenv/virtualenv_support/wheel-0.33.4-py2.py3-none-any.whl b/src/build_utils/virtualenv/virtualenv_support/wheel-0.33.4-py2.py3-none-any.whl deleted file mode 100644 index ded10b0cd..000000000 Binary files a/src/build_utils/virtualenv/virtualenv_support/wheel-0.33.4-py2.py3-none-any.whl and /dev/null differ diff --git a/src/rez/__init__.py b/src/rez/__init__.py index 36c06595e..477fe3983 100644 --- a/src/rez/__init__.py +++ b/src/rez/__init__.py @@ -5,13 +5,15 @@ from __future__ import print_function from rez.utils._version import _rez_version +import rez.deprecations import sys import os +import warnings __version__ = _rez_version __author__ = "Allan Johns" -__license__ = "LGPL" +__license__ = "Apache-2.0" module_root_path = __path__[0] # noqa @@ -59,3 +61,19 @@ def callback(sig, frame): if callback: signal.signal(signal.SIGUSR1, callback) # Register handler + + +# Log all rez warnings, ignoring possible user defined warning filters. +# We can't tell users to use something like PYTHONWARNINGS=default::rez.deprecations.RezDeprecationWarning +# because python reads PYTHONWARNINGS before it actually can import modules. So it +# basically can't import rez when PYTHONWARNINGS is read. +# This means we have to rely on a custom environment variable. +if os.getenv("REZ_LOG_DEPRECATION_WARNINGS"): + warnings.filterwarnings("default", category=rez.deprecations.RezDeprecationWarning) + + +if sys.version_info[:2] < (3, 7): + rez.deprecations.warn( + "Support for Python less than 3.7 is deprecated and will be removed in rez 3.0.0.", + rez.deprecations.RezDeprecationWarning, + ) diff --git a/src/rez/bind/_utils.py b/src/rez/bind/_utils.py index 713a72ccd..8630a7631 100644 --- a/src/rez/bind/_utils.py +++ b/src/rez/bind/_utils.py @@ -6,14 +6,14 @@ Utility functions for bind modules. """ from __future__ import absolute_import -from rez.vendor.version.version import Version +from rez.version import Version from rez.exceptions import RezBindError from rez.config import config from rez.util import which from rez.utils.execution import Popen from rez.utils.logging_ import print_debug from rez.vendor.six import six -from pipes import quote +from rez.utils.py23 import quote import subprocess import os.path import os diff --git a/src/rez/bind/arch.py b/src/rez/bind/arch.py index 7bb55c706..146317694 100644 --- a/src/rez/bind/arch.py +++ b/src/rez/bind/arch.py @@ -7,7 +7,7 @@ """ from __future__ import absolute_import from rez.package_maker import make_package -from rez.vendor.version.version import Version +from rez.version import Version from rez.bind._utils import check_version from rez.system import system diff --git a/src/rez/bind/hello_world.py b/src/rez/bind/hello_world.py index d03a984b4..b973c1ee6 100644 --- a/src/rez/bind/hello_world.py +++ b/src/rez/bind/hello_world.py @@ -12,19 +12,13 @@ from __future__ import absolute_import, print_function from rez.package_maker import make_package -from rez.vendor.version.version import Version +from rez.version import Version from rez.utils.lint_helper import env from rez.utils.execution import create_executable_script, ExecutableScriptMode +from rez.vendor.distlib.scripts import ScriptMaker from rez.bind._utils import make_dirs, check_version import os.path - - -def setup_parser(parser): - parser.add_argument( - "--py-script-mode", type=str, default="platform_specific", metavar="PY_SCRIPT_MODE", - help="py script mode to use (default: %(default)s).", - choices=(ExecutableScriptMode._member_names_), - ) +import shutil def commands(): @@ -48,36 +42,29 @@ def hello_world_source(): sys.exit(opts.retcode) -def bind(path, version_range=None, py_script_mode=None, opts=None, parser=None): +def bind(path, version_range=None, opts=None, parser=None): version = Version("1.0") check_version(version, version_range) - # Allow the user to override the `py_script_mode` via the command line - # or via python API, as is the case for unit tests. Fall back to - # `platform_specific` if not specified. - py_script_mode = opts.py_script_mode if opts else py_script_mode - if py_script_mode is None: - py_script_mode = ExecutableScriptMode.platform_specific - else: - # Extra error checking for the python API - if py_script_mode not in ExecutableScriptMode._member_names_: - raise ValueError( - "Invalid py_script_mode: {!r} Choose between: {!r}".format( - py_script_mode, ExecutableScriptMode._member_names_ - ) - ) - py_script_mode = ExecutableScriptMode[py_script_mode] - def make_root(variant, root): binpath = make_dirs(root, "bin") - filepath = os.path.join(binpath, "hello_world") + binpathtmp = make_dirs(root, "bintmp") + filepath = os.path.join(binpathtmp, "hello_world") create_executable_script( filepath, hello_world_source, - py_script_mode=py_script_mode, + py_script_mode=ExecutableScriptMode.single, ) + # We want to use ScriptMaker on all platofrms. This allows us to + # correctly setup the script to work everywhere, even on Windows. + # create_executable_script should be fixed to use ScriptMaker + # instead. + maker = ScriptMaker(binpathtmp, make_dirs(binpath)) + maker.make("hello_world") + shutil.rmtree(binpathtmp) + with make_package("hello_world", path, make_root=make_root) as pkg: pkg.version = version pkg.tools = ["hello_world"] diff --git a/src/rez/bind/os.py b/src/rez/bind/os.py index a6a88c280..28fe9d2e0 100644 --- a/src/rez/bind/os.py +++ b/src/rez/bind/os.py @@ -7,7 +7,7 @@ """ from __future__ import absolute_import from rez.package_maker import make_package -from rez.vendor.version.version import Version +from rez.version import Version from rez.bind._utils import check_version from rez.system import system diff --git a/src/rez/bind/platform.py b/src/rez/bind/platform.py index f0e2e1c1e..5aac37c7c 100644 --- a/src/rez/bind/platform.py +++ b/src/rez/bind/platform.py @@ -7,7 +7,7 @@ """ from __future__ import absolute_import from rez.package_maker import make_package -from rez.vendor.version.version import Version +from rez.version import Version from rez.bind._utils import check_version from rez.system import system diff --git a/src/rez/bind/rezgui.py b/src/rez/bind/rezgui.py index 1fcbb07d4..844c32ecc 100644 --- a/src/rez/bind/rezgui.py +++ b/src/rez/bind/rezgui.py @@ -10,7 +10,7 @@ from rez.package_maker import make_package from rez.bind._utils import check_version, make_dirs from rez.system import system -from rez.vendor.version.version import Version +from rez.version import Version from rez.utils.lint_helper import env from rez.utils.execution import create_executable_script import shutil diff --git a/src/rez/build_process.py b/src/rez/build_process.py index a924bf673..93ae29775 100644 --- a/src/rez/build_process.py +++ b/src/rez/build_process.py @@ -16,7 +16,7 @@ from rez.config import config from rez.vendor.enum import Enum from contextlib import contextmanager -from pipes import quote +from rez.utils.py23 import quote import getpass import os.path import sys @@ -34,7 +34,13 @@ def get_build_process_types(): def create_build_process(process_type, working_dir, build_system, package=None, vcs=None, ensure_latest=True, skip_repo_errors=False, ignore_existing_tag=False, verbose=False, quiet=False): - """Create a `BuildProcess` instance.""" + """Create a :class:`BuildProcess` instance. + + .. warning:: + + The working_dir argument and the pacakge keyword argument will are deprecated + and will be removed in rez 3.0.0 + """ from rez.plugin_managers import plugin_manager process_types = get_build_process_types() if process_type not in process_types: @@ -79,9 +85,9 @@ def __init__(self, working_dir, build_system, package=None, vcs=None, """Create a BuildProcess. Args: - working_dir (DEPRECATED): Ignored. + working_dir (DEPRECATED): Will be removed in rez 3.0.0. build_system (`BuildSystem`): Build system used to build the package. - package (DEPRECATED): Ignored. + package (DEPRECATED): Will be removed in rez 3.0.0. vcs (`ReleaseVCS`): Version control system to use for the release process. ensure_latest: If True, do not allow the release process to occur diff --git a/src/rez/build_process_.py b/src/rez/build_process_.py index 1088ab23f..8a43e2544 100644 --- a/src/rez/build_process_.py +++ b/src/rez/build_process_.py @@ -2,11 +2,11 @@ # Copyright Contributors to the Rez Project -import warnings +import rez.deprecations from rez.build_process import * # noqa -warnings.warn( +rez.deprecations.warn( "rez.build_process_ is deprecated; import rez.build_process instead", - DeprecationWarning + rez.deprecations.RezDeprecationWarning, ) diff --git a/src/rez/build_system.py b/src/rez/build_system.py index be70ea7a9..4e4134582 100644 --- a/src/rez/build_system.py +++ b/src/rez/build_system.py @@ -28,7 +28,7 @@ def get_valid_build_systems(working_dir, package=None): must be present for the 'custom' build system type. Returns: - List of class: Valid build system class types. + list[type[BuildSystem]]: Valid build system class types. """ from rez.plugin_managers import plugin_manager from rez.exceptions import PackageMetadataError @@ -87,8 +87,10 @@ def create_build_system(working_dir, buildsys_type=None, package=None, opts=None "No build system is associated with the path %s.\n" "\n" "There is a rezbuild.py file present, suggesting you were " - "using the deprecated bez build system. You need to use a " - "custom build command instead. You can port your existing " + "using the deprecated bez build system. This functionality " + "will be completely removed in rez 3.0.0.\n" + "You need to use a custom build command instead. " + "You can port your existing " "rezbuild.py like so:\n" "\n" "Add this line to package.py:\n" @@ -221,16 +223,17 @@ def build(self, context, variant, build_path, install_path, install=False, build_type: A BuildType (i.e local or central). Returns: - A dict containing the following information: + dict: A dict containing the following information: + - success: Bool indicating if the build was successful. - extra_files: List of created files of interest, not including - build targets. A good example is the interpreted context file, - usually named 'build.rxt.sh' or similar. These files should be - located under build_path. Rez may install them for debugging - purposes. + build targets. A good example is the interpreted context file, + usually named 'build.rxt.sh' or similar. These files should be + located under build_path. Rez may install them for debugging + purposes. - build_env_script: If this instance was created with write_build_scripts - as True, then the build should generate a script which, when run - by the user, places them in the build environment. + as True, then the build should generate a script which, when run + by the user, places them in the build environment. """ raise NotImplementedError diff --git a/src/rez/bundle_context.py b/src/rez/bundle_context.py index f300c6514..a12c39025 100644 --- a/src/rez/bundle_context.py +++ b/src/rez/bundle_context.py @@ -41,7 +41,9 @@ def bundle_context(context, dest_dir, force=False, skip_non_relocatable=False, quiet (bool): Suppress all output patch_libs (bool): If True, modify libs and executables within the bundle to patch any references to external packages back to their - equivalents within the bundle. See the wiki for more details on this. + equivalents within the bundle. See + https://rez.readthedocs.io/en/stable/context_bundles.html#patching-libraries + for more details on this. verbose (bool): Verbose mode (quiet will override) """ bundler = _ContextBundler( diff --git a/src/rez/cli/_entry_points.py b/src/rez/cli/_entry_points.py index 8841932d9..8590e9d72 100644 --- a/src/rez/cli/_entry_points.py +++ b/src/rez/cli/_entry_points.py @@ -55,7 +55,7 @@ def check_production_install(): sys.stderr.write( "Pip-based rez installation detected. Please be aware that rez command " "line tools are not guaranteed to function correctly in this case. See " - "https://github.com/AcademySoftwareFoundation/rez/wiki/Installation#why-not-pip-for-production " + "https://rez.readthedocs.io/en/stable/installation.html#why-not-pip-for-production " " for futher details.\n" ) diff --git a/src/rez/cli/mv.py b/src/rez/cli/mv.py index 3fe4287d2..e02688296 100644 --- a/src/rez/cli/mv.py +++ b/src/rez/cli/mv.py @@ -54,7 +54,7 @@ def list_repos_containing_pkg(pkg_name, pkg_version): def command(opts, parser, extra_arg_groups=None): - from rez.vendor.version.requirement import VersionedObject + from rez.version import VersionedObject from rez.packages import get_package_from_repository from rez.package_move import move_package import sys diff --git a/src/rez/cli/pip.py b/src/rez/cli/pip.py index 37ec0568c..2de7be4d0 100644 --- a/src/rez/cli/pip.py +++ b/src/rez/cli/pip.py @@ -19,7 +19,7 @@ def setup_parser(parser, completions=False): parser.add_argument( "--pip-version", dest="pip_ver", metavar="VERSION", help="pip version (rez package) to use, default is latest." - " This option is deprecated and will be removed in the future.") + " This option is deprecated and will be removed in 3.0.0.") parser.add_argument( "-i", "--install", action="store_true", help="install the package") @@ -49,20 +49,17 @@ def command(opts, parser, extra_arg_groups=None): logging.getLogger('rez').setLevel(logging.INFO) from rez.pip import pip_install_package - import warnings + import rez.deprecations # a bit weird, but there used to be more options. Leave like this for now if not opts.install: parser.error("Expected one of: --install") if opts.pip_ver: - with warnings.catch_warnings(): - # Cause all warnings to always be triggered. - warnings.simplefilter("always") - warnings.warn( - "The option --pip-version is deprecated and will be removed in a future version", - category=DeprecationWarning - ) + rez.deprecations.warn( + "The option --pip-version is deprecated and will be removed in 3.0.0", + category=rez.deprecations.RezDeprecationWarning, + ) pip_install_package( opts.PACKAGE, diff --git a/src/rez/cli/pkg-ignore.py b/src/rez/cli/pkg-ignore.py index 66a1dceab..557b1ca67 100644 --- a/src/rez/cli/pkg-ignore.py +++ b/src/rez/cli/pkg-ignore.py @@ -63,7 +63,7 @@ def list_repos_containing_pkg(pkg_name, pkg_version): def command(opts, parser, extra_arg_groups=None): from rez.package_repository import package_repository_manager - from rez.vendor.version.requirement import VersionedObject + from rez.version import VersionedObject import sys obj = VersionedObject(opts.PKG) diff --git a/src/rez/cli/rm.py b/src/rez/cli/rm.py index 31512e2d2..664152437 100644 --- a/src/rez/cli/rm.py +++ b/src/rez/cli/rm.py @@ -36,7 +36,7 @@ def setup_parser(parser, completions=False): def remove_package(opts, parser): - from rez.vendor.version.requirement import VersionedObject + from rez.version import VersionedObject from rez.package_remove import remove_package if opts.dry_run: @@ -55,7 +55,7 @@ def remove_package(opts, parser): def remove_package_family(opts, parser, force=False): - from rez.vendor.version.requirement import VersionedObject + from rez.version import VersionedObject from rez.package_remove import remove_package_family from rez.exceptions import PackageRepositoryError diff --git a/src/rez/cli/search.py b/src/rez/cli/search.py index 5327688fe..b339f5717 100644 --- a/src/rez/cli/search.py +++ b/src/rez/cli/search.py @@ -73,6 +73,7 @@ def command(opts, parser, extra_arg_groups=None): from rez.package_search import ResourceSearcher, ResourceSearchResultFormatter from rez.utils.formatting import get_epoch_time_from_str from rez.config import config + import rez.deprecations before_time = get_epoch_time_from_str(opts.before) after_time = get_epoch_time_from_str(opts.after) @@ -80,6 +81,12 @@ def command(opts, parser, extra_arg_groups=None): if after_time and before_time and (after_time >= before_time): parser.error("non-overlapping --before and --after") + if opts.sort: + rez.deprecations.warn( + "the '--sort' argument is deprecated and will be removed in 3.0.0. It currently has no effect.", + rez.deprecations.RezDeprecationWarning, + ) + if opts.no_warnings: config.override("warn_none", True) diff --git a/src/rez/cli/selftest.py b/src/rez/cli/selftest.py index 3cbb8256d..9f406e0d0 100644 --- a/src/rez/cli/selftest.py +++ b/src/rez/cli/selftest.py @@ -10,6 +10,7 @@ import sys import inspect import argparse +import shutil from pkgutil import iter_modules try: @@ -38,6 +39,9 @@ def setup_parser(parser, completions=False): "flag shadowed pytest '-capture=no' shorthand '-s', so the long " "name must be used for disabling stdout/err capturing in pytest." ) + parser.add_argument( + "--keep-tmpdirs", action="store_true", help="Keep temporary directories." + ) # make an Action that will append the appropriate test to the "--test" arg class AddTestModuleAction(argparse.Action): @@ -59,6 +63,9 @@ def __call__(self, parser, namespace, values, option_string=None): # create argparse entry for each module's unit test for name, module in sorted(tests): + if not module.__doc__: + raise RuntimeError("Module {0!r} doesn't have a docstring. Please add one.".format(module.__file__)) + parser.add_argument( "--%s" % name, action=AddTestModuleAction, nargs=0, dest="module_tests", default=[], @@ -71,22 +78,34 @@ def command(opts, parser, extra_arg_groups=None): if opts.only_shell: os.environ["__REZ_SELFTEST_SHELL"] = opts.only_shell + if opts.keep_tmpdirs: + os.environ["REZ_KEEP_TMPDIRS"] = "1" + if not opts.module_tests and not opts.tests: module_tests = all_module_tests else: module_tests = opts.module_tests - if use_pytest: - cwd = os.getcwd() - os.chdir(tests_dir) - try: - run_pytest(module_tests, opts.tests, opts.verbose, - extra_arg_groups) - finally: - os.chdir(cwd) + repo = os.path.join(os.getcwd(), "__tests_pkg_repo") + os.makedirs(repo, exist_ok=True) + create_python_package(os.path.join(os.getcwd(), "__tests_pkg_repo")) - else: - run_unittest(module_tests, opts.tests, opts.verbose) + os.environ["__REZ_SELFTEST_PYTHON_REPO"] = repo + + try: + if use_pytest: + cwd = os.getcwd() + os.chdir(tests_dir) + try: + run_pytest(module_tests, opts.tests, opts.verbose, + extra_arg_groups) + finally: + os.chdir(cwd) + + else: + run_unittest(module_tests, opts.tests, opts.verbose) + finally: + shutil.rmtree(repo) def run_unittest(module_tests, tests, verbosity): @@ -130,3 +149,25 @@ def run_pytest(module_tests, tests, verbosity, extra_arg_groups): exitcode = main(args=argv) sys.exit(exitcode) + + +def create_python_package(repo): + from rez.package_maker import make_package + from rez.utils.lint_helper import env, system + import venv + + print("Creating python package in {0!r}".format(repo)) + + def make_root(variant, root): + venv.create(root) + + def commands(): + if system.platform == "windows": + env.PATH.prepend("{this.root}/Scripts") + else: + env.PATH.prepend("{this.root}/bin") + + with make_package("python", repo, make_root=make_root, warn_on_skip=False) as pkg: + pkg.version = ".".join(map(str, sys.version_info[:3])) + pkg.tools = ["python"] + pkg.commands = commands diff --git a/src/rez/config.py b/src/rez/config.py index a916ee834..82c8aadcc 100644 --- a/src/rez/config.py +++ b/src/rez/config.py @@ -18,6 +18,7 @@ from rez.vendor.six import six from rez.vendor.yaml.error import YAMLError from rez.backport.lru_cache import lru_cache +import rez.deprecations from contextlib import contextmanager from inspect import ismodule import os @@ -28,6 +29,26 @@ basestring = six.string_types[0] +class _Deprecation(object): + def __init__(self, removed_in, extra=None): + self.__removed_in = removed_in + self.__extra = extra or "" + + def get_message(self, name, env_var=False): + if self.__removed_in: + return ( + "config setting named {0!r} {1}is " + "deprecated and will be removed in {2}. {3}" + ).format( + name, + "(configured through the {0} environment variable) ".format(env_var) + if env_var + else "", + self.__removed_in, + self.__extra + ).strip() + + # ----------------------------------------------------------------------------- # Schema Implementations # ----------------------------------------------------------------------------- @@ -72,12 +93,30 @@ def _validate(self, data): # next, env-var value = os.getenv(self._env_var_name) if value is not None: + if self.key in _deprecated_settings: + rez.deprecations.warn( + _deprecated_settings[self.key].get_message( + self.key, env_var=self._env_var_name + ), + rez.deprecations.RezDeprecationWarning, + pre_formatted=True, + filename=self._env_var_name, + ) return self._parse_env_var(value) # next, JSON-encoded env-var varname = self._env_var_name + "_JSON" value = os.getenv(varname) if value is not None: + if self.key in _deprecated_settings: + rez.deprecations.warn( + _deprecated_settings[self.key].get_message( + self.key, env_var=varname + ), + rez.deprecations.RezDeprecationWarning, + pre_formatted=True, + filename=varname, + ) from rez.utils import json try: @@ -402,6 +441,7 @@ def _parse_env_var(self, value): "alias_back": OptionalStr, "package_preprocess_function": OptionalStrOrFunction, "package_preprocess_mode": PreprocessMode_, + "error_on_missing_variant_requires": Bool, "context_tracking_host": OptionalStr, "variant_shortlinks_dirname": OptionalStr, "build_thread_count": BuildThreadCount_, @@ -481,6 +521,30 @@ def _parse_env_var(self, value): }) +# List of settings that are deprecated and should raise +# deprecation warnings if referenced in config files. +_deprecated_settings = { + "rxt_as_yaml": _Deprecation("3.0.0"), + "warn_old_commands": _Deprecation("the future"), + "error_old_commands": _Deprecation("the future"), + # Remove in 3.0 because it's currently a no-op + "debug_old_commands": _Deprecation("3.0.0"), + # Remove in 3.0 because it's currently a no-op + "warn_commands2": _Deprecation("3.0.0"), + # Remove in 3.0 because it's currently a no-op + "error_commands2": _Deprecation("3.0.0"), + "rez_1_environment_variables": _Deprecation( + "the future", + extra="Additionally, it will become disabled by default in 3.0.0.", + ), + "rez_1_cmake_variables": _Deprecation("3.0.0"), + "disable_rez_1_compatibility": _Deprecation( + "the future", + extra="Additionally, it will become enabled by default in 3.0.0.", + ) +} + + # settings common to each plugin type _plugin_config_dict = { "release_vcs": { @@ -955,6 +1019,7 @@ def _load_config_from_filepaths(filepaths): loaders = ((".py", _load_config_py), ("", _load_config_yaml)) + root_config = get_module_root_config() for filepath in filepaths: for extension, loader in loaders: if extension: @@ -967,6 +1032,20 @@ def _load_config_from_filepaths(filepaths): continue data_ = loader(filepath_with_ext) + + if filepath != root_config: + for key in data_: + if key in _deprecated_settings: + rez.deprecations.warn( + _deprecated_settings[key].get_message( + key, + env_var=False, + ), + rez.deprecations.RezDeprecationWarning, + pre_formatted=True, + filename=filepath_with_ext, + ) + deep_update(data, data_) sourced_filepaths.append(filepath_with_ext) break @@ -980,3 +1059,9 @@ def get_module_root_config(): # singleton config = Config._create_main_config() + +if os.getenv("REZ_LOG_DEPRECATION_WARNINGS"): + # If REZ_LOG_DEPRECATION_WARNINGS is set, force all configs + # to be loaded so that we can raise warnings appropriately with all + # the commands, etc. + config.data diff --git a/src/rez/data/tests/bind/__init__.py b/src/rez/data/tests/bind/__init__.py new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/src/rez/data/tests/bind/__init__.py @@ -0,0 +1 @@ + diff --git a/src/rez/data/tests/bind/os.py b/src/rez/data/tests/bind/os.py new file mode 100644 index 000000000..be2bbdcca --- /dev/null +++ b/src/rez/data/tests/bind/os.py @@ -0,0 +1,3 @@ +""" +Custom bind module override +""" diff --git a/src/rez/data/tests/builds/packages/anti/1.0.0/package.py b/src/rez/data/tests/builds/packages/anti/1.0.0/package.py index 253997d26..40330ffa9 100644 --- a/src/rez/data/tests/builds/packages/anti/1.0.0/package.py +++ b/src/rez/data/tests/builds/packages/anti/1.0.0/package.py @@ -5,7 +5,7 @@ description = "package with anti package" -private_build_requires = ["build_util"] +private_build_requires = ["build_util", "python"] requires = ["floob", "!loco"] def commands(): diff --git a/src/rez/data/tests/builds/packages/bah/2.1/package.py b/src/rez/data/tests/builds/packages/bah/2.1/package.py index d830ac843..fef3d813b 100644 --- a/src/rez/data/tests/builds/packages/bah/2.1/package.py +++ b/src/rez/data/tests/builds/packages/bah/2.1/package.py @@ -4,7 +4,7 @@ uuid = "3c027ce6593244af947e305fc48eec96" description = "bah humbug" -private_build_requires = ["build_util"] +private_build_requires = ["build_util", "python"] variants = [ ["foo-1.0"], diff --git a/src/rez/data/tests/builds/packages/build_util/1/package.py b/src/rez/data/tests/builds/packages/build_util/1/package.py index 5d3c567b3..a44d1a7b9 100644 --- a/src/rez/data/tests/builds/packages/build_util/1/package.py +++ b/src/rez/data/tests/builds/packages/build_util/1/package.py @@ -4,6 +4,8 @@ uuid = "9982b60993af4a4d89e8372472a49d02" description = "build utilities" +private_build_requires = ["python"] + def commands(): env.PYTHONPATH.append('{root}/python') diff --git a/src/rez/data/tests/builds/packages/floob/package.py b/src/rez/data/tests/builds/packages/floob/package.py index cde376101..5495dfe2b 100644 --- a/src/rez/data/tests/builds/packages/floob/package.py +++ b/src/rez/data/tests/builds/packages/floob/package.py @@ -7,7 +7,7 @@ uuid = "156730d7122441e3a5745cc81361f49a" description = "floobtasticator" -private_build_requires = ["build_util"] +private_build_requires = ["build_util", "python"] def commands(): env.PYTHONPATH.append('{root}/python') diff --git a/src/rez/data/tests/builds/packages/foo/1.0.0/package.py b/src/rez/data/tests/builds/packages/foo/1.0.0/package.py index 6f1772423..9ced5fa48 100644 --- a/src/rez/data/tests/builds/packages/foo/1.0.0/package.py +++ b/src/rez/data/tests/builds/packages/foo/1.0.0/package.py @@ -6,7 +6,7 @@ build_requires = ["floob"] -private_build_requires = ["build_util"] +private_build_requires = ["build_util", "python"] def pre_build_commands(): env.FOO_TEST_VAR = "hello" diff --git a/src/rez/data/tests/builds/packages/foo/1.1.0/package.py b/src/rez/data/tests/builds/packages/foo/1.1.0/package.py index 9c0674238..a1b1c4e87 100644 --- a/src/rez/data/tests/builds/packages/foo/1.1.0/package.py +++ b/src/rez/data/tests/builds/packages/foo/1.1.0/package.py @@ -6,7 +6,7 @@ build_requires = ["floob"] -private_build_requires = ["build_util"] +private_build_requires = ["build_util", "python"] @include("late_utils") def commands(): diff --git a/src/rez/data/tests/release/package.yaml b/src/rez/data/tests/release/package.yaml index 9abeed385..5cf370092 100644 --- a/src/rez/data/tests/release/package.yaml +++ b/src/rez/data/tests/release/package.yaml @@ -5,4 +5,6 @@ authors: uuid: '9982b60993af4a4d89e8372472a49d02' description: 'foo type thing' +private_build_requires: ["python"] + build_command: "python {root}/build.py {install}" diff --git a/src/rez/data/tests/release/variants/package.yaml b/src/rez/data/tests/release/variants/package.yaml index db139bc09..6d9c2391f 100644 --- a/src/rez/data/tests/release/variants/package.yaml +++ b/src/rez/data/tests/release/variants/package.yaml @@ -7,4 +7,6 @@ variants: - ['spangle-1.0'] - ['spangle-1.1'] +private_build_requires: ["python"] + build_command: "python {root}/build.py {install}" diff --git a/src/rez/data/tests/solver/packages/missing_variant_requires/1/package.py b/src/rez/data/tests/solver/packages/missing_variant_requires/1/package.py new file mode 100644 index 000000000..6cf364e53 --- /dev/null +++ b/src/rez/data/tests/solver/packages/missing_variant_requires/1/package.py @@ -0,0 +1,10 @@ +name = "missing_variant_requires" +version = "1" + +def commands(): + pass + +variants = [ + ["noexist"], + ["nada"] +] diff --git a/src/rez/deprecations.py b/src/rez/deprecations.py new file mode 100644 index 000000000..831512563 --- /dev/null +++ b/src/rez/deprecations.py @@ -0,0 +1,35 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright Contributors to the Rez Project + + +import warnings + + +def warn(message, category, pre_formatted=False, stacklevel=1, filename=None, **kwargs): + """ + Wrapper around warnings.warn that allows to passa a pre-formatter + warning message. This allows to warn about things that aren't coming + from python files, like environment variables, etc. + """ + if not pre_formatted: + message = warnings.warn( + message, category=category, stacklevel=stacklevel + 1, **kwargs + ) + return + + original_formatwarning = warnings.formatwarning + if pre_formatted: + + def formatwarning(_, category, *args, **kwargs): + return "{0}{1}: {2}\n".format( + "{0}: ".format(filename) if filename else "", category.__name__, message + ) + + warnings.formatwarning = formatwarning + + warnings.warn(message, category=category, stacklevel=stacklevel + 1, **kwargs) + warnings.formatwarning = original_formatwarning + + +class RezDeprecationWarning(DeprecationWarning): + pass diff --git a/src/rez/package_bind.py b/src/rez/package_bind.py index 353ca7885..d50b1c828 100644 --- a/src/rez/package_bind.py +++ b/src/rez/package_bind.py @@ -19,10 +19,10 @@ def get_bind_modules(verbose=False): """Get available bind modules. Returns: - dict: Map of (name, filepath) listing all bind modules. + dict[str, str]: Map of (name, filepath) listing all bind modules. """ builtin_path = os.path.join(module_root_path, "bind") - searchpaths = config.bind_module_path + [builtin_path] + searchpaths = [builtin_path] + config.bind_module_path bindnames = {} for path in searchpaths: @@ -85,14 +85,14 @@ def bind_package(name, path=None, version_range=None, no_deps=False, Args: name (str): Package name. path (str): Package path to install into; local packages path if None. - version_range (`VersionRange`): If provided, only bind the software if + version_range (rez.vendor.version.version.VersionRange): If provided, only bind the software if it falls within this version range. no_deps (bool): If True, don't bind dependencies. bind_args (list of str): Command line options. quiet (bool): If True, suppress superfluous output. Returns: - List of `Variant`: The variant(s) that were installed as a result of + list[rez.packages.Variant]: The variant(s) that were installed as a result of binding this package. """ pending = set([name]) diff --git a/src/rez/package_cache.py b/src/rez/package_cache.py index 663e61330..5a4733309 100644 --- a/src/rez/package_cache.py +++ b/src/rez/package_cache.py @@ -149,7 +149,7 @@ def add_variant(self, variant, force=False): is no guarantee the resulting variant payload will be functional). Returns: - 2-tuple: + tuple: 2-tuple: - str: Path to cached payload - int: One of VARIANT_FOUND, VARIANT_CREATED, VARIANT_COPYING, VARIANT_COPY_STALLED """ @@ -205,7 +205,7 @@ def add_variant(self, variant, force=False): # Package belongs to a temp repo (this occurs when a package is # tested on pre_build/pre_release - see - # https://github.com/AcademySoftwareFoundation/rez/wiki/Package-Definition-Guide#tests) + # https://rez.readthedocs.io/en/stable/package_definition.html#tests) # if package.repository.name() == "filesystem" and \ package.repository.location.startswith(config.tmpdir + os.sep): @@ -477,7 +477,8 @@ def get_variants(self): """Get variants and their current statuses from the cache. Returns: - List of 3-tuple: + tuple: List of 3-tuple: + - `Variant`: The cached variant - str: Local cache path for variant, if determined ('' otherwise) - int: Status. One of: diff --git a/src/rez/package_copy.py b/src/rez/package_copy.py index 319c20211..80c6d0f5a 100644 --- a/src/rez/package_copy.py +++ b/src/rez/package_copy.py @@ -31,19 +31,21 @@ def copy_package(package, dest_repository, variants=None, shallow=False, """Copy a package from one package repository to another. This copies the package definition and payload. The package can also be - re-named and/or re-versioned using the `dest_name` and `dest_version` args. + re-named and/or re-versioned using the ``dest_name`` and ``dest_version`` args. The result is a dict describing which package variants were and were not copied. For example: - { - "copied": [ - (`Variant`, `Variant`) - ], - "skipped": [ - (`Variant`, `Variant`) - ] - } + .. code-block:: text + + { + "copied": [ + (`Variant`, `Variant`) + ], + "skipped": [ + (`Variant`, `Variant`) + ] + } Each 2-tuple in the 'copied' or 'skipped' list contains the source and destination variant respectively. In the 'skipped' list, the source variant @@ -51,21 +53,21 @@ def copy_package(package, dest_repository, variants=None, shallow=False, target variant that caused the source not to be copied. Skipped variants will only be present when `overwrite` is False. - Note: - Whether or not a package can be copied is determined by its 'relocatable' - attribute (see the `default_relocatable` config setting for more details). - An attempt to copy a non-relocatable package will fail. You can override - this behaviour with the `force` argument. + .. note:: + Whether or not a package can be copied is determined by its :attr:`relocatable` + attribute (see the :data:`default_relocatable` config setting for more details). + An attempt to copy a non-relocatable package will fail. You can override + this behaviour with the ``force`` argument. Args: - package (`Package`): Package to copy. - dest_repository (`PackageRepository` or str): The package repository, or + package (Package): Package to copy. + dest_repository (PackageRepository or str): The package repository, or a package repository path, to copy the package into. - variants (list of int): Indexes of variants to build, or all if None. + variants (list[int]): Indexes of variants to build, or all if None. shallow (bool): If True, symlinks of each variant's root directory are created, rather than the payload being copied. dest_name (str): If provided, copy the package to a new package name. - dest_version (str or `Version`): If provided, copy the package to a new + dest_version (str or Version): If provided, copy the package to a new version. overwrite (bool): Overwrite variants if they already exist in the destination package. In this case, the existing payload is removed @@ -81,7 +83,7 @@ def copy_package(package, dest_repository, variants=None, shallow=False, is kept intact. Note that this will have no effect if variant(s) are copied into an existing package. skip_payload (bool): If True, do not copy the package payload. - overrides (dict): See `PackageRepository.install_variant`. + overrides (dict): See :meth:`.PackageRepository.install_variant`. verbose (bool): Verbose mode. dry_run (bool): Dry run mode. Dest variants in the result will be None in this case. diff --git a/src/rez/package_filter.py b/src/rez/package_filter.py index b234c8d7d..1e29066c9 100644 --- a/src/rez/package_filter.py +++ b/src/rez/package_filter.py @@ -7,7 +7,7 @@ from rez.config import config from rez.utils.data_utils import cached_property, cached_class_property from rez.vendor.six import six -from rez.vendor.version.requirement import VersionedObject, Requirement +from rez.version import VersionedObject, Requirement from hashlib import sha1 import fnmatch import re @@ -17,14 +17,16 @@ class PackageFilterBase(object): + """Base class for package filters.""" + def excludes(self, package): """Determine if the filter excludes the given package. Args: - package (`Package`): Package to filter. + package (Package): Package to filter. Returns: - `Rule` object that excludes the package, or None if the package was + typing.Optional[Rule]: Rule object that excludes the package, or None if the package was not excluded. """ raise NotImplementedError @@ -33,7 +35,7 @@ def add_exclusion(self, rule): """Add an exclusion rule. Args: - rule (`Rule`): Rule to exclude on. + rule (Rule): Rule to exclude on. """ raise NotImplementedError @@ -41,7 +43,7 @@ def add_inclusion(self, rule): """Add an inclusion rule. Args: - rule (`Rule`): Rule to include on. + rule (Rule): Rule to include on. """ raise NotImplementedError @@ -51,21 +53,25 @@ def from_pod(cls, data): raise NotImplementedError def to_pod(self): - """Convert to POD type, suitable for storing in an rxt file.""" + """Convert to POD type, suitable for storing in an rxt file. + + Returns: + dict[str, list[str]]: + """ raise NotImplementedError def iter_packages(self, name, range_=None, paths=None): - """Same as iter_packages in packages.py, but also applies this filter. + """Same as :func:`~rez.packages.iter_packages`, but also applies this filter. Args: name (str): Name of the package, eg 'maya'. range_ (VersionRange or str): If provided, limits the versions returned - to those in `range_`. - paths (list of str, optional): paths to search for packages, defaults - to `config.packages_path`. + to those in ``range_``. + paths (typing.Optional[list[str]]): paths to search for packages, defaults + to :data:`packages_path`. Returns: - `Package` iterator. + typing.Iterator[Package]: iterator """ for package in iter_packages(name, range_, paths): if not self.excludes(package): @@ -73,6 +79,12 @@ def iter_packages(self, name, range_=None, paths=None): @property def sha1(self): + """ + SHA1 representation + + Returns: + str: + """ return sha1(str(self).encode("utf-8")).hexdigest() def __repr__(self): @@ -80,16 +92,15 @@ def __repr__(self): class PackageFilter(PackageFilterBase): - """A package filter. - + """ A package filter is a set of rules that hides some packages but leaves others visible. For example, a package filter might be used to hide all packages - whos version ends in the string '.beta'. A package filter might also be used + whos version ends in the string ``.beta``. A package filter might also be used simply to act as a blacklist, hiding some specific packages that are known to be problematic. Rules can be added as 'exclusion' or 'inclusion' rules. A package is only - excluded iff it matches one or more exclusion rules, and does not match any + excluded if it matches one or more exclusion rules, and does not match any inclusion rules. """ def __init__(self): @@ -175,6 +186,11 @@ def cost(self): @classmethod def from_pod(cls, data): + """Convert from POD types to equivalent package filter. + + Returns: + PackageFilter: + """ f = PackageFilter() for namespace, func in (("excludes", f.add_exclusion), ("includes", f.add_inclusion)): @@ -243,6 +259,8 @@ def add_exclusion(self, rule): def add_inclusion(self, rule): """ + See also: :meth:`PackageFilterBase.add_inclusion` + Note: Adding an inclusion to a filter list applies that inclusion across all filters. @@ -251,6 +269,11 @@ def add_inclusion(self, rule): f.add_inclusion(rule) def excludes(self, package): + """Returns the first rule that exlcudes ``package``, if any. + + Returns: + Rule: + """ for f in self.filters: rule = f.excludes(package) if rule: @@ -268,6 +291,11 @@ def copy(self): @classmethod def from_pod(cls, data): + """Convert from POD types to equivalent package filter. + + Returns: + PackageFilterList: + """ flist = PackageFilterList() for dict_ in data: f = PackageFilter.from_pod(dict_) @@ -291,23 +319,29 @@ def __str__(self): @cached_class_property def singleton(cls): - """Filter list as configured by rezconfig.package_filter.""" + """Filter list as configured by :data:`package_filter`. + + Returns: + PackageFilterList: + """ return cls.from_pod(config.package_filter) -# filter that does not exclude any packages +#: filter that does not exclude any packages no_filter = PackageFilterList() class Rule(object): + """Base package filter rule""" + + #: Rule name name = None - """Relative cost of rule - cheaper rules are checked first.""" def match(self, package): """Apply the rule to the package. Args: - package (`Package`): Package to filter. + package (Package): Package to filter. Returns: bool: True if the package matches the filter, False otherwise. @@ -316,7 +350,11 @@ def match(self, package): def family(self): """Returns a package family string if this rule only applies to a given - package family, otherwise None.""" + package family, otherwise None. + + Returns: + str | None: + """ return self._family def cost(self): @@ -327,13 +365,13 @@ def cost(self): def parse_rule(cls, txt): """Parse a rule from a string. - See rezconfig.package_filter for an overview of valid strings. + See :data:`package_filter` for an overview of valid strings. Args: txt (str): String to parse. Returns: - `Rule` instance. + Rule: """ types = {"glob": GlobRule, "regex": RegexRule, @@ -414,7 +452,7 @@ def __str__(self): class RegexRule(RegexRuleBase): """A rule that matches a package if its qualified name matches a regex string. - For example, the package 'foo-1.beta' would match the regex rule '.*\\.beta$'. + For example, the package ``foo-1.beta`` would match the regex rule ``.*\\.beta$``. """ name = "regex" @@ -422,7 +460,7 @@ def __init__(self, s): """Create a regex rule. Args: - s (str): Regex pattern. Eg '.*\\.beta$'. + s (str): Regex pattern. Eg ``.*\\.beta$``. """ self.txt = s self._family = self._extract_family(s) @@ -432,7 +470,7 @@ def __init__(self, s): class GlobRule(RegexRuleBase): """A rule that matches a package if its qualified name matches a glob string. - For example, the package 'foo-1.2' would match the glob rule 'foo-*'. + For example, the package ``foo-1.2`` would match the glob rule ``foo-*``. """ name = "glob" @@ -440,7 +478,7 @@ def __init__(self, s): """Create a glob rule. Args: - s (str): Glob pattern. Eg 'foo.*', '*.beta'. + s (str): Glob pattern. Eg ``foo.*``, ``*.beta``. """ self.txt = s self._family = self._extract_family(s) @@ -451,7 +489,7 @@ class RangeRule(Rule): """A rule that matches a package if that package does not conflict with a given requirement. - For example, the package 'foo-1.2' would match the requirement rule 'foo<10'. + For example, the package ``foo-1.2`` would match the requirement rule ``foo<10``. """ name = "range" @@ -480,13 +518,13 @@ class TimestampRule(Rule): given timestamp. Note: - The 'timestamp' argument used for resolves is ANDed with any package - filters - providing a filter containing timestamp rules does not override - the value of 'timestamp'. + The ``timestamp`` argument used for resolves is ANDed with any package + filters. Providing a filter containing timestamp rules does not override + the value of ``timestamp``. - Note: - Do NOT use a timestamp rule to mimic what the 'timestamp' resolve argument - does. 'timestamp' is treated differently - the memcache caching system + Warning: + Do NOT use a timestamp rule to mimic what the ``timestamp`` resolve argument + does. ``timestamp`` is treated differently - the memcache caching system is aware of it, so timestamped resolves get cached. Non-timestamped resolves also get cached, but their cache entries are invalidated more often (when new packages are released). diff --git a/src/rez/package_maker.py b/src/rez/package_maker.py index 8e786a76e..bb14c0da8 100644 --- a/src/rez/package_maker.py +++ b/src/rez/package_maker.py @@ -16,7 +16,7 @@ from rez.package_py_utils import expand_requirement from rez.vendor.schema.schema import Schema, Optional, Or, Use, And from rez.vendor.six import six -from rez.vendor.version.version import Version +from rez.version import Version from contextlib import contextmanager import os @@ -175,9 +175,9 @@ def make_package(name, path, make_base=None, make_root=None, skip_existing=True, Args: name (str): Package name. path (str): Package repository path to install package into. - make_base (callable): Function that is used to create the package + make_base (typing.Callable): Function that is used to create the package payload, if applicable. - make_root (callable): Function that is used to create the package + make_root (typing.Callable): Function that is used to create the package variant payloads, if applicable. skip_existing (bool): If True, detect if a variant already exists, and skip with a warning message if so. diff --git a/src/rez/package_maker__.py b/src/rez/package_maker__.py index 24de6fd67..27a047c65 100644 --- a/src/rez/package_maker__.py +++ b/src/rez/package_maker__.py @@ -2,11 +2,11 @@ # Copyright Contributors to the Rez Project -import warnings +import rez.deprecations from rez.package_maker import * # noqa -warnings.warn( +rez.deprecations.warn( "rez.package_maker__ is deprecated; import rez.package_maker instead", - DeprecationWarning + rez.deprecations.RezDeprecationWarning, ) diff --git a/src/rez/package_order.py b/src/rez/package_order.py index 15aa61e69..e00839fea 100644 --- a/src/rez/package_order.py +++ b/src/rez/package_order.py @@ -7,11 +7,13 @@ from rez.config import config from rez.utils.data_utils import cached_class_property -from rez.vendor.version.version import Version +from rez.version import Version class PackageOrder(object): """Package reorderer base class.""" + + #: Orderer name name = None def __init__(self): @@ -31,11 +33,12 @@ def reorder(self, iterable, key=None): Args: iterable: Iterable list of packages, or objects that contain packages. - key (callable): Callable, where key(iterable) gives a `Package`. If - None, iterable is assumed to be a list of `Package` objects. + key (typing.Callable[typing.Any, Package]): Callable, where key(iterable) + gives a :class:`~rez.packages.Package`. If None, iterable is assumed + to be a list of :class:`~rez.packages.Package` objects. Returns: - List of `iterable` type, reordered. + list: Reordered ``iterable`` """ raise NotImplementedError @@ -64,7 +67,7 @@ class NullPackageOrder(PackageOrder): This orderer is useful in cases where you want to apply some default orderer to a set of packages, but may want to explicitly NOT reorder a particular - package. You would use a `NullPackageOrder` in a `PerFamilyOrder` to do this. + package. You would use a :class:`NullPackageOrder` in a :class:`PerFamilyOrder` to do this. """ name = "no_order" @@ -81,7 +84,9 @@ def to_pod(self): """ Example (in yaml): - type: no_order + .. code-block:: yaml + + type: no_order """ return {} @@ -91,7 +96,7 @@ def from_pod(cls, data): class SortedOrder(PackageOrder): - """An orderer that sorts wrt version. + """An orderer that sorts based on :attr:`Package.version `. """ name = "sorted" @@ -116,8 +121,10 @@ def to_pod(self): """ Example (in yaml): - type: sorted - descending: true + .. code-block:: yaml + + type: sorted + descending: true """ return {"descending": self.descending} @@ -135,10 +142,10 @@ def __init__(self, order_dict, default_order=None): """Create a reorderer. Args: - order_dict (dict of (str, `PackageOrder`): Orderers to apply to + order_dict (dict[str, PackageOrder]): Orderers to apply to each package family. - default_order (`PackageOrder`): Orderer to apply to any packages - not specified in `order_dict`. + default_order (PackageOrder): Orderer to apply to any packages + not specified in ``order_dict``. """ self.order_dict = order_dict.copy() self.default_order = default_order @@ -175,17 +182,19 @@ def to_pod(self): """ Example (in yaml): - type: per_family - orderers: - - packages: ['foo', 'bah'] - type: version_split - first_version: '4.0.5' - - packages: ['python'] - type: sorted - descending: false - default_order: - type: sorted - descending: true + .. code-block:: yaml + + type: per_family + orderers: + - packages: ['foo', 'bah'] + type: version_split + first_version: '4.0.5' + - packages: ['python'] + type: sorted + descending: false + default_order: + type: sorted + descending: true """ orderers = {} packages = {} @@ -234,7 +243,7 @@ class VersionSplitPackageOrder(PackageOrder): """Orders package versions <= a given version first. For example, given the versions [5, 4, 3, 2, 1], an orderer initialized - with version=3 would give the order [3, 2, 1, 5, 4]. + with ``version=3`` would give the order [3, 2, 1, 5, 4]. """ name = "version_split" @@ -242,7 +251,7 @@ def __init__(self, first_version): """Create a reorderer. Args: - first_version (`Version`): Start with versions <= this value. + first_version (Version): Start with versions <= this value. """ self.first_version = first_version @@ -281,8 +290,10 @@ def to_pod(self): """ Example (in yaml): - type: version_split - first_version: "3.0.0" + .. code-block:: yaml + + type: version_split + first_version: "3.0.0" """ return dict(first_version=str(self.first_version)) @@ -294,37 +305,41 @@ def from_pod(cls, data): class TimestampPackageOrder(PackageOrder): """A timestamp order function. - Given a time T, this orderer returns packages released before T, in descending - order, followed by those released after. If `rank` is non-zero, version + Given a time ``T``, this orderer returns packages released before ``T``, in descending + order, followed by those released after. If ``rank`` is non-zero, version changes at that rank and above are allowed over the timestamp. For example, consider the common case where we want to prioritize packages - released before T, except for newer patches. Consider the following package - versions, and time T: - - 2.2.1 - 2.2.0 - 2.1.1 - 2.1.0 - 2.0.6 - 2.0.5 - <-- T - 2.0.0 - 1.9.0 - - A timestamp orderer set to rank=3 (patch versions) will attempt to consume + released before ``T``, except for newer patches. Consider the following package + versions, and time ``T``: + + .. code-block:: text + + 2.2.1 + 2.2.0 + 2.1.1 + 2.1.0 + 2.0.6 + 2.0.5 + <-- T + 2.0.0 + 1.9.0 + + A timestamp orderer set to ``rank=3`` (patch versions) will attempt to consume the packages in the following order: - 2.0.6 - 2.0.5 - 2.0.0 - 1.9.0 - 2.1.1 - 2.1.0 - 2.2.1 - 2.2.0 + .. code-block:: text + + 2.0.6 + 2.0.5 + 2.0.0 + 1.9.0 + 2.1.1 + 2.1.0 + 2.2.1 + 2.2.0 - Notice that packages before T are preferred, followed by newer versions. + Notice that packages before ``T`` are preferred, followed by newer versions. Newer versions are consumed in ascending order, except within rank (this is why 2.1.1 is consumed before 2.1.0). """ @@ -421,9 +436,11 @@ def to_pod(self): """ Example (in yaml): - type: soft_timestamp - timestamp: 1234567 - rank: 3 + .. code-block:: yaml + + type: soft_timestamp + timestamp: 1234567 + rank: 3 """ return dict(timestamp=self.timestamp, rank=self.rank) @@ -478,6 +495,14 @@ def from_pod(data): def register_orderer(cls): + """Register an orderer + + Args: + cls (type[PackageOrder]): Package orderer class to register. + + returns: + bool: True if successfully registered, else False. + """ if isclass(cls) and issubclass(cls, PackageOrder) and \ hasattr(cls, "name") and cls.name: _orderers[cls.name] = cls diff --git a/src/rez/package_py_utils.py b/src/rez/package_py_utils.py index d07b45f2f..23117bcdb 100644 --- a/src/rez/package_py_utils.py +++ b/src/rez/package_py_utils.py @@ -20,20 +20,20 @@ def expand_requirement(request, paths=None): - """Expands a requirement string like 'python-2.*', 'foo-2.*+<*', etc. + """Expands a requirement string like ``python-2.*``, ``foo-2.*+<*``, etc. Wildcards are expanded to the latest version that matches. There is also a - special wildcard '**' that will expand to the full version, but it cannot - be used in combination with '*'. + special wildcard ``**`` that will expand to the full version, but it cannot + be used in combination with ``*``. - Wildcards MUST placehold a whole version token, not partial - while 'foo-2.*' - is valid, 'foo-2.v*' is not. + Wildcards MUST placehold a whole version token, not partial - while ``foo-2.*`` + is valid, ``foo-2.v*`` is not. - Wildcards MUST appear at the end of version numbers - while 'foo-1.*.*' is - valid, 'foo-1.*.0' is not. + Wildcards MUST appear at the end of version numbers - while ``foo-1.*.*`` is + valid, ``foo-1.*.0`` is not. It is possible that an expansion will result in an invalid request string - (such as 'foo-2+<2'). The appropriate exception will be raised if this + (such as ``foo-2+<2``). The appropriate exception will be raised if this happens. Examples: @@ -46,9 +46,9 @@ def expand_requirement(request, paths=None): python<3.0.5 Args: - request (str): Request to expand, eg 'python-2.*' - paths (list of str, optional): paths to search for package families, - defaults to `config.packages_path`. + request (str): Request to expand, eg ``python-2.*`` + paths (typing.Optional[list[str]]): paths to search for package families, + defaults to :data:`packages_path`. Returns: str: Expanded request string. @@ -56,8 +56,8 @@ def expand_requirement(request, paths=None): if '*' not in request: return request - from rez.vendor.version.version import VersionRange - from rez.vendor.version.requirement import Requirement + from rez.version import VersionRange + from rez.version import Requirement from rez.packages import get_latest_package from uuid import uuid4 @@ -159,17 +159,24 @@ def expand_requires(*requests): ["boost-1.55"] Args: - requests (list of str): Requirements to expand. Each value may have + requests (list[str]): Requirements to expand. Each value may have trailing wildcards. Returns: - List of str: Expanded requirements. + list[str]: Expanded requirements. """ return [expand_requirement(x) for x in requests] def exec_command(attr, cmd): - """Runs a subproc to calculate a package attribute. + """Runs a subprocess to calculate a package attribute. + + Args: + attr (str): Package attribute + cmd (list[str]): Command to run + + Returns: + tuple(str): Returns a tuple of (stdout, stderr). """ import subprocess @@ -189,7 +196,7 @@ def exec_python(attr, src, executable="python"): Args: attr (str): Name of package attribute being created. - src (list of str): Python code to execute, will be converted into + src (list[str]): Python code to execute, will be converted into semicolon-delimited single line of code. Returns: @@ -227,11 +234,11 @@ def find_site_python(module_name, paths=None): Args: module_name (str): Target python module. - paths (list of str, optional): paths to search for packages, - defaults to `config.packages_path`. + paths (typing.Optional[list[str]]): paths to search for packages, + defaults to :data:`packages_path`. Returns: - `Package`: Native python package containing the named module. + Package: Native python package containing the named module. """ from rez.packages import iter_packages import subprocess diff --git a/src/rez/package_remove.py b/src/rez/package_remove.py index cbd35676a..51192f36f 100644 --- a/src/rez/package_remove.py +++ b/src/rez/package_remove.py @@ -3,7 +3,7 @@ from rez.package_repository import package_repository_manager -from rez.vendor.version.version import Version +from rez.version import Version from rez.utils.logging_ import print_info from rez.vendor.six import six from rez.config import config @@ -57,7 +57,7 @@ def remove_packages_ignored_since(days, paths=None, dry_run=False, verbose=False Args: days (int): Remove packages ignored >= this many days - paths (list of str, optional): Paths to search for packages, defaults + paths (typing.Optional[list[str]]): Paths to search for packages, defaults to `config.packages_path`. dry_run: Dry run mode verbose (bool): Verbose mode diff --git a/src/rez/package_repository.py b/src/rez/package_repository.py index 67bba6049..2c7be5ab2 100644 --- a/src/rez/package_repository.py +++ b/src/rez/package_repository.py @@ -108,7 +108,7 @@ def uid(self): database address + index, and so on. Returns: - hashable value: Value that uniquely identifies this repository. + tuple[str, str]: Value that uniquely identifies this repository. """ return self._uid() diff --git a/src/rez/package_resources.py b/src/rez/package_resources.py index 364932457..53c566b83 100644 --- a/src/rez/package_resources.py +++ b/src/rez/package_resources.py @@ -12,7 +12,7 @@ from rez.utils.formatting import PackageRequest from rez.exceptions import PackageMetadataError, ResourceError from rez.config import config, Config, create_config -from rez.vendor.version.version import Version +from rez.version import Version from rez.vendor.schema.schema import Schema, SchemaError, Optional, Or, And, Use from rez.vendor.six import six @@ -270,13 +270,9 @@ def late_bound(schema): class PackageRepositoryResource(Resource): """Base class for all package-related resources. - - Attributes: - schema_error (`Exception`): Type of exception to throw on bad data. - repository_type (str): Type of package repository associated with this - resource type. """ schema_error = PackageMetadataError + #: Type of package repository associated with this resource type. repository_type = None @classmethod diff --git a/src/rez/package_resources_.py b/src/rez/package_resources_.py index 170b8e99e..78057ba80 100644 --- a/src/rez/package_resources_.py +++ b/src/rez/package_resources_.py @@ -2,11 +2,11 @@ # Copyright Contributors to the Rez Project -import warnings +import rez.deprecations from rez.package_resources import * # noqa -warnings.warn( +rez.deprecations.warn( "rez.package_resources_ is deprecated; import rez.package_resources instead", - DeprecationWarning + rez.deprecations.RezDeprecationWarning, ) diff --git a/src/rez/package_search.py b/src/rez/package_search.py index f7a25d1a4..26b7f60dc 100644 --- a/src/rez/package_search.py +++ b/src/rez/package_search.py @@ -23,7 +23,7 @@ from rez.config import config -from rez.vendor.version.requirement import Requirement +from rez.version import Requirement def get_reverse_dependency_tree(package_name, depth=None, paths=None, @@ -46,7 +46,8 @@ def get_reverse_dependency_tree(package_name, depth=None, paths=None, private_build_requires. Returns: - A 2-tuple: + tuple: A 2-tuple: + - (list of list of str): Lists of package names, where each list is a single depth in the tree. The first list is always [`package_name`]. - `pygraph.digraph` object, where nodes are package names, and @@ -219,7 +220,8 @@ def iter_resources(self, resources_request=None): are supported. If None, returns all matching resource types. Returns: - 2-tuple: + tuple: 2-tuple: + - str: resource type (family, package, variant); - Iterator of `ResourceSearchResult`: Matching resources. Will be in alphabetical order if families, and version ascending for @@ -234,7 +236,8 @@ def search(self, resources_request=None): are supported. If None, returns all matching resource types. Returns: - 2-tuple: + tuple: 2-tuple: + - str: resource type (family, package, variant); - List of `ResourceSearchResult`: Matching resources. Will be in alphabetical order if families, and version ascending for @@ -382,7 +385,7 @@ def format_search_results(self, search_results): search_results (list of `ResourceSearchResult`): Search to format. Returns: - List of 2-tuple: Text and color to print in. + tuple: List of 2-tuple: Text and color to print in. """ formatted_lines = [] diff --git a/src/rez/package_serialise.py b/src/rez/package_serialise.py index a962caace..8d1f702c0 100644 --- a/src/rez/package_serialise.py +++ b/src/rez/package_serialise.py @@ -7,7 +7,7 @@ from rez.serialise import FileFormat from rez.package_resources import help_schema, late_bound from rez.vendor.schema.schema import Schema, Optional, And, Or, Use -from rez.vendor.version.version import Version +from rez.version import Version from rez.utils.schema import extensible_schema_dict from rez.utils.sourcecode import SourceCode from rez.utils.formatting import PackageRequest, indent, \ @@ -121,7 +121,7 @@ def dump_package_data(data, buf, format_=FileFormat.py, skip_attributes=None): Args: data (dict): Data source - must conform to `package_serialise_schema`. - buf (file-like object): Destination stream. + buf (typing.IO): Destination stream. format_ (`FileFormat`): Format to dump data in. skip_attributes (list of str): List of attributes to not print. """ diff --git a/src/rez/package_test.py b/src/rez/package_test.py index b1d000797..89b0c977e 100644 --- a/src/rez/package_test.py +++ b/src/rez/package_test.py @@ -10,8 +10,8 @@ from rez.utils.colorize import heading, Printer from rez.utils.logging_ import print_info, print_warning, print_error from rez.vendor.six import six -from rez.vendor.version.requirement import Requirement, RequirementList -from pipes import quote +from rez.version import Requirement, RequirementList +from rez.utils.py23 import quote import time import sys import os @@ -27,14 +27,16 @@ class PackageTestRunner(object): An example tests entry in a package.py might look like this: - tests = { - "unit": "python -m unittest -s {root}/tests", - "CI": { - "command": "python {root}/ci_tests/main.py", - "requires": ["maya-2017"], - "replace": True - } - } + .. code-block:: python + + tests = { + "unit": "python -m unittest -s {root}/tests", + "CI": { + "command": "python {root}/ci_tests/main.py", + "requires": ["maya-2017"], + "replace": True + } + } By default tests are run in an environment containing the current package. @@ -42,8 +44,8 @@ class PackageTestRunner(object): command. If a dict, the "command" string is the command, and the "requires" list is added to the test env. - Command strings automatically expand references such as '{root}', much - as happens in a *commands* function. + Command strings automatically expand references such as ``{root}``, much + as happens in a :data:`commands` function. Commands can also be a list - in this case, the test process is launched directly, rather than interpreted via a shell. @@ -55,23 +57,23 @@ def __init__(self, package_request, use_current_env=False, """Create a package tester. Args: - package_request (str or `PackageRequest`): The package to test. + package_request (str or PackageRequest): The package to test. use_current_env (bool): If True, run the test directly in the current rez-resolved environment, if there is one, and if it contains packages that meet the test's requirements. - extra_package_requests (list of str or `PackageRequest`): Extra + extra_package_requests (list[str] or PackageRequest): Extra requests, these are appended to the test environment. package_paths: List of paths to search for pkgs, defaults to - config.packages_path. - stdout (file-like object): Defaults to sys.stdout. - stderr (file-like object): Defaults to sys.stderr. + :data:`packages_path`. + stdout (typing.IO): Defaults to :data:`sys.stdout`. + stderr (typing.IO): Defaults to :data:`sys.stderr`. verbose (int): Verbose mode (valid values: 0, 1, 2) dry_run (bool): If True, do everything except actually run tests. - cumulative_test_results (`PackageTestResults`): If supplied, test + cumulative_test_results (PackageTestResults): If supplied, test run results can be stored across multiple runners. - context_kwargs: Extra arguments which are passed to the - `ResolvedContext` instances used to run the tests within. - Ignored if `use_current_env` is True. + context_kwargs (dict[typing.Any, typing.Any]): Extra arguments which are passed to the + :class:`~rez.resolved_context.ResolvedContext` instances used to run the tests within. + Ignored if ``use_current_env`` is True. """ self.package_request = package_request self.use_current_env = use_current_env @@ -105,7 +107,7 @@ def get_package(self): """Get the target package. Returns: - `Package`: Package to run tests on. + Package: Package to run tests on. """ if self.package is not None: return self.package @@ -616,7 +618,7 @@ def _get_context(self, requires, quiet=False): def _get_target_variants(self, test_name): """ If the test is not variant-specific, then attempt to find the 'preferred' - variant (as per setting 'variant_select_mode'). Otherwise, just run tests + variant (as per setting :data:`variant_select_mode`). Otherwise, just run tests over all variants. """ package = self.get_package() @@ -655,9 +657,9 @@ def _get_target_variants(self, test_name): class PackageTestResults(object): - """Contains results of running tests with a `PackageTestRunner`. + """Contains results of running tests with a :class:`PackageTestRunner`. - Use this class (and pass it to the `PackageTestRunner` constructor) if you + Use this class (and pass it to the :class:`PackageTestRunner` constructor) if you need to gather test run results from separate runners, and display them in a single table. """ diff --git a/src/rez/packages.py b/src/rez/packages.py index 2cac8f24a..b9078688b 100644 --- a/src/rez/packages.py +++ b/src/rez/packages.py @@ -14,8 +14,8 @@ from rez.utils.schema import schema_keys from rez.utils.resources import ResourceHandle, ResourceWrapper from rez.exceptions import PackageFamilyNotFoundError, ResourceError -from rez.vendor.version.version import Version, VersionRange -from rez.vendor.version.requirement import VersionedObject +from rez.version import Version, VersionRange +from rez.version import VersionedObject from rez.vendor.six import six from rez.serialise import FileFormat from rez.config import config @@ -117,7 +117,7 @@ def print_info(self, buf=None, format_=FileFormat.yaml, """Print the contents of the package. Args: - buf (file-like object): Stream to write to. + buf (typing.IO): Stream to write to. format_ (`FileFormat`): Format to write in. skip_attributes (list of str): List of attributes to not print. include_release (bool): If True, include release-related attributes, @@ -530,7 +530,7 @@ def iter_package_families(paths=None): families. Args: - paths (list of str, optional): paths to search for package families, + paths (typing.Optional[list[str]]): paths to search for package families, defaults to `config.packages_path`. Returns: @@ -553,7 +553,7 @@ def iter_packages(name, range_=None, paths=None): name (str): Name of the package, eg 'maya'. range_ (VersionRange or str): If provided, limits the versions returned to those in `range_`. - paths (list of str, optional): paths to search for packages, defaults + paths (typing.Optional[list[str]]): paths to search for packages, defaults to `config.packages_path`. Returns: @@ -584,7 +584,7 @@ def get_package(name, version, paths=None): Args: name (str): Name of the package, eg 'maya'. version (Version or str): Version of the package, eg '1.0.0' - paths (list of str, optional): paths to search for package, defaults + paths (typing.Optional[list[str]]): paths to search for package, defaults to `config.packages_path`. Returns: @@ -665,7 +665,7 @@ def get_package_from_string(txt, paths=None): Args: txt (str): String such as 'foo', 'bah-1.3'. - paths (list of str, optional): paths to search for package, defaults + paths (typing.Optional[list[str]]): paths to search for package, defaults to `config.packages_path`. Returns: @@ -914,7 +914,7 @@ def get_latest_package(name, range_=None, paths=None, error=False): Args: name (str): Package name. range_ (`VersionRange`): Version range to search within. - paths (list of str, optional): paths to search for package families, + paths (typing.Optional[list[str]]): paths to search for package families, defaults to `config.packages_path`. error (bool): If True, raise an error if no package is found. @@ -937,7 +937,7 @@ def get_latest_package_from_string(txt, paths=None, error=False): Args: txt (str): Request, eg 'foo-1.2+' - paths (list of str, optional): paths to search for packages, defaults + paths (typing.Optional[list[str]]): paths to search for packages, defaults to `config.packages_path`. error (bool): If True, raise an error if no package is found. diff --git a/src/rez/packages_.py b/src/rez/packages_.py index e95146d56..5f4d30d60 100644 --- a/src/rez/packages_.py +++ b/src/rez/packages_.py @@ -2,11 +2,11 @@ # Copyright Contributors to the Rez Project -import warnings +import rez.deprecations from rez.packages import * # noqa -warnings.warn( +rez.deprecations.warn( "rez.packages_ is deprecated; import rez.packages instead", - DeprecationWarning + rez.deprecations.RezDeprecationWarning, ) diff --git a/src/rez/pip.py b/src/rez/pip.py index be29d175e..ece540535 100644 --- a/src/rez/pip.py +++ b/src/rez/pip.py @@ -5,7 +5,7 @@ from __future__ import print_function, absolute_import from rez.packages import get_latest_package -from rez.vendor.version.version import Version +from rez.version import Version from rez.vendor.distlib.database import DistributionPath from rez.vendor.enum.enum import Enum from rez.vendor.packaging.version import Version as PackagingVersion @@ -23,7 +23,7 @@ from rez.config import config import os -from pipes import quote +from rez.utils.py23 import quote from pprint import pformat import re import shutil diff --git a/src/rez/plugin_managers.py b/src/rez/plugin_managers.py index a5a16bc36..2f46fd2b2 100644 --- a/src/rez/plugin_managers.py +++ b/src/rez/plugin_managers.py @@ -383,7 +383,7 @@ def get_failed_plugins(self, plugin_type): """Return a list of plugins for the given type that failed to load. Returns: - List of 2-tuples: + tuple: List of 2-tuples: name (str): Name of the plugin. reason (str): Error message. """ diff --git a/src/rez/release_vcs.py b/src/rez/release_vcs.py index 66acbec5e..31c749636 100644 --- a/src/rez/release_vcs.py +++ b/src/rez/release_vcs.py @@ -8,7 +8,7 @@ from rez.utils.execution import Popen from rez.utils.logging_ import print_debug from rez.utils.filesystem import walk_up_dirs -from pipes import quote +from rez.utils.py23 import quote import subprocess @@ -162,7 +162,7 @@ def get_changelog(self, previous_revision=None, max_revisions=None): Args: previous_revision: The revision to give the changelog since. If - None, give the entire changelog. + None, give the entire changelog. Returns: Changelog, as a string. diff --git a/src/rez/resolved_context.py b/src/rez/resolved_context.py index 79a76299f..2c4a4fe49 100644 --- a/src/rez/resolved_context.py +++ b/src/rez/resolved_context.py @@ -36,13 +36,14 @@ read_graph_from_string from rez.utils.resolve_graph import failure_detail_from_graph from rez.vendor.six import six -from rez.vendor.version.version import VersionRange -from rez.vendor.version.requirement import Requirement +from rez.version import VersionRange +from rez.version import Requirement from rez.vendor.enum import Enum from rez.vendor import yaml from rez.utils import json from rez.utils.yaml import dump_yaml from rez.utils.platform_ import platform_ +import rez.deprecations from contextlib import contextmanager from functools import wraps @@ -60,23 +61,32 @@ class RezToolsVisibility(Enum): """Determines if/how rez cli tools are added back to PATH within a - resolved environment.""" - never = 0 # Don't expose rez in resolved env - append = 1 # Append to PATH in resolved env - prepend = 2 # Prepend to PATH in resolved env + resolved environment. + """ + #: Don't expose rez in resolved env + never = 0 + #: Append to PATH in resolved env + append = 1 + #: Prepend to PATH in resolved env + prepend = 2 class SuiteVisibility(Enum): """Defines what suites on $PATH stay visible when a new rez environment is - resolved.""" - never = 0 # Don't attempt to keep any suites visible in a new env - always = 1 # Keep suites visible in any new env - parent = 2 # Keep only the parent suite of a tool visible - parent_priority = 3 # Keep all suites visible and the parent takes precedence + resolved. + """ + #: Don't attempt to keep any suites visible in a new env + never = 0 + #: Keep suites visible in any new env + always = 1 + #: Keep only the parent suite of a tool visible + parent = 2 + #: Keep all suites visible and the parent takes precedence + parent_priority = 3 class PatchLock(Enum): - """ Enum to represent the 'lock type' used when patching context objects. + """Enum to represent the 'lock type' used when patching context objects. """ no_lock = ("No locking", -1) lock_2 = ("Minor version updates only (X.*)", 1) @@ -107,7 +117,7 @@ def get_lock_request(name, version, patch_lock, weak=True): patch_lock (PatchLock): Lock type to apply. Returns: - `PackageRequest` object, or None if there is no equivalent request. + typing.Optional[PackageRequest]: PackageRequest object, or None if there is no equivalent request. """ ch = '~' if weak else '' if patch_lock == PatchLock.lock: @@ -168,41 +178,40 @@ def __init__(self, package_requests, verbosity=0, timestamp=None, """Perform a package resolve, and store the result. Args: - package_requests: List of strings or PackageRequest objects - representing the request. - verbosity: Verbosity level. One of [0,1,2]. - timestamp: Ignore packages released after this epoch time. Packages + package_requests (list[typing.Union[str, PackageRequest]]): request + verbosity (int): Verbosity level. One of [0,1,2]. + timestamp (float): Ignore packages released after this epoch time. Packages released at exactly this time will not be ignored. - building: True if we're resolving for a build. - caching: If True, cache(s) may be used to speed the resolve. If - False, caches will not be used. If None, config.resolve_caching + building (bool): True if we're resolving for a build. + caching (bool): If True, cache(s) may be used to speed the resolve. If + False, caches will not be used. If None, :data:`resolve_caching` is used. - package_paths: List of paths to search for pkgs, defaults to - config.packages_path. - package_filter (`PackageFilterBase`): Filter used to exclude certain - packages. Defaults to settings from config.package_filter. Use - `package_filter.no_filter` to remove all filtering. - package_orderers (list of `PackageOrder`): Custom package ordering. - Defaults to settings from config.package_orderers. - add_implicit_packages: If True, the implicit package list defined - by config.implicit_packages is appended to the request. + package_paths (list[str]): List of paths to search for pkgs, defaults to + :data:`packages_path`. + package_filter (PackageFilterList): Filter used to exclude certain + packages. Defaults to settings from :data:`package_filter`. Use + :data:`rez.package_filter.no_filter` to remove all filtering. + package_orderers (list[PackageOrder]): Custom package ordering. + Defaults to settings from :data:`package_orderers`. + add_implicit_packages (bool): If True, the implicit package list defined + by :data:`implicit_packages` is appended to the request. max_fails (int): Abort the resolve if the number of failed steps is greater or equal to this number. If -1, does not abort. time_limit (int): Abort the resolve if it takes longer than this many seconds. If -1, there is no time limit. - callback: See `Solver`. + callback: See :class:`.Solver`. package_load_callback: If not None, this callable will be called prior to each package being loaded. It is passed a single - `Package` object. - buf (file-like object): Where to print verbose output to, defaults + :class:`.Package` object. + buf (typing.IO): Where to print verbose output to, defaults to stdout. suppress_passive (bool): If True, don't print debugging info that has had no effect on the solve. This argument only has an - effect if `verbosity` > 2. + effect if ``verbosity`` > 2. print_stats (bool): If True, print advanced solver stats at the end. package_caching (bool|None): If True, apply package caching settings as per the config. If None, enable as determined by config - setting 'package_cache_during_build'. + setting :data:`package_cache_during_build`. """ self.load_path = None @@ -357,7 +366,7 @@ def status(self): """Return the current status of the context. Returns: - ResolverStatus. + ResolverStatus: """ return self.status_ @@ -369,7 +378,7 @@ def requested_packages(self, include_implicit=False): to the result. Returns: - List of `PackageRequest` objects. + list[PackageRequest]: """ if include_implicit: return self._package_requests + self.implicit_packages @@ -381,7 +390,7 @@ def resolved_packages(self): """Get packages in the resolve. Returns: - List of `Variant` objects, or None if the resolve failed. + typing.Optional[list[Variant]]: Resolved variant objects, or None if the resolve failed. """ return self._resolved_packages @@ -390,7 +399,7 @@ def resolved_ephemerals(self): """Get non-conflict ephemerals in the resolve. Returns: - List of `Requirement` objects, or None if the resolve failed. + typing.Optional[list[Requirement]]: Requirement objects, or None if the resolve failed. """ return self._resolved_ephemerals @@ -457,11 +466,11 @@ def retargeted(self, package_paths, package_names=None, skip_missing=False): package_names (list of str): Only retarget these packages. If None, retarget all packages. skip_missing (bool): If True, skip retargeting of variants that - cannot be found in `package_paths`. By default, a - `PackageNotFoundError` is raised. + cannot be found in ``package_paths``. By default, a + :exc:`.PackageNotFoundError` is raised. Returns: - ResolvecContext`: The retargeted context. + ResolvedContext: The retargeted context. """ retargeted_variants = [] @@ -529,9 +538,9 @@ def get_patched_request(self, package_requests=None, in the order that they appear in `package_requests`. Args: - package_requests (list of str or list of `PackageRequest`): + package_requests (list[typing.Union[str, PackageRequest]): Overriding requests. - package_subtractions (list of str): Any original request with a + package_subtractions (list[str]): Any original request with a package name in this list is removed, before the new requests are added. strict (bool): If True, the current context's resolve is used as the @@ -540,12 +549,12 @@ def get_patched_request(self, package_requests=None, and further - for example, rank=3 means that only version patch numbers are allowed to increase, major and minor versions will not change. This is only applied to packages that have not been - explicitly overridden in `package_requests`. If rank <= 1, or - `strict` is True, rank is ignored. + explicitly overridden in ``package_requests``. If rank <= 1, or + ``strict`` is True, rank is ignored. Returns: - List of `PackageRequest` objects that can be used to construct a - new `ResolvedContext` object. + list[PackageRequest]: PackageRequests objects that can be used to construct a + new :class:`ResolvedContext` object. """ # assemble source request if strict: @@ -651,6 +660,12 @@ def write_to_buffer(self, buf): doc = self.to_dict() if config.rxt_as_yaml: + rez.deprecations.warn( + "Writing the RXT file using the YAML format is deprecated. " + "Both this functionality and the rxt_as_yaml setting will " + "be removed in rez 3.0.0", + rez.deprecations.RezDeprecationWarning, + ) content = dump_yaml(doc) else: content = json.dumps(doc, indent=4, separators=(",", ": "), @@ -663,7 +678,7 @@ def get_current(cls): """Get the context for the current env, if there is one. Returns: - `ResolvedContext`: Current context, or None if not in a resolved env. + ResolvedContext: Current context, or None if not in a resolved env. """ filepath = os.getenv("REZ_RXT_FILE") if not filepath or not os.path.exists(filepath): @@ -717,15 +732,16 @@ def get_resolve_diff(self, other): of a package is ignored. Returns: - A dict containing: + dict: A dict containing: + - 'newer_packages': A dict containing items: - - package name (str); - - List of `Package` objects. These are the packages up to and - including the newer package in `self`, in ascending order. + - package name (str); + - List of `Package` objects. These are the packages up to and + including the newer package in `self`, in ascending order. - 'older_packages': A dict containing: - - package name (str); - - List of `Package` objects. These are the packages down to and - including the older package in `self`, in descending order. + - package name (str); + - List of `Package` objects. These are the packages down to and + including the older package in `self`, in descending order. - 'added_packages': Set of `Package` objects present in `self` but not in `other`; - 'removed_packages': Set of `Package` objects present in `other`, @@ -795,7 +811,7 @@ def print_info(self, buf=sys.stdout, verbosity=0, source_order=False, """Prints a message summarising the contents of the resolved context. Args: - buf (file-like object): Where to print this info to. + buf (typing.IO): Where to print this info to. verbosity (bool): Verbose mode. source_order (bool): If True, print resolved packages in the order they are sourced, rather than alphabetical order. @@ -984,8 +1000,9 @@ def print_resolve_diff(self, other, heading=None): """Print the difference between the resolve of two contexts. Args: - other (`ResolvedContext`): Context to compare to. + other (ResolvedContext): Context to compare to. heading: One of: + - None: Do not display a heading; - True: Display the filename of each context as a heading, if both contexts have a filepath; @@ -1107,9 +1124,12 @@ def validate(self): def get_environ(self, parent_environ=None): """Get the environ dict resulting from interpreting this context. - @param parent_environ Environment to interpret the context within, - defaults to os.environ if None. - @returns The environment dict generated by this context, when + Args: + parent_environ: Environment to interpret the context within, + defaults to os.environ if None. + + Returns: + The environment dict generated by this context, when interpreted in a python rex interpreter. """ interp = Python(target_environ={}, passive=True) @@ -1127,7 +1147,7 @@ def get_key(self, key, request_only=False): packages that were also present in the request. Returns: - Dict of {pkg-name: (variant, value)}. + Dict of ``{pkg-name: (variant, value)}``. """ values = {} requested_names = [x.name for x in self._package_requests @@ -1150,7 +1170,7 @@ def get_tools(self, request_only=False): that were also present in the request. Returns: - Dict of {pkg-name: (variant, [tools])}. + Dict of ``{pkg-name: (variant, [tools])}``. """ return self.get_key("tools", request_only=request_only) @@ -1184,7 +1204,7 @@ def get_conflicting_tools(self, request_only=False): that were also present in the request. Returns: - Dict of {tool-name: set([Variant])}. + Dict of ``{tool-name: set([Variant])}``. """ from collections import defaultdict @@ -1206,7 +1226,7 @@ def get_shell_code(self, shell=None, parent_environ=None, style=OutputStyle.file type is used. parent_environ (dict): Environment to interpret the context within, defaults to os.environ if None. - style (): Style to format shell code in. + style (OutputStyle): Style to format shell code in. """ executor = self._create_executor(interpreter=create_shell(shell), parent_environ=parent_environ) @@ -1223,7 +1243,7 @@ def get_actions(self, parent_environ=None): context. This is provided mainly for testing purposes. Args: - parent_environ Environment to interpret the context within, + parent_environ: Environment to interpret the context within, defaults to os.environ if None. Returns: @@ -1325,7 +1345,7 @@ def execute_rex_code(self, code, filename=None, shell=None, Popen_args: args to pass to the shell process object constructor. Returns: - `subprocess.Popen` object for the shell process. + subprocess.Popen: Subprocess object for the shell process. """ def _actions_callback(executor): executor.execute_code(code, filename=filename) @@ -1383,10 +1403,11 @@ def execute_shell(self, shell=None, parent_environ=None, rcfile=None, Popen_args: args to pass to the shell process object constructor. Returns: - If blocking: A 3-tuple of (returncode, stdout, stderr). + If blocking, a 3-tuple of (returncode, stdout, stderr). Note that if you want to get anything other than None for stdout and/or stderr, you need to give stdout=PIPE and/or stderr=PIPE. - If non-blocking - A subprocess.Popen object for the shell process. + + If non-blocking, a subprocess.Popen object for the shell process. """ sh = create_shell(shell) @@ -2088,7 +2109,8 @@ def normalized(path): commands.set_package(pkg) try: - executor.execute_code(commands, isolate=True) + with executor.reset_globals(): + executor.execute_code(commands) except exc_type as e: exc = e diff --git a/src/rez/resolver.py b/src/rez/resolver.py index 4acc624de..5fb999525 100644 --- a/src/rez/resolver.py +++ b/src/rez/resolver.py @@ -10,7 +10,7 @@ from rez.utils.logging_ import log_duration from rez.config import config from rez.vendor.enum import Enum -from rez.vendor.version.requirement import Requirement +from rez.version import Requirement from contextlib import contextmanager from hashlib import sha1 diff --git a/src/rez/rex.py b/src/rez/rex.py index aa25707d6..9ad5b0600 100644 --- a/src/rez/rex.py +++ b/src/rez/rex.py @@ -12,6 +12,7 @@ from contextlib import contextmanager from string import Formatter +import rez.deprecations from rez.system import system from rez.config import config from rez.exceptions import RexError, RexUndefinedVariableError, \ @@ -987,7 +988,7 @@ def formatted(self, func): """Return the string with non-literal parts formatted. Args: - func (callable): Callable that translates a string into a + func (typing.Callable): Callable that translates a string into a formatted string. Returns: @@ -1510,6 +1511,12 @@ def _apply(): exec_namespace=self.globals) if isolate: + rez.deprecations.warn( + "the 'isolate' argument is deprecated and will be removed in 3.0.0. " + "Use the reset_globals method/context manager instead.", + category=rez.deprecations.RezDeprecationWarning, + stacklevel=2, + ) with self.reset_globals(): _apply() else: diff --git a/src/rez/rex_bindings.py b/src/rez/rex_bindings.py index 2aea8387f..f1496deb3 100644 --- a/src/rez/rex_bindings.py +++ b/src/rez/rex_bindings.py @@ -11,8 +11,8 @@ unnecessary data from Rex, and provide APIs that will not change. """ from rez.vendor.six import six -from rez.vendor.version.version import VersionRange -from rez.vendor.version.requirement import Requirement +from rez.version import VersionRange +from rez.version import Requirement basestring = six.string_types[0] @@ -38,6 +38,8 @@ def __getattr__(self, attr): class VersionBinding(Binding): """Binds a version.Version object. + Examples: + >>> v = VersionBinding(Version("1.2.3alpha")) >>> v.major 1 @@ -212,11 +214,14 @@ def get_range(self, name, default=None): class EphemeralsBinding(RO_MappingBinding): """Binds a list of resolved ephemeral packages. - Note that the leading '.' is implied when referring to ephemerals. Eg: + Note: + The leading '.' is implied when referring to ephemerals. Eg: + + .. code-block:: python - # in package.py - def commands(): - if "foo.cli" in ephemerals: # will match '.foo.cli-*' request + # in package.py + def commands(): + if "foo.cli" in ephemerals: # will match '.foo.cli-*' request """ def __init__(self, ephemerals): doc = dict( @@ -244,24 +249,28 @@ def intersects(obj, range_): Examples: - # in package.py - def commands(): - # test a request - if intersects(request.maya, '2019+'): - info('requested maya allows >=2019.*') + .. code-block:: python + + # in package.py + def commands(): + # test a request + if intersects(request.maya, '2019+'): + info('requested maya allows >=2019.*') + + # tests if a resolved version intersects with given range + if intersects(resolve.maya, '2019+') + ... - # tests if a resolved version intersects with given range - if intersects(resolve.maya, '2019+') - ... + # same as above + if intersects(resolve.maya.version, '2019+') + ... - # same as above - if intersects(resolve.maya.version, '2019+') - ... + .. code-block:: python - # disable my cli tools if .foo.cli-0 was specified - def commands(): - if intersects(ephemerals.get('foo.cli', '1'), '1'): - env.PATH.append('{root}/bin') + # disable my cli tools if .foo.cli-0 was specified + def commands(): + if intersects(ephemerals.get('foo.cli', '1'), '1'): + env.PATH.append('{root}/bin') Args: obj (VariantBinding or str): Object to test, either a diff --git a/src/rez/rezconfig.py b/src/rez/rezconfig.py index b04dfcab6..25b91d71e 100644 --- a/src/rez/rezconfig.py +++ b/src/rez/rezconfig.py @@ -40,7 +40,7 @@ (based on Python's os.path.sep). So for Linux paths, / should be used. On Windows \ (unescaped) should be used. -Note: The comments in this file are extracted and turned into Wiki content. Pay +Note: The comments in this file are extracted and turned into documentation. Pay attention to the comment formatting and follow the existing style closely. """ @@ -499,6 +499,17 @@ # this value is False. allow_unversioned_packages = True +# Defines whether a resolve should immediately fail if any variants have a required package that can't be found. +# This can be useful to disable if you have packages that aren't available to all users. +# It is enabled by default. If a variant has requires that cannot be found , it will error immediately rather than +# trying the other variants. +# If disabled, it will try other variants before giving up. +# +# .. warning:: +# Memcached isn't tested with scenarios where you expect users to have access to different sets of packages. +# It expects that every user can access the same set of packages, which may cause incorrect resolves +# when this option is disabled. +error_on_missing_variant_requires = True ############################################################################### # Environment Resolution @@ -708,7 +719,6 @@ # - "override": Package's preprocess function completely overrides the global preprocess. package_preprocess_mode = "override" - ############################################################################### # Context Tracking ############################################################################### @@ -1077,16 +1087,30 @@ # written in JSON, which is a LOT faster. You would only set to true for # backwards compatibility reasons. Note that rez will detect either format on # rxt file load. +# +# .. deprecated:: 2.114.0 +# The ability to store RXT files using the YAML format will be removed in 3.0.0. rxt_as_yaml = False # Warn or disallow when a package is found to contain old rez-1-style commands. +# +# .. deprecated:: 2.114.0 +# Will be removed in a future release. warn_old_commands = True # See :data:`warn_old_commands`. +# +# .. deprecated:: 2.114.0 +# Will be removed in a future release. error_old_commands = False # Print old commands and their converted rex equivalent. Note that this can # cause very verbose output. +# +# This currently has no effect. +# +# .. deprecated:: 2.114.0 +# Will be removed in rez 3.0.0. debug_old_commands = False # Warn or disallow an extra commands entry called "commands2". This is provided @@ -1095,16 +1119,29 @@ # instead of "commands". Unlike "commands", "commands2" only allows new rex- # style commands. Once you have fully deprecated Rez-1, you should stop using # "commands2". -# TODO: DEPRECATE +# +# This currently has no effect. +# +# .. deprecated:: 2.114.0 +# Will be removed in rez 3.0.0. warn_commands2 = False # See :data:`warn_commands2`. +# +# This currently has no effect. +# +# .. deprecated:: 2.114.0 +# Will be removed in rez 3.0.0. error_commands2 = False # If True, Rez will continue to generate the given environment variables in # resolved environments, even though their use has been deprecated in Rez-2. # The variables in question, and their Rez-2 equivalent (if any) are: # +# .. deprecated:: 2.114.0 +# Will be removed in a future release. Additionally, the default will change +# from disabled to enabled in rez 3.0.0. +# # ================== ========================== # REZ-1 REZ-2 # ================== ========================== @@ -1122,12 +1159,18 @@ # release time, even though their use has been deprecated in Rez-2. The # variables in question, and their Rez-2 equivalent (if any) are: # +# .. versionchanged:: 2.114.0 +# Now disabled by default. +# +# .. deprecated:: 2.114.0 +# This will be removed in 3.0.0. +# # ======= ======================== # REZ-1 REZ-2 # ======= ======================== # CENTRAL :envvar:`REZ_BUILD_TYPE` # ======= ======================== -rez_1_cmake_variables = True +rez_1_cmake_variables = False # If True, override all compatibility-related settings so that Rez-1 support is # deprecated. This means that: @@ -1137,8 +1180,15 @@ # * :data:`rez_1_environment_variables` will be set to False. # * :data:`rez_1_cmake_variables` will be set to False. # -# You should aim to do this - it will mean your packages are more strictly +# You should aim to do this. It will mean your packages are more strictly # validated, and you can more easily use future versions of Rez. +# +# .. versionchanged:: 2.114.0 +# Now enabled by default. +# +# .. deprecated:: 2.114.0 +# Will be removed in a future release. Additionally, the default will change +# from disabled to enabled in rez 3.0.0. disable_rez_1_compatibility = False diff --git a/src/rez/serialise.py b/src/rez/serialise.py index ae0efabfd..5be6f8052 100644 --- a/src/rez/serialise.py +++ b/src/rez/serialise.py @@ -111,17 +111,17 @@ def load_from_file(filepath, format_=FileFormat.py, update_data_callback=None, """Load data from a file. Note: - Any functions from a .py file will be converted to `SourceCode` objects. + Any functions from a .py file will be converted to :class:`.SourceCode` objects. Args: filepath (str): File to load. - format_ (`FileFormat`): Format of file contents. - update_data_callback (callable): Used to change data before it is + format_ (FileFormat): Format of file contents. + update_data_callback (typing.Callable): Used to change data before it is returned or cached. disable_memcache (bool): If True, don't r/w to memcache. Returns: - dict. + dict: """ filepath = os.path.realpath(filepath) cache_filepath = file_cache.get(filepath) @@ -224,10 +224,10 @@ def load_py(stream, filepath=None): """Load python-formatted data from a stream. Args: - stream (file-like object). + stream (typing.IO): Returns: - dict. + dict: """ with add_sys_paths(config.package_definition_build_python_paths): return _load_py(stream, filepath=filepath) @@ -273,7 +273,7 @@ def _load_py(stream, filepath=None): class EarlyThis(object): - """The 'this' object for @early bound functions. + """The ``this`` object for ``@early`` bound functions. Just exposes raw package data as object attributes. """ @@ -298,10 +298,11 @@ def process_python_objects(data, filepath=None): """Replace certain values in the given package data dict. Does things like: - * evaluates @early decorated functions, and replaces with return value; - * converts functions into `SourceCode` instances so they can be serialized + + * evaluates ``@early`` decorated functions, and replaces with return value; + * converts functions into :class:`.SourceCode` instances so they can be serialized out to installed packages, and evaluated later; - * strips some values (modules, __-leading variables) that are never to be + * strips some values (modules, ``__``-leading variables) that are never to be part of installed packages. Returns: @@ -401,10 +402,10 @@ def load_yaml(stream, **kwargs): """Load yaml-formatted data from a stream. Args: - stream (file-like object). + stream (typing.IO): Returns: - dict. + dict: """ # if there's an error parsing the yaml, and you pass yaml.load a string, # it will print lines of context, but will print "" instead of a @@ -430,10 +431,10 @@ def load_txt(stream, **kwargs): """Load text data from a stream. Args: - stream (file-like object). + stream (typing.IO): Returns: - string. + str: """ content = stream.read() return content diff --git a/src/rez/shells.py b/src/rez/shells.py index 2c2fe5955..01bdff58e 100644 --- a/src/rez/shells.py +++ b/src/rez/shells.py @@ -17,7 +17,7 @@ from rez.vendor.six import six import os import os.path -import pipes +from rez.utils.py23 import quote basestring = six.string_types[0] @@ -32,7 +32,7 @@ def get_shell_types(): """Returns the available shell types: bash, tcsh etc. Returns: - List of str: Shells. + list[str]: Shells. """ from rez.plugin_managers import plugin_manager return list(plugin_manager.get_plugins('shell')) @@ -42,7 +42,7 @@ def get_shell_class(shell=None): """Get the plugin class associated with the given or current shell. Returns: - class: Plugin class for shell. + type[Shell]: Plugin class for shell. """ if not shell: shell = config.default_shell @@ -58,7 +58,7 @@ def create_shell(shell=None, **kwargs): """Returns a Shell of the given or current type. Returns: - `Shell`: Instance of given shell. + Shell: Instance of given shell. """ if not shell: shell = config.default_shell @@ -128,7 +128,9 @@ def startup_capabilities(cls, rcfile=False, norc=False, stdin=False, """ Given a set of options related to shell startup, return the actual options that will be applied. - @returns 4-tuple representing applied value of each option. + + Returns: + tuple: 4-tuple representing applied value of each option. """ raise NotImplementedError @@ -252,8 +254,9 @@ def spawn_shell(self, context_file, tmpdir, rcfile=None, norc=False, pre_command=None, add_rez=True, package_commands_sourced_first=None, **Popen_args): """Spawn a possibly interactive subshell. + Args: - context:_file File that must be sourced in the new shell, this + context_file: File that must be sourced in the new shell, this configures the Rez environment. tmpdir: Tempfiles, if needed, should be created within this path. rcfile: Custom startup script. @@ -279,7 +282,7 @@ def spawn_shell(self, context_file, tmpdir, rcfile=None, norc=False, popen_args: args to pass to the shell process object constructor. Returns: - A subprocess.Popen object representing the shell process. + subprocess.Popen: A subprocess.Popen object representing the shell process. """ raise NotImplementedError @@ -408,8 +411,8 @@ def normalize_path(self, path): class UnixShell(Shell): - """ - A base class for common *nix shells, such as bash and tcsh. + r""" + A base class for common \*nix shells, such as bash and tcsh. """ rcfile_arg = None norc_arg = None @@ -440,6 +443,7 @@ def supports_stdin(cls): def get_startup_sequence(cls, rcfile, norc, stdin, command): """ Return a dict containing: + - 'stdin': resulting stdin setting. - 'command': resulting command setting. - 'do_rcfile': True if a file should be sourced directly. @@ -595,7 +599,7 @@ def _create_ex(): try: p = Popen(cmd, env=env, **Popen_args) except Exception as e: - cmd_str = ' '.join(map(pipes.quote, cmd)) + cmd_str = ' '.join(map(quote, cmd)) raise RezSystemError("Error running command:\n%s\n%s" % (cmd_str, str(e))) return p diff --git a/src/rez/solver.py b/src/rez/solver.py index 2ec0f35fd..9990e6457 100644 --- a/src/rez/solver.py +++ b/src/rez/solver.py @@ -23,9 +23,8 @@ from rez.vendor.pygraph.algorithms.accessibility import accessibility from rez.exceptions import PackageNotFoundError, ResolveError, \ PackageFamilyNotFoundError, RezSystemError -from rez.vendor.version.version import VersionRange -from rez.vendor.version.requirement import VersionedObject, Requirement, \ - RequirementList +from rez.version import VersionRange +from rez.version import VersionedObject, Requirement, RequirementList from rez.vendor.enum import Enum from contextlib import contextmanager from itertools import product, chain @@ -1383,10 +1382,16 @@ def _create_phase(status=None): # Raise with more info when match found searched = "; ".join(self.solver.package_paths) requested = ", ".join(requesters) + + fail_message = ("package family not found: {}, was required by: {} (searched: {})" + .format(req.name, requested, searched)) + # TODO: Test with memcached to see if this can cause any conflicting behaviour + # where a package may show as missing/available inadvertently + if not config.error_on_missing_variant_requires: + print(fail_message, file=sys.stderr) + return _create_phase(SolverStatus.failed) raise PackageFamilyNotFoundError( - "package family not found: %s, " - "was required by: %s (searched: %s)" - % (req.name, requested, searched)) + fail_message) scopes.append(scope) if self.pr: @@ -2212,6 +2217,7 @@ def failure_reason(self, failure_index=None): failure_index: Index of the fail to return the graph for (can be negative). If None, the most appropriate failure is chosen according to these rules: + - If the fail is cyclic, the most recent fail (the one containing the cycle) is used; - If a callback has caused a failure, the most recent fail is used; @@ -2398,7 +2404,10 @@ def _get_failed_phase(self, index=None): except IndexError: raise IndexError("failure index out of range") - fail_description = phase.failure_reason.description() + if phase.failure_reason is None: + fail_description = "Solver failed with unknown reason." + else: + fail_description = phase.failure_reason.description() if prepend_abort_reason and self.abort_reason: fail_description = "%s:\n%s" % (self.abort_reason, fail_description) diff --git a/src/rez/suite.py b/src/rez/suite.py index 0ee4a4fac..897c974d9 100644 --- a/src/rez/suite.py +++ b/src/rez/suite.py @@ -39,6 +39,7 @@ class Suite(object): context's tools override tools from other contexts. There are several ways to avoid tool name clashes: + - Hide a tool. This removes it from the suite even if it does not clash; - Prefix/suffix a context. When you do this, all the tools in the context have the prefix/suffix applied; @@ -300,7 +301,8 @@ def get_tools(self): """Get the tools exposed by this suite. Returns: - A dict, keyed by aliased tool name, with dict entries: + dict: A dict, keyed by aliased tool name, with dict entries: + - tool_name (str): The original, non-aliased name of the tool; - tool_alias (str): Aliased tool name (same as key); - context_name (str): Name of the context containing the tool; @@ -354,7 +356,8 @@ def get_hidden_tools(self): Hidden tools are those that have been explicitly hidden via `hide_tool`. Returns: - A list of dicts, where each dict contains: + list[dict]: A list of dicts, where each dict contains: + - tool_name (str): The original, non-aliased name of the tool; - tool_alias (str): Aliased tool name (same as key); - context_name (str): Name of the context containing the tool; diff --git a/src/rez/tests/test_bind.py b/src/rez/tests/test_bind.py new file mode 100644 index 000000000..b457412da --- /dev/null +++ b/src/rez/tests/test_bind.py @@ -0,0 +1,49 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright Contributors to the Rez Project + + +""" +test package_bind module +""" +import os +import unittest +from rez import package_bind +from rez.tests.util import TestBase + + +class TestPackageBind(TestBase): + def test_get_bind_modules(self): + """Test get_bind_modules returns the expected modules""" + self.assertEqual( + sorted(package_bind.get_bind_modules().keys()), + [ + "PyQt", + "PySide", + "arch", + "cmake", + "gcc", + "hello_world", + "os", + "pip", + "platform", + "python", + "rez", + "rezgui", + "setuptools", + "sip", + ] + ) + + def test_os_module_override(self): + """Test that bind_module_path can override built-in bind modules""" + self.update_settings({ + "bind_module_path": [self.data_path("bind")] + }) + + os_module_path = os.path.join(self.data_path("bind"), "os.py") + os_bind_module = package_bind.find_bind_module("os") + self.assertEqual(os_bind_module, os_module_path) + + +if __name__ == '__main__': + unittest.main() diff --git a/src/rez/tests/test_build.py b/src/rez/tests/test_build.py index 4c2a51e67..43a0ccb1f 100644 --- a/src/rez/tests/test_build.py +++ b/src/rez/tests/test_build.py @@ -178,6 +178,8 @@ def test_build_whack(self, shell): def test_builds(self, shell): """Test an interdependent set of builds. """ + self.inject_python_repo() + self._test_build_build_util() self._test_build_floob() self._test_build_foo() @@ -189,6 +191,8 @@ def test_builds(self, shell): def test_builds_anti(self, shell): """Test we can build packages that contain anti packages """ + self.inject_python_repo() + self._test_build_build_util() self._test_build_floob() self._test_build_anti() @@ -207,6 +211,7 @@ def test_build_cmake(self): self._test_build_translate_lib() self._test_build_sup_world() + @unittest.skipIf(platform_.name == "windows", "Skipping because make and GCC are not common on Windows") @program_dependent("make", "g++") def test_build_custom(self): """Test a make-based package that uses the custom_build attribute.""" diff --git a/src/rez/tests/test_commands.py b/src/rez/tests/test_commands.py index 86cf85f2e..df545bc9a 100644 --- a/src/rez/tests/test_commands.py +++ b/src/rez/tests/test_commands.py @@ -5,7 +5,7 @@ """ test package commands """ -from rez.vendor.version.requirement import VersionedObject +from rez.version import VersionedObject from rez.rex import Comment, EnvAction, Shebang, Setenv, Alias, Appendenv from rez.resolved_context import ResolvedContext from rez.utils.filesystem import canonical_path diff --git a/src/rez/tests/test_completion.py b/src/rez/tests/test_completion.py index 87be442fd..ef8c390c6 100644 --- a/src/rez/tests/test_completion.py +++ b/src/rez/tests/test_completion.py @@ -52,7 +52,7 @@ def _eq(prefix, expected_completions): _eq("", ["bahish", "nada", "nopy", "pybah", "pydad", "pyfoo", "pymum", "pyodd", "pyson", "pysplit", "python", "pyvariants", "test_variant_split_start", "test_variant_split_mid1", - "test_variant_split_mid2", "test_variant_split_end"]) + "test_variant_split_mid2", "test_variant_split_end", "missing_variant_requires"]) _eq("py", ["pybah", "pydad", "pyfoo", "pymum", "pyodd", "pyson", "pysplit", "python", "pyvariants"]) _eq("pys", ["pyson", "pysplit"]) diff --git a/src/rez/tests/test_config.py b/src/rez/tests/test_config.py index 2e9bd092b..fd1c2de05 100644 --- a/src/rez/tests/test_config.py +++ b/src/rez/tests/test_config.py @@ -6,16 +6,22 @@ test configuration settings """ import unittest -from rez.tests.util import TestBase +from rez.tests.util import TestBase, TempdirMixin, restore_os_environ from rez.exceptions import ConfigurationError -from rez.config import Config, get_module_root_config, _replace_config +from rez.config import Config, get_module_root_config, _replace_config, _Deprecation from rez.system import system from rez.utils.data_utils import RO_AttrDictWrapper from rez.packages import get_developer_package from rez.vendor.six import six +from rez.deprecations import RezDeprecationWarning import os +import sys import os.path import subprocess +import functools +import shutil +if sys.version_info[:3] >= (3, 3): + import unittest.mock class TestConfig(TestBase): @@ -301,5 +307,95 @@ def test_8(self): raise +@unittest.skipIf(sys.version_info[0] < 3, "Skip on python 2") +class TestDeprecations(TestBase, TempdirMixin): + @classmethod + def setUpClass(cls): + cls.settings = {} + TempdirMixin.setUpClass() + + @classmethod + def tearDownClass(cls): + TempdirMixin.tearDownClass() + + def test_deprecation_from_user_config(self): + user_home = os.path.join(self.root, "user_home") + self.addCleanup(functools.partial(shutil.rmtree, user_home)) + + os.makedirs(user_home) + + with open(os.path.join(user_home, ".rezconfig.py"), "w") as fd: + fd.write("packages_path = ['/tmp/asd']") + + fake_deprecated_settings = { + "packages_path": _Deprecation("0.0.0"), + } + + with unittest.mock.patch( + "rez.config._deprecated_settings", + fake_deprecated_settings + ): + with restore_os_environ(): + os.environ["HOME"] = user_home + # On Windows, os.path.expanduser will read HOME and then USERPROFILE with Python 3.7. + # https://docs.python.org/3.7/library/os.path.html#os.path.expanduser + # Also on Windows but for Python 3.8+, it will look for USERPROFILE and then HOME. + # https://docs.python.org/3.8/library/os.path.html#os.path.expanduser + os.environ["USERPROFILE"] = user_home + config = Config._create_main_config() + with self.assertWarns(RezDeprecationWarning) as warn: + _ = config.data + # Assert just to ensure the test was set up properly. + self.assertEqual(config.data["packages_path"], ["/tmp/asd"]) + + self.assertEqual( + str(warn.warning), + "config setting named 'packages_path' is deprecated and will be removed in 0.0.0.", + ) + + def test_deprecation_from_env_var(self): + fake_deprecated_settings = { + "packages_path": _Deprecation("0.0.0"), + } + + with unittest.mock.patch( + "rez.config._deprecated_settings", + fake_deprecated_settings + ): + with restore_os_environ(): + # Test with non-json env var + os.environ["REZ_PACKAGES_PATH"] = "/tmp/asd2" + os.environ["REZ_DISABLE_HOME_CONFIG"] = "1" + config = Config._create_main_config() + with self.assertWarns(RezDeprecationWarning) as warn: + _ = config.data + # Assert just to ensure the test was set up properly. + self.assertEqual(config.data["packages_path"], ["/tmp/asd2"]) + + self.assertEqual( + str(warn.warning), + "config setting named 'packages_path' (configured through the " + "REZ_PACKAGES_PATH environment variable) is deprecated and will " + "be removed in 0.0.0.", + ) + + with restore_os_environ(): + # Test with json env var + os.environ["REZ_PACKAGES_PATH_JSON"] = '["/tmp/asd2"]' + os.environ["REZ_DISABLE_HOME_CONFIG"] = "1" + config = Config._create_main_config() + with self.assertWarns(RezDeprecationWarning) as warn: + _ = config.data + # Assert just to ensure the test was set up properly. + self.assertEqual(config.data["packages_path"], ["/tmp/asd2"]) + + self.assertEqual( + str(warn.warning), + "config setting named 'packages_path' (configured through the " + "REZ_PACKAGES_PATH_JSON environment variable) is deprecated and will " + "be removed in 0.0.0.", + ) + + if __name__ == "__main__": unittest.main() diff --git a/src/rez/tests/test_context.py b/src/rez/tests/test_context.py index 2ef577479..ee742a8df 100644 --- a/src/rez/tests/test_context.py +++ b/src/rez/tests/test_context.py @@ -73,7 +73,8 @@ def test_execute_command(self): def test_execute_command_environ(self): """Test that execute_command properly sets environ dict.""" - r = ResolvedContext(["hello_world"]) + self.inject_python_repo() + r = ResolvedContext(["hello_world", "python"]) self._test_execute_command_environ(r) def _test_execute_command_environ(self, r): @@ -110,14 +111,15 @@ def test_serialize(self): def test_retarget(self): """Test that a retargeted context behaves identically.""" + self.inject_python_repo() # make a copy of the pkg repo packages_path2 = os.path.join(self.root, "packages2") shutil.copytree(self.packages_path, packages_path2) # create a context, retarget to pkg repo copy - r = ResolvedContext(["hello_world"]) - r2 = r.retargeted(package_paths=[packages_path2]) + r = ResolvedContext(["hello_world", "python"]) + r2 = r.retargeted(package_paths=[packages_path2, os.environ["__REZ_SELFTEST_PYTHON_REPO"]]) # check the pkg we contain is in the copied pkg repo variant = r2.resolved_packages[0] @@ -128,6 +130,8 @@ def test_retarget(self): def test_bundled(self): """Test that a bundled context behaves identically.""" + self.inject_python_repo() + def _test_bundle(path): # load the bundled context r2 = ResolvedContext.load(os.path.join(path, "context.rxt")) @@ -145,7 +149,7 @@ def _test_bundle(path): bundle_path = os.path.join(self.root, "bundle") # create context and bundle it - r = ResolvedContext(["hello_world"]) + r = ResolvedContext(["hello_world", "python"]) bundle_context( context=r, dest_dir=bundle_path, @@ -172,7 +176,7 @@ def _test_bundle(path): os.mkdir(hard_path) os.symlink(hard_path, bundles_path) - r = ResolvedContext(["hello_world"]) + r = ResolvedContext(["hello_world", "python"]) bundle_context( context=r, dest_dir=bundle_path3, diff --git a/src/rez/tests/test_copy_package.py b/src/rez/tests/test_copy_package.py index 95315a772..9f33db8d8 100644 --- a/src/rez/tests/test_copy_package.py +++ b/src/rez/tests/test_copy_package.py @@ -16,7 +16,7 @@ from rez.resolved_context import ResolvedContext from rez.packages import get_latest_package from rez.package_copy import copy_package -from rez.vendor.version.version import VersionRange +from rez.version import VersionRange from rez.tests.util import TestBase, TempdirMixin @@ -51,6 +51,7 @@ def tearDownClass(cls): def setup_once(self): # build packages used by this test + self.inject_python_repo() self._build_package("build_util", "1") self._build_package("floob") self._build_package("foo", "1.0.0") diff --git a/src/rez/tests/test_package_filter.py b/src/rez/tests/test_package_filter.py index 296eb891b..66d7481b4 100644 --- a/src/rez/tests/test_package_filter.py +++ b/src/rez/tests/test_package_filter.py @@ -7,7 +7,7 @@ """ from rez.tests.util import TestBase from rez.packages import iter_packages -from rez.vendor.version.requirement import Requirement +from rez.version import Requirement from rez.package_filter import PackageFilter, PackageFilterList, GlobRule, \ RegexRule, RangeRule, TimestampRule diff --git a/src/rez/tests/test_packages.py b/src/rez/tests/test_packages.py index b94a0dde9..55f5254c6 100644 --- a/src/rez/tests/test_packages.py +++ b/src/rez/tests/test_packages.py @@ -20,8 +20,8 @@ from rez.utils.formatting import PackageRequest from rez.utils.sourcecode import SourceCode import unittest -from rez.vendor.version.version import Version -from rez.vendor.version.util import VersionError +from rez.version import Version +from rez.version import VersionError from rez.utils.filesystem import canonical_path import shutil import os.path @@ -57,7 +57,8 @@ 'late_binding-1.0', 'timestamped-1.0.5', 'timestamped-1.0.6', 'timestamped-1.1.0', 'timestamped-1.1.1', 'timestamped-1.2.0', 'timestamped-2.0.0', 'timestamped-2.1.0', 'timestamped-2.1.5', - 'multi-1.0', 'multi-1.1', 'multi-1.2', 'multi-2.0' + 'multi-1.0', 'multi-1.1', 'multi-1.2', 'multi-2.0', + 'missing_variant_requires-1' ]) diff --git a/src/rez/tests/test_packages_order.py b/src/rez/tests/test_packages_order.py index f90181eef..702963f5f 100644 --- a/src/rez/tests/test_packages_order.py +++ b/src/rez/tests/test_packages_order.py @@ -12,7 +12,7 @@ TimestampPackageOrder, SortedOrder, PackageOrderList, from_pod from rez.packages import iter_packages from rez.tests.util import TestBase, TempdirMixin -from rez.vendor.version.version import Version +from rez.version import Version class _BaseTestPackagesOrder(TestBase, TempdirMixin): diff --git a/src/rez/tests/test_pip_utils.py b/src/rez/tests/test_pip_utils.py index c5911fe06..c42bf6766 100644 --- a/src/rez/tests/test_pip_utils.py +++ b/src/rez/tests/test_pip_utils.py @@ -9,8 +9,8 @@ import rez.vendor.packaging.version import rez.vendor.distlib.database -from rez.vendor.version.version import VersionRange -from rez.vendor.version.requirement import Requirement +from rez.version import VersionRange +from rez.version import Requirement from rez.vendor.packaging.requirements import Requirement as packaging_Requirement from rez.vendor.packaging.specifiers import SpecifierSet from rez.exceptions import PackageRequestError diff --git a/src/rez/tests/test_release.py b/src/rez/tests/test_release.py index 91b0989b4..fab55aed7 100644 --- a/src/rez/tests/test_release.py +++ b/src/rez/tests/test_release.py @@ -106,9 +106,10 @@ def _standardize_variants(variants): @install_dependent() def test_1(self, shell): """Basic release.""" - self._setup_release() + self.inject_python_repo() # release should fail because release path does not exist + self._setup_release() builder = self._create_builder() with self.assertRaises(ReleaseError): builder.release() @@ -169,6 +170,8 @@ def test_1(self, shell): def test_2_variant_add(self, shell): """Test variant installation on release """ + self.inject_python_repo() + orig_src_path = self.src_path self.src_path = os.path.join(self.src_path, "variants") try: diff --git a/src/rez/tests/test_rex.py b/src/rez/tests/test_rex.py index 6ae2601a7..32a37d495 100644 --- a/src/rez/tests/test_rex.py +++ b/src/rez/tests/test_rex.py @@ -13,8 +13,8 @@ from rez.exceptions import RexError, RexUndefinedVariableError from rez.config import config import unittest -from rez.vendor.version.version import Version -from rez.vendor.version.requirement import Requirement +from rez.version import Version +from rez.version import Requirement from rez.tests.util import TestBase from rez.utils.backcompat import convert_old_commands from rez.package_repository import package_repository_manager diff --git a/src/rez/tests/test_shells.py b/src/rez/tests/test_shells.py index f1ef78d39..77b2fcb62 100644 --- a/src/rez/tests/test_shells.py +++ b/src/rez/tests/test_shells.py @@ -86,7 +86,10 @@ def test_aaa_shell_presence(self): shells = set(x for x in shells if x) if not shells: - self.skipTest("Not ensuring presence of shells from explicit list") + self.skipTest( + "Not ensuring presence of shells from explicit list because " + "$_REZ_ENSURE_TEST_SHELLS is either empty or not defined" + ) return # check for missing shells @@ -188,6 +191,71 @@ def test_command(self, shell): stdout=subprocess.PIPE, text=True) self.assertEqual(_stdout(p), "Hello Rez World!") + @per_available_shell() + def test_per_available_shell_decorator(self, shell): + """ + Test that the "per_available_shell" decorator correctly sets the default shell + and that ResolvedContext.execute_shell will use the default shell as expected. + """ + # Based on: + # * https://stackoverflow.com/a/3327022 + # * https://stackoverflow.com/a/61469226 + # * https://stackoverflow.com/a/27776822 + data = { + "bash": { + "command": "echo $BASH", + "assert": lambda x: self.assertEqual(os.path.basename(x), "bash"), + }, + "gitbash": { + "command": "uname -s", + "assert": lambda x: self.assertRegex(x, r"^(MINGW|CYGWIN|MSYS).*$") + }, + "csh": { + "command": "echo $shell", + # csh should usually resolve to csh, but on macOS, it will resolve to tcsh, + # at least on GitHub Actions Hosted runners. + "assert": lambda x: self.assertEqual( + os.path.basename(x), "csh" if system.platform != "osx" else "tcsh" + ), + }, + "tcsh": { + "command": "echo $shell", + "assert": lambda x: self.assertEqual(os.path.basename(x), "tcsh"), + }, + "sh": { + # This is a hack. $0 doesn't work when run through execute_shell, + # but will work when running "rez-env --shell sh -c 'echo $0'" + # Output result to /dev/null because we don't want the content to affect the test, + # we just want to test is the variable exists. + "command": "set -o nounset; echo $REZ_STORED_PROMPT_SH > /dev/null", + "assert": lambda x: self.assertEqual(x, ""), + }, + "zsh": { + "command": "echo $ZSH_NAME", + "assert": lambda x: self.assertEqual(os.path.basename(x), "zsh"), + }, + "powershell": { + "command": "echo $PSVersionTable.PSEdition", + "assert": lambda x: self.assertEqual(x, "Desktop"), + }, + "pwsh": { + "command": "echo $PSVersionTable.PSEdition", + "assert": lambda x: self.assertEqual(x, "Core"), + }, + "cmd": { + "command": "dir 2>&1 *`|echo CMD", + "assert": lambda x: self.assertEqual(x, "CMD"), + }, + } + + if shell not in data: + self.fail("Please add support for {0!r} in the test".format(shell)) + + r = self._create_context([]) + p = r.execute_shell(command=data[shell]["command"], + stdout=subprocess.PIPE, text=True) + data[shell]["assert"](_stdout(p).strip()) + @per_available_shell() def test_command_returncode(self, shell): sh = create_shell(shell) diff --git a/src/rez/tests/test_solver.py b/src/rez/tests/test_solver.py index 6422bcf07..1244dede5 100644 --- a/src/rez/tests/test_solver.py +++ b/src/rez/tests/test_solver.py @@ -7,7 +7,8 @@ """ from __future__ import print_function -from rez.vendor.version.requirement import Requirement +import rez.exceptions +from rez.version import Requirement from rez.solver import Solver, Cycle, SolverStatus from rez.config import config import unittest @@ -214,6 +215,7 @@ def test_07(self): def test_08(self): """Cyclic failures.""" + def _test(*pkgs): s = self._fail(*pkgs) self.assertTrue(isinstance(s.failure_reason(), Cycle)) @@ -248,6 +250,14 @@ def test_11_variant_splitting(self): "test_variant_split_mid2-2.0[0]", "test_variant_split_start-1.0[1]"]) + def test_12_missing_variant_requires(self): + config.override("error_on_missing_variant_requires", True) + with self.assertRaises(rez.exceptions.PackageFamilyNotFoundError): + self._solve(["missing_variant_requires"], []) + + config.override("error_on_missing_variant_requires", False) + self._solve(["missing_variant_requires"], ["nada[]", "missing_variant_requires-1[1]"]) + if __name__ == '__main__': unittest.main() diff --git a/src/rez/tests/test_utils_resolve_graph.py b/src/rez/tests/test_utils_resolve_graph.py new file mode 100644 index 000000000..83f726787 --- /dev/null +++ b/src/rez/tests/test_utils_resolve_graph.py @@ -0,0 +1,49 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright Contributors to the Rez Project + + +""" +unit tests for 'utils.resolve_graph' module +""" +from rez.tests.util import TestBase +from rez.utils import resolve_graph +import rez.utils.graph_utils +import unittest + + +class TestResolveGraph(TestBase): + def test_conflict_graph_with_cycle(self): + """ Tests creating a test digraph which contains a cycle foo-1.0.0 => bar-0.0.1 => !foo-1.0.0 + Note that the solver doesn't detect this as a cycle. See #1568 + """ + + g = ''' + digraph g { + _1 [label="foo-1", fontsize="10", fillcolor="#FFFFAA", style="filled,dashed"]; + _2 [label="bar", fontsize="10", fillcolor="#FFFFAA", style="filled,dashed"]; + _6 [label="foo-1.0.0[]", fontsize="10", fillcolor="#AAFFAA", style="filled"]; + _7 [label="bar-0.0.1[]", fontsize="10", fillcolor="#AAFFAA", style="filled"]; + _8 [label="!foo-1.0.0", fontsize="10", fillcolor="#F6F6F6", style="filled,dashed"]; + _1 -> _6 [arrowsize="0.5"]; + _2 -> _7 [arrowsize="0.5"]; + _6 -> _2 [arrowsize="0.5"]; + _7 -> _8 [arrowsize="0.5"]; + _8 -> _6 [arrowsize="1", style="bold", color="red", fontcolor="red", label=CONFLICT]; + } + ''' + graph = rez.utils.graph_utils.read_graph_from_string(g) + # strip extra quoting from fill color + for k, v in graph.node_attr.items(): + for index, a in enumerate(v): + if a[0] == "fillcolor": + stripped_color = a[1].strip("'").strip('"') + v[index] = ("fillcolor", stripped_color) + + # attempt to graph result + result = resolve_graph.failure_detail_from_graph(graph) + self.assertTrue("foo-1 --> foo-1.0.0 --> bar-0.0.1 --> !foo-1.0.0" in result) + self.assertTrue("bar --> bar-0.0.1 --> !foo-1.0.0" in result) + + +if __name__ == '__main__': + unittest.main() diff --git a/src/rez/tests/test_version.py b/src/rez/tests/test_version.py index ebffd3013..72c26b612 100644 --- a/src/rez/tests/test_version.py +++ b/src/rez/tests/test_version.py @@ -3,15 +3,507 @@ """ -unit tests for 'version' module +unit tests for 'rez.version' module """ +import random +import textwrap import unittest -from rez.vendor.version.test import TestVersionSchema +from rez.version import Version, AlphanumericVersionToken, \ + VersionRange, reverse_sort_key +from rez.version._version import _ReversedComparable +from rez.version import Requirement, RequirementList +from rez.version import VersionError -class TestVersions(TestVersionSchema): + +def _print(txt=''): + # uncomment for verbose output + #print txt pass +class TestVersionSchema(unittest.TestCase): + make_token = AlphanumericVersionToken + + def __init__(self, fn): + unittest.TestCase.__init__(self, fn) + + def _test_strict_weak_ordering(self, a, b): + self.assertTrue(a == a) + self.assertTrue(b == b) + + e = (a == b) + ne = (a != b) + lt = (a < b) + lte = (a <= b) + gt = (a > b) + gte = (a >= b) + + _print('\n' + textwrap.dedent( + """ + '%s' '%s' + ==: %s + !=: %s + <: %s + <=: %s + >: %s + >=: %s + """).strip() % (a, b, e, ne, lt, lte, gt, gte)) + + self.assertTrue(e != ne) + if e: + self.assertTrue(not lt) + self.assertTrue(not gt) + self.assertTrue(lte) + self.assertTrue(gte) + else: + self.assertTrue(lt != gt) + self.assertTrue(lte != gte) + self.assertTrue(lt == lte) + self.assertTrue(gt == gte) + + if not isinstance(a, _ReversedComparable): + self._test_strict_weak_ordering(reverse_sort_key(a), + reverse_sort_key(b)) + + def _test_ordered(self, items): + def _test(fn, items_, op_str): + for i, a in enumerate(items_): + for b in items_[i + 1:]: + _print("'%s' %s '%s'" % (a, op_str, b)) + self.assertTrue(fn(a, b)) + + _test(lambda a, b: a < b, items, '<') + _test(lambda a, b: a <= b, items, '<=') + _test(lambda a, b: a != b, items, '!=') + _test(lambda a, b: a > b, list(reversed(items)), '>') + _test(lambda a, b: a >= b, list(reversed(items)), '>=') + _test(lambda a, b: a != b, list(reversed(items)), '!=') + + def _create_random_token(self): + s = self.make_token.create_random_token_string() + return self.make_token(s) + + def _create_random_version(self): + ver_str = '.'.join(self.make_token.create_random_token_string() + for i in range(random.randint(0, 6))) + return Version(ver_str, make_token=self.make_token) + + def test_misc(self): + self.assertEqual(Version("1.2.12").as_tuple(), ("1", "2", "12")) + + def test_token_strict_weak_ordering(self): + # test equal tokens + tok = self._create_random_token() + self._test_strict_weak_ordering(tok, tok) + + # test random tokens + for i in range(100): + tok1 = self._create_random_token() + tok2 = self._create_random_token() + self._test_strict_weak_ordering(tok1, tok2) + + def test_version_strict_weak_ordering(self): + # test equal versions + ver = self._create_random_version() + self._test_strict_weak_ordering(ver, ver) + + # test random versions + for i in range(100): + ver1 = self._create_random_version() + ver2 = self._create_random_version() + self._test_strict_weak_ordering(ver1, ver2) + + def test_token_comparisons(self): + def _lt(a, b): + _print("'%s' < '%s'" % (a, b)) + self.assertTrue(self.make_token(a) < self.make_token(b)) + self.assertTrue(Version(a) < Version(b)) + + _print() + _lt("3", "4") + _lt("01", "1") + _lt("beta", "1") + _lt("alpha3", "alpha4") + _lt("alpha", "alpha3") + _lt("gamma33", "33gamma") + + def test_version_comparisons(self): + def _eq(a, b): + _print("'%s' == '%s'" % (a, b)) + self.assertTrue(Version(a) == Version(b)) + + _print() + _eq("", "") + _eq("1", "1") + _eq("1.2", "1-2") + _eq("1.2-3", "1-2.3") + + ascending = ["", + "0.0.0", + "1", + "2", + "2.alpha1", + "2.alpha2", + "2.beta", + "2.0", + "2.0.8.8", + "2.1", + "2.1.0"] + self._test_ordered([Version(x) for x in ascending]) + + def _eq2(a, b): + _print("'%s' == '%s'" % (a, b)) + self.assertTrue(a == b) + + # test behaviour in sets + a = Version("1.0") + b = Version("1.0") + c = Version("1.0alpha") + d = Version("2.0.0") + + _eq2(set([a]) - set([a]), set()) + _eq2(set([a]) - set([b]), set()) + _eq2(set([a, a]) - set([a]), set()) + _eq2(set([b, c, d]) - set([a]), set([c, d])) + _eq2(set([b, c]) | set([c, d]), set([b, c, d])) + _eq2(set([b, c]) & set([c, d]), set([c])) + + def test_version_range(self): + def _eq(a, b): + _print("'%s' == '%s'" % (a, b)) + a_range = VersionRange(a) + b_range = VersionRange(b) + + self.assertTrue(a_range == b_range) + self.assertTrue(a_range.issuperset(a_range)) + self.assertTrue(a_range.issuperset(b_range)) + self.assertTrue(VersionRange(str(a_range)) == a_range) + self.assertTrue(VersionRange(str(b_range)) == a_range) + self.assertTrue(hash(a_range) == hash(b_range)) + + a_ = a.replace('.', '-') + a_ = a_.replace("--", "..") + a_range_ = VersionRange(a_) + self.assertTrue(a_range_ == a_range) + self.assertTrue(hash(a_range_) == hash(a_range)) + + range_strs = a.split('|') + ranges = [VersionRange(x) for x in range_strs] + ranges_ = ranges[0].union(ranges[1:]) + self.assertTrue(ranges_ == a_range) + + self.assertTrue(a_range | b_range == a_range) + self.assertTrue(a_range - b_range is None) + self.assertTrue(b_range - a_range is None) + self.assertTrue(VersionRange() & a_range == a_range) + self.assertTrue(b_range.span() & a_range == a_range) + + a_inv = a_range.inverse() + self.assertTrue(a_inv == ~b_range) + + if a_inv: + self.assertTrue(~a_inv == a_range) + self.assertTrue(a_range | a_inv == VersionRange()) + self.assertTrue(a_range & a_inv is None) + + a_ranges = a_range.split() + a_range_ = a_ranges[0].union(a_ranges[1:]) + self.assertTrue(a_range_ == b_range) + + def _and(a, b, c): + _print("'%s' & '%s' == '%s'" % (a, b, c)) + a_range = VersionRange(a) + b_range = VersionRange(b) + c_range = None if c is None else VersionRange(c) + self.assertTrue(a_range & b_range == c_range) + self.assertTrue(b_range & a_range == c_range) + + a_or_b = a_range | b_range + a_and_b = a_range & b_range + a_sub_b = a_range - b_range + b_sub_a = b_range - a_range + ranges = [a_and_b, a_sub_b, b_sub_a] + ranges = [x for x in ranges if x] + self.assertTrue(ranges[0].union(ranges[1:]) == a_or_b) + + def _inv(a, b): + a_range = VersionRange(a) + b_range = VersionRange(b) + self.assertTrue(~a_range == b_range) + self.assertTrue(~b_range == a_range) + self.assertTrue(a_range | b_range == VersionRange()) + self.assertTrue(a_range & b_range is None) + + # simple cases + _print() + _eq("", "") + _eq("1", "1") + _eq("1.0.0", "1.0.0") + _eq("3+<3_", "3") + _eq("_+<__", "_") + _eq("1.2+<=2.0", "1.2..2.0") + _eq("10+,<20", "10+<20") + _eq("1+<1.0", "1+<1.0") + _eq(">=2", "2+") + _eq(">=1.21.1,<1.23", ">=1.21.1<1.23") + _eq(">1.21.1,<1.23", ">1.21.1<1.23") + _eq(">1.21.1<1.23", ">1.21.1<1.23") + _eq(">1.21.1,<=1.23", ">1.21.1<=1.23") + + # Reverse order which is a syntax pip packages use more often now. + # Only allowed when separated by a comma. + _eq("<1.23,>=1.21.1", ">=1.21.1<1.23") + _eq("<1.23,>1.21.1", ">1.21.1<1.23") + + # optimised cases + _eq("3|3", "3") + _eq("3|1", "1|3") + _eq("5|3|1", "1|3|5") + _eq("1|1_", "1+<1__") + _eq("1|1_|1__", "1+,<1___") + _eq("|", "") + _eq("||", "||||||||") + _eq("1|1_+", "1+") + _eq("<1|1", "<1_") + _eq("1+<3|3+<5", "1+<5") + _eq(">4<6|1+<3", "1+<3|>4,<6") + _eq("4+<6|1+<3|", "") + _eq("4|2+", "2+") + _eq("3|<5", "<5") + _eq("<3|>3", ">3|<3") + _eq("3+|<3", "") + _eq("3+|<4", "") + _eq("2+<=6|3+<5", "2..6") + _eq("3+,<5|2+<=6", "2+<=6") + _eq("2|2+", "2+") + _eq("2|2.1+", "2+") + _eq("2|<2.1", "<2_") + _eq("3..3", "==3") + _eq(">=3,<=3", "==3") + + # AND'ing + _and("3", "3", "3") + _and("1", "==1", "==1") + _and("", "==1", "==1") + _and("3", "4", None) + _and("<3", "5+", None) + _and("4+<6", "6+<8", None) + _and("2+", "<=4", "2..4") + _and("1", "1.0", "1.0") + _and("4..6", "6+<8", "==6") + + # inverse + _inv("3+", "<3") + _inv("<=3", ">3") + _inv("3.5", "<3.5|3.5_+") + self.assertTrue(~VersionRange() is None) + + # odd (but valid) cases + _eq(">", ">") # greater than the empty version + _eq("+", "") # greater or equal to empty version (is all vers) + _eq(">=", "") # equivalent to above + _eq("<=", "==") # less or equal to empty version (is only empty) + _eq("..", "==") # from empty version to empty version + _eq("+<=", "==") # equivalent to above + + invalid_range = [ + "4+<2", # lower bound greater than upper + ">3<3", # both greater and less than same version + ">3<=3", # greater and less or equal to same version + "3+<3" # greater and equal to, and less than, same version + ] + + for s in invalid_range: + self.assertRaises(VersionError, VersionRange, s) + + invalid_syntax = [ + "<", # less than the empty version + "><", # both greater and less than empty version + ">3>4", # both are lower bounds + "<3<4", # both are upper bounds + "<4>3", # upper bound before lower without comma + ",<4", # leading comma + "4+,", # trailing comma + "1>=", # pre-lower-op in post + "+1", # post-lower-op in pre + "4<", # pre-upper-op in post + "1+<2<3" # more than two bounds + ] + + for s in invalid_syntax: + self.assertRaises(VersionError, VersionRange, s) + + # test simple logic + self.assertTrue(VersionRange("").is_any()) + self.assertTrue(VersionRange("2+<4").bounded()) + self.assertTrue(VersionRange("2+").lower_bounded()) + self.assertTrue(not VersionRange("2+").upper_bounded()) + self.assertTrue(not VersionRange("2+").bounded()) + self.assertTrue(VersionRange("<2").upper_bounded()) + self.assertTrue(not VersionRange("<2").lower_bounded()) + self.assertTrue(not VersionRange("<2").bounded()) + + # test range from version(s) + v = Version("3") + self.assertTrue(VersionRange.from_version(v, "eq") == VersionRange("==3")) + self.assertTrue(VersionRange.from_version(v, "gt") == VersionRange(">3")) + self.assertTrue(VersionRange.from_version(v, "gte") == VersionRange("3+")) + self.assertTrue(VersionRange.from_version(v, "lt") == VersionRange("<3")) + self.assertTrue(VersionRange.from_version(v, "lte") == VersionRange("<=3")) + + range1 = VersionRange.from_version(Version("2"), "gte") + range2 = VersionRange.from_version(Version("4"), "lte") + _eq(str(range1 & range2), "2..4") + + v2 = Version("6.0") + v3 = Version("4") + self.assertTrue(VersionRange.from_versions([v, v2, v3]) + == VersionRange("==3|==4|==6.0")) + + # test behaviour in sets + def _eq2(a, b): + _print("'%s' == '%s'" % (a, b)) + self.assertTrue(a == b) + + a = VersionRange("1+<=2.5") + b = VersionRange("1..2.5") + c = VersionRange(">=5") + d = VersionRange(">6.1.0") + e = VersionRange("3.2") + + _eq2(set([a]) - set([a]), set()) + _eq2(set([a]) - set([b]), set()) + _eq2(set([a, a]) - set([a]), set()) + _eq2(set([b, c, d, e]) - set([a]), set([c, d, e])) + _eq2(set([b, c, e]) | set([c, d]), set([b, c, d, e])) + _eq2(set([b, c]) & set([c, d]), set([c])) + + def test_containment(self): + # basic containment + self.assertTrue(Version("3") in VersionRange("3+")) + self.assertTrue(Version("5") in VersionRange("3..5")) + self.assertTrue(Version("5_") not in VersionRange("3..5")) + self.assertTrue(Version("3.0.0") in VersionRange("3+")) + self.assertTrue(Version("3.0.0") not in VersionRange("3.1+")) + self.assertTrue(Version("3") in VersionRange("<1|5|6|8|7|3|60+")) + self.assertTrue(Version("3") in VersionRange("<1|5|6|8|7|==3|60+")) + self.assertTrue(VersionRange("2.1+<4") in VersionRange("<4")) + self.assertTrue(VersionRange("2.1..4") not in VersionRange("<4")) + self.assertTrue(VersionRange("3") in VersionRange("3")) + self.assertTrue(VersionRange("==3") in VersionRange("3")) + self.assertTrue(VersionRange("3.5+<3_") in VersionRange("3")) + self.assertTrue(VersionRange("3") not in VersionRange("4+<6")) + self.assertTrue(VersionRange("3+<10") not in VersionRange("4+<6")) + + # iterating over sorted version list + numbers = [2, 3, 5, 10, 11, 13, 14] + versions = [Version(str(x)) for x in numbers] + rev_versions = list(reversed(versions)) + composite_range = VersionRange.from_versions(versions) + + entries = [(VersionRange(""), 7), + (VersionRange("0+"), 7), + (VersionRange("5+"), 5), + (VersionRange("6+"), 4), + (VersionRange("50+"), 0), + (VersionRange(">5"), 4), + (VersionRange("5"), 1), + (VersionRange("6"), 0), + (VersionRange("<5"), 2), + (VersionRange("<6"), 3), + (VersionRange("<50"), 7), + (VersionRange("<=5"), 3), + (VersionRange("<1"), 0), + (VersionRange("2|9+"), 5), + (VersionRange("3+<6|12+<13.5"), 3), + (VersionRange("<1|20+"), 0), + (VersionRange(">0<20"), 7)] + + for range_, count in entries: + # brute-force containment tests + matches = set(x for x in versions if x in range_) + self.assertEqual(len(matches), count) + + # more optimal containment tests + def _test_it(it): + matches_ = set(version for contains, version in it if contains) + self.assertEqual(matches_, matches) + + _test_it(range_.iter_intersect_test(versions)) + _test_it(range_.iter_intersect_test(rev_versions, descending=True)) + + # throw in an intersection test + self.assertEqual(composite_range.intersects(range_), (count != 0)) + int_range = composite_range & range_ + versions_ = [] if int_range is None else int_range.to_versions() + self.assertEqual(set(versions_), matches) + + # throw in a superset test as well + self.assertEqual(range_.issuperset(composite_range), (count == 7)) + if count: + self.assertTrue(composite_range.issuperset(int_range)) + + def test_requirement_list(self): + def _eq(reqs, expected_reqs): + _print("requirements(%s) == requirements(%s)" + % (' '.join(reqs), ' '.join(expected_reqs))) + reqs_ = [Requirement(x) for x in reqs] + reqlist = RequirementList(reqs_) + _print("result: %s" % str(reqlist)) + + exp_reqs_ = [Requirement(x) for x in expected_reqs] + self.assertTrue(reqlist.requirements == exp_reqs_) + + exp_names = set(x.name for x in exp_reqs_ if not x.conflict) + self.assertTrue(reqlist.names == exp_names) + + exp_confl_names = set(x.name for x in exp_reqs_ if x.conflict) + self.assertTrue(reqlist.conflict_names == exp_confl_names) + + def _confl(reqs, a, b): + _print("requirements(%s) == %s <--!--> %s" % (' '.join(reqs), a, b)) + reqs_ = [Requirement(x) for x in reqs] + reqlist = RequirementList(reqs_) + _print("result: %s" % str(reqlist)) + + a_req = Requirement(a) + b_req = Requirement(b) + self.assertTrue(reqlist.conflict == (a_req, b_req)) + + _print() + _eq(["foo"], + ["foo"]) + _eq(["foo", "bah"], + ["foo", "bah"]) + _eq(["bah", "foo"], + ["bah", "foo"]) + _eq(["foo-4+", "foo-4.5"], + ["foo-4.5"]) + _eq(["bah-2.4", "foo", "bah-2.4.1+"], + ["bah-2.4.1+<2.4_", "foo"]) + _eq(["foo-2+", "!foo-4+"], + ["foo-2+<4"]) + _eq(["!bah-1", "!bah-3"], + ["!bah-1|3"]) + _eq(["!bah-5", "foo-2.3", "!bah-5.6+"], + ["!bah-5+", "foo-2.3"]) + _eq(["~bah-4", "foo", "bah<4.2"], + ["bah-4+<4.2", "foo"]) + _eq(["~bah", "!foo", "bah<4.2"], + ["bah<4.2", "!foo"]) + _eq(["~bah-3+", "~bah-5"], + ["~bah-5"]) + + _confl(["foo-1", "foo-2"], + "foo-1", "foo-2") + _confl(["foo-2", "foo-1"], + "foo-2", "foo-1") + _confl(["foo", "~bah-5+", "bah-2"], + "~bah-5+", "bah-2") + _confl(["foo", "~bah-5+", "bah-7..12", "bah-2"], + "bah-7..12", "bah-2") + + if __name__ == '__main__': unittest.main() diff --git a/src/rez/tests/util.py b/src/rez/tests/util.py index 7a832ce27..9e5f32ad3 100644 --- a/src/rez/tests/util.py +++ b/src/rez/tests/util.py @@ -19,6 +19,7 @@ import functools import sys import json +import copy from contextlib import contextmanager # https://pypi.org/project/parameterized @@ -48,6 +49,11 @@ def setUpClass(cls): cls.settings = {} def setUp(self): + # We have some tests that unfortunately don't clean themselves up + # after they are done. Store the origianl environment to be + # restored in tearDown + self.__environ = copy.deepcopy(os.environ) + self.maxDiff = None os.environ["REZ_QUIET"] = "true" @@ -65,6 +71,7 @@ def setup_once(self): def tearDown(self): self.teardown_config() + os.environ = self.__environ @classmethod def data_path(cls, *dirs): @@ -133,6 +140,13 @@ def get_settings_env(self): for k, v in self.settings.items() ) + def inject_python_repo(self): + self.update_settings( + { + "packages_path": config.packages_path + [os.environ["__REZ_SELFTEST_PYTHON_REPO"]], + } + ) + class TempdirMixin(object): """Mixin that adds tmpdir create/delete.""" @@ -242,11 +256,50 @@ def per_available_shell(exclude=None, include=None): # https://pypi.org/project/parameterized if use_parameterized: - return parameterized.expand(shells, skip_on_empty=True) + + class rez_parametrized(parameterized): + + # Taken from https://github.com/wolever/parameterized/blob/b9f6a640452bcfdea08efc4badfe5bfad043f099/parameterized/parameterized.py#L612 # noqa + @classmethod + def param_as_standalone_func(cls, p, func, name): + # @wraps(func) + def standalone_func(*args, **kwargs): + # Make sure to set the default shell to the requested shell. This + # simplifies tests and removes the need to remember passing the shell + # kward to execute_shell and co inside the tests. + # Subclassing parameterized is fragile, but we can't do better for now. + settings = {"default_shell": p.args[0]} + + # TODO: If & when path normalization is set to True by default, + # this should be removed. For now, we need to enable it for + # gitbash, because it requires path normalization. + if p.args[0] == "gitbash": + settings["enable_path_normalization"] = True + + args[0].update_settings(settings) + return func(*(args + p.args), **p.kwargs, **kwargs) + + standalone_func.__name__ = name + + # place_as is used by py.test to determine what source file should be + # used for this test. + standalone_func.place_as = func + + # Remove __wrapped__ because py.test will try to look at __wrapped__ + # to determine which parameters should be used with this test case, + # and obviously we don't need it to do any parameterization. + try: + del standalone_func.__wrapped__ + except AttributeError: + pass + return standalone_func + + return rez_parametrized.expand(shells, skip_on_empty=True) def decorator(func): @functools.wraps(func) def wrapper(self, shell=None): + for shell in shells: print("\ntesting in shell: %s..." % shell) # The default shell if none is configured is the system shell @@ -293,7 +346,7 @@ def wrapper(self, *args, **kwargs): else: self.skipTest( "Must be run via 'rez-selftest' tool, see " - "https://github.com/AcademySoftwareFoundation/rez/wiki/Installation#installation-script" + "https://rez.readthedocs.io/en/stable/installation.html#installation-script" ) return wrapper return decorator diff --git a/src/rez/utils/_version.py b/src/rez/utils/_version.py index 08ae7bcb8..58245d0ec 100644 --- a/src/rez/utils/_version.py +++ b/src/rez/utils/_version.py @@ -3,4 +3,5 @@ # Update this value to version up Rez. Do not place anything else in this file. -_rez_version = "2.113.0" +# Using .devN allows us to run becnmarks and create proper benchmark reports on PRs. +_rez_version = "2.114.1" diff --git a/src/rez/utils/colorize.py b/src/rez/utils/colorize.py index f20fb28ae..4fa3e8900 100644 --- a/src/rez/utils/colorize.py +++ b/src/rez/utils/colorize.py @@ -7,7 +7,6 @@ import sys import logging from rez.vendor import colorama -from rez.config import config # Important - we don't want to init Colorama at startup, # because colorama prints a RESET_ALL character at exit. This in turn adds @@ -221,6 +220,7 @@ def _color(str_, fore_color=None, back_color=None, styles=None): .. _Colorama: https://pypi.python.org/pypi/colorama """ + from rez.config import config # Avoid circular import if not config.get("color_enabled", False): return str_ @@ -239,6 +239,7 @@ def _color(str_, fore_color=None, back_color=None, styles=None): def _get_style_from_config(key): + from rez.config import config # Avoid circular import fore_color = config.get("%s_fore" % key, '') back_color = config.get("%s_back" % key, '') styles = config.get("%s_styles" % key, None) @@ -251,13 +252,11 @@ class ColorizedStreamHandler(logging.StreamHandler): This handler uses the `Colorama`_ module to style the log messages based on the rez configuration. - Attributes: - STYLES (dict): A mapping between the Python logger levels and a function - that can be used to provide the appropriate styling. - .. _Colorama: https://pypi.python.org/pypi/colorama """ + #: A mapping between the Python logger levels and a function that can be used + #: to provide the appropriate styling. STYLES = { 50: critical, 40: error, @@ -283,6 +282,7 @@ def is_tty(self): @property def is_colorized(self): + from rez.config import config # Avoid circular import return config.get("color_enabled", False) == "force" or self.is_tty def _get_style_function_for_level(self, level): @@ -315,6 +315,7 @@ def emit(self, record): class Printer(object): def __init__(self, buf=sys.stdout): + from rez.config import config # Avoid circular import self.colorize = ( config.get("color_enabled", False) == "force" or stream_is_tty(buf) diff --git a/src/rez/utils/data_utils.py b/src/rez/utils/data_utils.py index c3a8a4ea1..35d0c0a2a 100644 --- a/src/rez/utils/data_utils.py +++ b/src/rez/utils/data_utils.py @@ -50,8 +50,8 @@ class DelayLoad(object): Supported formats: - - yaml (*.yaml, *.yml) - - json (*.json) + - yaml (``*.yaml``, ``*.yml``) + - json (``*.json``) """ def __init__(self, filepath): self.filepath = os.path.expanduser(filepath) @@ -168,7 +168,7 @@ def get_dict_diff(d1, d2): was affected. Returns: - 3-tuple: + tuple: 3-tuple: - list of added keys; - list of removed key; - list of changed keys. @@ -286,6 +286,9 @@ class cached_class_property(object): """ def __init__(self, func, name=None): self.func = func + # Make sure that Sphinx autodoc can follow and get the docstring from our wrapped function. + # TODO: Doesn't work... + functools.update_wrapper(self, func) def __get__(self, instance, owner=None): assert owner diff --git a/src/rez/utils/elf.py b/src/rez/utils/elf.py index e22bf8beb..402638ad5 100644 --- a/src/rez/utils/elf.py +++ b/src/rez/utils/elf.py @@ -6,7 +6,7 @@ Functions that wrap readelf/patchelf utils on linux. """ import os -import pipes +from rez.utils.py23 import quote import subprocess from rez.utils.filesystem import make_path_writable @@ -64,7 +64,7 @@ def _run(*nargs, **popen_kwargs): out, err = proc.communicate() if proc.returncode: - cmd_ = ' '.join(pipes.quote(x) for x in nargs) + cmd_ = ' '.join(quote(x) for x in nargs) raise RuntimeError( "Command %s - failed with exitcode %d: %s" diff --git a/src/rez/utils/execution.py b/src/rez/utils/execution.py index a94e1def4..2a3071549 100644 --- a/src/rez/utils/execution.py +++ b/src/rez/utils/execution.py @@ -125,7 +125,7 @@ def create_executable_script(filepath, body, program=None, py_script_mode=None): Args: filepath (str): File to create. - body (str or callable): Contents of the script. If a callable, its code + body (str or typing.Callable): Contents of the script. If a callable, its code is used as the script body. program (str): Name of program to launch the script. Default is 'python' py_script_mode(ExecutableScriptMode): What kind of script to create. diff --git a/src/rez/utils/filesystem.py b/src/rez/utils/filesystem.py index be2906b2d..e9c6ddfaf 100644 --- a/src/rez/utils/filesystem.py +++ b/src/rez/utils/filesystem.py @@ -507,14 +507,14 @@ def to_posixpath(path): def canonical_path(path, platform=None): - """ Resolves symlinks, and formats filepath. + r""" Resolves symlinks, and formats filepath. Resolves symlinks, lowercases if filesystem is case-insensitive, formats filepath using slashes appropriate for platform. Args: path (str): Filepath being formatted - platform (rez.utils.platform_.Platform): Indicates platform path is being + platform (rez.utils.platform\_.Platform): Indicates platform path is being formatted for. Defaults to current platform. Returns: @@ -542,38 +542,38 @@ def encode_filesystem_name(input_str): The rules for the encoding are: - 1) Any lowercase letter, digit, period, or dash (a-z, 0-9, ., or -) is + 1. Any lowercase letter, digit, period, or dash (a-z, 0-9, ., or -) is encoded as-is. - 2) Any underscore is encoded as a double-underscore ("__") + 2. Any underscore is encoded as a double-underscore (``__``) - 3) Any uppercase ascii letter (A-Z) is encoded as an underscore followed + 3. Any uppercase ascii letter (A-Z) is encoded as an underscore followed by the corresponding lowercase letter (ie, "A" => "_a") - 4) All other characters are encoded using their UTF-8 encoded unicode - representation, in the following format: "_NHH..., where: - a) N represents the number of bytes needed for the UTF-8 encoding, - except with N=0 for one-byte representation (the exception for N=1 - is made both because it means that for "standard" ascii characters - in the range 0-127, their encoding will be _0xx, where xx is their - ascii hex code; and because it mirrors the ways UTF-8 encoding - itself works, where the number of bytes needed for the character can - be determined by counting the number of leading "1"s in the binary - representation of the character, except that if it is a 1-byte - sequence, there are 0 leading 1's). - b) HH represents the bytes of the corresponding UTF-8 encoding, in - hexadecimal (using lower-case letters) - - As an example, the character "*", whose (hex) UTF-8 representation - of 2A, would be encoded as "_02a", while the "euro" symbol, which - has a UTF-8 representation of E2 82 AC, would be encoded as - "_3e282ac". (Note that, strictly speaking, the "N" part of the - encoding is redundant information, since it is essentially encoded - in the UTF-8 representation itself, but it makes the resulting - string more human-readable, and easier to decode). - - As an example, the string "Foo_Bar (fun).txt" would get encoded as: - _foo___bar_020_028fun_029.txt + 4. All other characters are encoded using their UTF-8 encoded unicode + representation, in the following format: ``_NHH...``, where: + + * N represents the number of bytes needed for the UTF-8 encoding, + except with N=0 for one-byte representation (the exception for N=1 + is made both because it means that for "standard" ascii characters + in the range 0-127, their encoding will be _0xx, where xx is their + ascii hex code; and because it mirrors the ways UTF-8 encoding + itself works, where the number of bytes needed for the character can + be determined by counting the number of leading "1"s in the binary + representation of the character, except that if it is a 1-byte + sequence, there are 0 leading 1's). + * HH represents the bytes of the corresponding UTF-8 encoding, in + hexadecimal (using lower-case letters) + + As an example, the character ``*``, whose (hex) UTF-8 representation + of 2A, would be encoded as "_02a", while the "euro" symbol, which + has a UTF-8 representation of E2 82 AC, would be encoded as + "_3e282ac". (Note that, strictly speaking, the "N" part of the + encoding is redundant information, since it is essentially encoded + in the UTF-8 representation itself, but it makes the resulting + string more human-readable, and easier to decode). + + As an example, the string "Foo_Bar (fun).txt" would get encoded as ``_foo___bar_020_028fun_029.txt``. """ if isinstance(input_str, six.string_types): input_str = unicode(input_str) diff --git a/src/rez/utils/formatting.py b/src/rez/utils/formatting.py index 3b7db24d2..e39708e4f 100644 --- a/src/rez/utils/formatting.py +++ b/src/rez/utils/formatting.py @@ -9,7 +9,7 @@ from string import Formatter from rez.vendor.enum import Enum -from rez.vendor.version.requirement import Requirement +from rez.version import Requirement from rez.exceptions import PackageRequestError from rez.vendor.six import six from pprint import pformat diff --git a/src/rez/utils/logging_.py b/src/rez/utils/logging_.py index d601a4a37..201305393 100644 --- a/src/rez/utils/logging_.py +++ b/src/rez/utils/logging_.py @@ -85,7 +85,7 @@ def view_file_logs(globbed_path, loglevel_index=None): Prints to stdout. Args: - globbed_path (str): Logfiles, eg '/foo/logs/*.log' + globbed_path (str): Logfiles, eg ``/foo/logs/*.log`` loglevel_index (int): Position on each log line where log level (INFO etc) is expected. This is used for colorisation only, and if None, no colors are applied. diff --git a/src/rez/utils/memcached.py b/src/rez/utils/memcached.py index 52fc54bf4..d66108578 100644 --- a/src/rez/utils/memcached.py +++ b/src/rez/utils/memcached.py @@ -283,7 +283,9 @@ def memcached(servers, key=None, from_cache=None, to_cache=None, time=0, being returned. If you do not want a result to be cached, wrap the return value of your function in a `DoNotCache` object. - Example: + Examples: + + .. code-block:: python @memcached('127.0.0.1:11211') def _listdir(path): @@ -300,11 +302,11 @@ def _listdir(path): Args: servers (str or list of str): memcached server uri(s), eg '127.0.0.1:11211'. This arg can be None also, in which case memcaching is disabled. - key (callable, optional): Function that, given the target function's args, + key (typing.Optional[typing.Callable]): Function that, given the target function's args, returns the string key to use in memcached. - from_cache (callable, optional): If provided, and a cache hit occurs, the + from_cache (typing.Optional[typing.Callable]): If provided, and a cache hit occurs, the cached value will be translated by this function before being returned. - to_cache (callable, optional): If provided, and a cache miss occurs, the + to_cache (typing.Optional[typing.Callable]): If provided, and a cache miss occurs, the function's return value will be translated by this function before being cached. time (int): Tells memcached the time which this value should expire, either diff --git a/src/rez/utils/patching.py b/src/rez/utils/patching.py index 6986b1c23..1252867bb 100644 --- a/src/rez/utils/patching.py +++ b/src/rez/utils/patching.py @@ -2,7 +2,7 @@ # Copyright Contributors to the Rez Project -from rez.vendor.version.requirement import Requirement +from rez.version import Requirement def get_patched_request(requires, patchlist): diff --git a/src/rez/utils/pip.py b/src/rez/utils/pip.py index beeeddca1..a4c70e08c 100644 --- a/src/rez/utils/pip.py +++ b/src/rez/utils/pip.py @@ -16,8 +16,8 @@ InvalidVersion as packaging_InvalidVersion ) from rez.vendor.packaging.requirements import Requirement as packaging_Requirement -from rez.vendor.version.requirement import Requirement -from rez.vendor.version.version import Version, VersionRange +from rez.version import Requirement +from rez.version import Version, VersionRange from rez.utils.logging_ import print_warning from rez.exceptions import PackageRequestError @@ -49,23 +49,23 @@ def pip_to_rez_version(dist_version, allow_legacy=True): The python version schema specification isn't 100% compatible with rez. - 1: version epochs (they make no sense to rez, so they'd just get stripped - of the leading N!; - 2: python versions are case insensitive, so they should probably be + 1. version epochs (they make no sense to rez, so they'd just get stripped + of the leading ``N!``; + 2. python versions are case insensitive, so they should probably be lowercased when converted to a rez version. - 3: local versions are also not compatible with rez + 3. local versions are also not compatible with rez The canonical public version identifiers MUST comply with the following scheme: - [N!]N(.N)*[{a|b|rc}N][.postN][.devN] + ``[N!]N(.N)*[{a|b|rc}N][.postN][.devN]`` - Epoch segment: N! - skip - Release segment: N(.N)* 0 as is - Pre-release segment: {a|b|c|rc|alpha|beta|pre|preview}N - always lowercase - Post-release segment: .{post|rev|r}N - always lowercase - Development release segment: .devN - always lowercase + Epoch segment: ``N!`` - skip + Release segment: N(.N)* 0`` as is + Pre-release segment: ``{a|b|c|rc|alpha|beta|pre|preview}N`` - always lowercase + Post-release segment: ``.{post|rev|r}N`` - always lowercase + Development release segment: ``.devN`` - always lowercase Local version identifiers MUST comply with the following scheme: - [+] - use - instead of + + ``[+]`` - use - instead of + Args: dist_version (str): The distribution version to be converted. @@ -76,7 +76,7 @@ def pip_to_rez_version(dist_version, allow_legacy=True): Raises: InvalidVersion: When legacy mode is not allowed and a PEP440 - incompatible version is detected. + incompatible version is detected. .. _PEP 440 (all possible matches): https://www.python.org/dev/peps/pep-0440/#appendix-b-parsing-version-strings-with-regular-expressions @@ -164,20 +164,22 @@ def pip_specifier_to_rez_requirement(specifier): Example conversions: - | PEP440 | rez | - |-------------|-------------| - | ==1 | 1+<1.1 | - | ==1.* | 1 | - | >1 | 1.1+ | - | <1 | <1 | - | >=1 | 1+ | - | <=1 | <1.1 | - | ~=1.2 | 1.2+<2 | - | ~=1.2.3 | 1.2.3+<1.3 | - | !=1 | <1|1.1+ | - | !=1.2 | <1.2|1.2.1+ | - | !=1.* | <1|2+ | - | !=1.2.* | <1.2|1.3+ | + ============== =============== + PEP440 rez + ============== =============== + ``==1`` ``1+<1.1`` + ``==1.*`` ``1`` + ``>1`` ``1.1+`` + ``<1`` ``<1`` + ``>=1`` ``1+`` + ``<=1`` ``<1.1`` + ``~=1.2`` ``1.2+<2`` + ``~=1.2.3`` ``1.2.3+<1.3`` + ``!=1`` ``<1|1.1+`` + ``!=1.2`` ``<1.2|1.2.1+`` + ``!=1.*`` ``<1|2+`` + ``!=1.2.*`` ``<1.2|1.3+`` + ============== =============== Args: specifier (`package.SpecifierSet`): Pip specifier. @@ -353,14 +355,16 @@ def get_rez_requirements(installed_dist, python_version, name_casings=None): Example result: - { - "requires": ["foo-1.2+<2"], - "variant_requires": ["future", "python-2.7"], - "metadata": { - # metadata pertinent to rez - ... - } - } + .. code-block:: python + + { + "requires": ["foo-1.2+<2"], + "variant_requires": ["future", "python-2.7"], + "metadata": { + # metadata pertinent to rez + ... + } + } Each requirement has had its package name converted to the rez equivalent. The 'variant_requires' key contains requirements specific to the current diff --git a/src/rez/utils/platform_mapped.py b/src/rez/utils/platform_mapped.py index ce7c7f01b..aeb37dc28 100644 --- a/src/rez/utils/platform_mapped.py +++ b/src/rez/utils/platform_mapped.py @@ -14,16 +14,18 @@ def platform_mapped(func): regular expression is being used. For example: - config.platform_map = { - "os": { - r"Scientific Linux-(.*)": r"Scientific-\1", # Scientific Linux-x.x -> Scientific-x.x - r"Ubuntu-14.\d": r"Ubuntu-14", # Any Ubuntu-14.x -> Ubuntu-14 - }, - "arch": { - "x86_64": "64bit", # Maps both x86_64 and amd64 -> 64bit (don't) - "amd64": "64bit", - }, - } + .. code-block:: python + + config.platform_map = { + "os": { + r"Scientific Linux-(.*)": r"Scientific-\1", # Scientific Linux-x.x -> Scientific-x.x + r"Ubuntu-14.\d": r"Ubuntu-14", # Any Ubuntu-14.x -> Ubuntu-14 + }, + "arch": { + "x86_64": "64bit", # Maps both x86_64 and amd64 -> 64bit (don't) + "amd64": "64bit", + }, + } """ def inner(*args, **kwargs): diff --git a/src/rez/utils/py23.py b/src/rez/utils/py23.py index 09168a8e1..8eee97f81 100644 --- a/src/rez/utils/py23.py +++ b/src/rez/utils/py23.py @@ -12,6 +12,18 @@ from rez.vendor.six import six +try: + from html import escape # noqa: F401 +except ImportError: + # Python 2 + from cgi import escape # noqa: F401 + +try: + from shlex import quote # noqa: F401 +except ImportError: + # Python 2 + from pipes import quote # noqa: F401 + def get_function_arg_names(func): """Get names of a function's args. diff --git a/src/rez/utils/py_dist.py b/src/rez/utils/py_dist.py index 904a37870..4bdc0e909 100644 --- a/src/rez/utils/py_dist.py +++ b/src/rez/utils/py_dist.py @@ -134,6 +134,7 @@ def convert_dist(name, dest_path, make_variant=True, ignore_dirs=None, ignore_dirs (bool): List of directory names to not copy from the dist. python_requirement (str): How the package should depend on python. One of: + - "major": depend on python-X - "major_minor": depend on python-X.X - any other value: this string is used as the literal version diff --git a/src/rez/utils/resolve_graph.py b/src/rez/utils/resolve_graph.py index c0c0aa306..5108a7ca6 100644 --- a/src/rez/utils/resolve_graph.py +++ b/src/rez/utils/resolve_graph.py @@ -53,7 +53,7 @@ def _cycled_detail_from_graph(graph, cycled_edge): visited = list() while True: visited.append(node) - down = next((ne for ne in graph.node_neighbors[node]), None) + down = next((ne for ne in graph.node_neighbors[node] if ne not in visited), None) if down in cycled_edge: visited.append(down) break @@ -83,7 +83,7 @@ def _conflicted_detail_from_graph(graph, conflicted_edge): visited = list() while True: visited.append(node) - down = next((ne for ne in graph.node_neighbors[node]), None) + down = next((ne for ne in graph.node_neighbors[node] if ne not in visited), None) if down is None: break diff --git a/src/rez/utils/resources.py b/src/rez/utils/resources.py index 2fca65e7a..489004eda 100644 --- a/src/rez/utils/resources.py +++ b/src/rez/utils/resources.py @@ -67,16 +67,14 @@ class Resource(six.with_metaclass(LazyAttributeMeta, object)): Note: You can access the entire validated resource data dict using the `validated_data` function, and test full validation using `validate_data`. - - Attributes: - key (str): Unique identifier of the resource type. - schema (Schema): Schema for the resource data. Must validate a dict. - Can be None, in which case the resource does not load any data. - schema_error (Exception): The exception type to raise on key - validation failure. """ + #: Unique identifier of the resource type. key = None + #: Schema for the resource data. + #: Must validate a dict. Can be None, in which case the resource does + #: not load any data. schema = None + #: The exception type to raise on key validation failure. schema_error = Exception @classmethod diff --git a/src/rez/utils/schema.py b/src/rez/utils/schema.py index a0d23b441..5dc6671dd 100644 --- a/src/rez/utils/schema.py +++ b/src/rez/utils/schema.py @@ -23,10 +23,12 @@ def schema_keys(schema): Non-string keys are ignored. Returns: - Set of string keys of a schema which is in the form (eg): + set[str]: Set of string keys of a schema which is in the form (eg): - schema = Schema({Required("foo"): int, - Optional("bah"): basestring}) + .. code-block:: python + + schema = Schema({Required("foo"): int, + Optional("bah"): basestring}) """ def _get_leaf(value): if isinstance(value, Schema): @@ -50,9 +52,9 @@ def dict_to_schema(schema_dict, required, allow_custom_keys=True, modifier=None) Args: required (bool): Whether to make schema keys optional or required. - allow_custom_keys (bool, optional): If True, creates a schema that + allow_custom_keys (typing.Optional[bool]): If True, creates a schema that allows custom items in dicts. - modifier (callable): Functor to apply to dict values - it is applied + modifier (typing.Optional[typing.Callable]): Functor to apply to dict values - it is applied via `Schema.Use`. Returns: diff --git a/src/rez/utils/yaml.py b/src/rez/utils/yaml.py index f1e95c6f8..5cef870f9 100644 --- a/src/rez/utils/yaml.py +++ b/src/rez/utils/yaml.py @@ -5,8 +5,8 @@ from rez.utils.sourcecode import SourceCode from rez.vendor import yaml from rez.vendor.yaml.dumper import SafeDumper -from rez.vendor.version.version import Version -from rez.vendor.version.requirement import Requirement +from rez.version import Version +from rez.version import Requirement from types import FunctionType, BuiltinFunctionType from inspect import getsourcelines from textwrap import dedent diff --git a/src/rez/vendor/README.md b/src/rez/vendor/README.md index d244475ab..03fff75d4 100644 --- a/src/rez/vendor/README.md +++ b/src/rez/vendor/README.md @@ -215,7 +215,7 @@ Updated (July 2019) along with pydot to allow for packaging lib to be used. schema -0.3.1 (Apr 28, 2014) +0.3.1 (Apr 28, 2014) (https://github.com/keleshev/schema/blob/916ba05e22b7b370b3586f97c40695e7b9e7fe33) MIT diff --git a/src/rez/vendor/memcache/memcache.py b/src/rez/vendor/memcache/memcache.py index 12effb31d..377320383 100644 --- a/src/rez/vendor/memcache/memcache.py +++ b/src/rez/vendor/memcache/memcache.py @@ -325,7 +325,8 @@ def get_stats(self, stat_args=None): readline = s.readline while 1: line = readline() - if not line or line.decode('ascii').strip() == 'END': + # Rez: Patch for https://github.com/AcademySoftwareFoundation/rez/issues/1563. + if not line or line.decode('ascii').strip() in ('END', 'RESET'): break stats = line.decode('ascii').split(' ', 2) serverData[stats[1]] = stats[2] diff --git a/src/rez/vendor/version/__init__.py b/src/rez/vendor/version/__init__.py index ac31011e1..ec87df48e 100644 --- a/src/rez/vendor/version/__init__.py +++ b/src/rez/vendor/version/__init__.py @@ -1,2 +1,11 @@ # SPDX-License-Identifier: Apache-2.0 # Copyright Contributors to the Rez Project + + +import rez.deprecations + +rez.deprecations.warn( + "module 'rez.vendor.version' is deprecated and will be removed in 3.0.0. Use 'rez.version' instead.", + rez.deprecations.RezDeprecationWarning, + stacklevel=2 +) diff --git a/src/rez/vendor/version/requirement.py b/src/rez/vendor/version/requirement.py index 7c4cc6a9e..04f53ae5f 100644 --- a/src/rez/vendor/version/requirement.py +++ b/src/rez/vendor/version/requirement.py @@ -2,416 +2,4 @@ # Copyright Contributors to the Rez Project -from rez.vendor.version.version import Version, VersionRange -from rez.vendor.version.util import _Common -import re - - -class VersionedObject(_Common): - """Definition of a versioned object, eg "foo-1.0". - - "foo" is also a valid object definiton - when there is no version part, we - are defining an unversioned object. - - Note that '-', '@' or '#' can be used as the seperator between object name - and version, however this is purely cosmetic - "foo-1" is the same as "foo@1". - """ - sep_regex_str = r'[-@#]' - sep_regex = re.compile(sep_regex_str) - - def __init__(self, s): - self.name_ = None - self.version_ = None - self.sep_ = '-' - if s is None: - return - - m = self.sep_regex.search(s) - if m: - i = m.start() - self.name_ = s[:i] - self.sep_ = s[i] - ver_str = s[i + 1:] - self.version_ = Version(ver_str) - else: - self.name_ = s - self.version_ = Version() - - @classmethod - def construct(cls, name, version=None): - """Create a VersionedObject directly from an object name and version. - - Args: - name: Object name string. - version: Version object. - """ - other = VersionedObject(None) - other.name_ = name - other.version_ = Version() if version is None else version - return other - - @property - def name(self): - """Name of the object.""" - return self.name_ - - @property - def version(self): - """Version of the object.""" - return self.version_ - - def as_exact_requirement(self): - """Get the versioned object, as an exact requirement string. - - Returns: - Equivalent requirement string, eg "maya==2016.1" - """ - sep_str = '' - ver_str = '' - if self.version_: - sep_str = "==" - ver_str = str(self.version_) - return self.name_ + sep_str + ver_str - - def __eq__(self, other): - return (isinstance(other, VersionedObject) - and (self.name_ == other.name_) - and (self.version_ == other.version_)) - - def __hash__(self): - return hash((self.name_, self.version_)) - - def __str__(self): - sep_str = '' - ver_str = '' - if self.version_: - sep_str = self.sep_ - ver_str = str(self.version_) - return self.name_ + sep_str + ver_str - - -class Requirement(_Common): - """Requirement for a versioned object. - - Examples of valid requirement strings: - - foo-1.0 - foo@1.0 - foo#1.0 - foo-1+ - foo-1+<4.3 - foo<3 - foo==1.0.1 - - Defines a requirement for an object. For example, "foo-5+" means that you - require any version of "foo", version 5 or greater. An unversioned - requirement can also be used ("foo"), this means you require any version of - foo. You can drop the hyphen between object name and version range if the - version range starts with a non-alphanumeric character - eg "foo<2". - - There are two different prefixes that can be applied to a requirement: - - - "!": The conflict requirement. This means that you require this version - range of an object NOT to be present. To conflict with all versions of an - object, use "!foo". - - - "~": This is known as a "weak reference", and means, "I do not require this - object, but if present, it must be within this range." It is equivalent to - the *conflict of the inverse* of the given version range. - - There is one subtle case to be aware of. "~foo" is a requirement that has no - effect - ie, it means "I do not require foo, but if foo is present, it can - be any version." This statement is still valid, but will produce a - Requirement object with a None range. - """ - sep_regex = re.compile(r'[-@#=<>]') - - def __init__(self, s, invalid_bound_error=True): - self.name_ = None - self.range_ = None - self.negate_ = False - self.conflict_ = False - self._str = None - self.sep_ = '-' - if s is None: - return - - self.conflict_ = s.startswith('!') - if self.conflict_: - s = s[1:] - elif s.startswith('~'): - s = s[1:] - self.negate_ = True - self.conflict_ = True - - m = self.sep_regex.search(s) - if m: - i = m.start() - self.name_ = s[:i] - req_str = s[i:] - if req_str[0] in ('-', '@', '#'): - self.sep_ = req_str[0] - req_str = req_str[1:] - - self.range_ = VersionRange( - req_str, invalid_bound_error=invalid_bound_error) - if self.negate_: - self.range_ = ~self.range_ - elif self.negate_: - self.name_ = s - # rare case - '~foo' equates to no effect - self.range_ = None - else: - self.name_ = s - self.range_ = VersionRange() - - @classmethod - def construct(cls, name, range=None): - """Create a requirement directly from an object name and VersionRange. - - Args: - name: Object name string. - range: VersionRange object. If None, an unversioned requirement is - created. - """ - other = Requirement(None) - other.name_ = name - other.range_ = VersionRange() if range is None else range - return other - - @property - def name(self): - """Name of the required object.""" - return self.name_ - - @property - def range(self): - """VersionRange of the requirement.""" - return self.range_ - - @property - def conflict(self): - """True if the requirement is a conflict requirement, eg "!foo", "~foo-1". - """ - return self.conflict_ - - @property - def weak(self): - """True if the requirement is weak, eg "~foo". - - Note that weak requirements are also conflict requirements, but not - necessarily the other way around. - """ - return self.negate_ - - def safe_str(self): - """Return a string representation that is safe for the current filesystem, - and guarantees that no two different Requirement objects will encode to - the same value.""" - return str(self) - - def conflicts_with(self, other): - """Returns True if this requirement conflicts with another `Requirement` - or `VersionedObject`.""" - if isinstance(other, Requirement): - if (self.name_ != other.name_) or (self.range is None) \ - or (other.range is None): - return False - elif self.conflict: - return False if other.conflict \ - else self.range_.issuperset(other.range_) - elif other.conflict: - return other.range_.issuperset(self.range_) - else: - return not self.range_.intersects(other.range_) - else: # VersionedObject - if (self.name_ != other.name_) or (self.range is None): - return False - if self.conflict: - return (other.version_ in self.range_) - else: - return (other.version_ not in self.range_) - - def merged(self, other): - """Returns the merged result of two requirements. - - Two requirements can be in conflict and if so, this function returns - None. For example, requests for "foo-4" and "foo-6" are in conflict, - since both cannot be satisfied with a single version of foo. - - Some example successful requirements merges are: - - "foo-3+" and "!foo-5+" == "foo-3+<5" - - "foo-1" and "foo-1.5" == "foo-1.5" - - "!foo-2" and "!foo-5" == "!foo-2|5" - """ - if self.name_ != other.name_: - return None # cannot merge across object names - - def _r(r_): - r = Requirement(None) - r.name_ = r_.name_ - r.negate_ = r_.negate_ - r.conflict_ = r_.conflict_ - r.sep_ = r_.sep_ - return r - - if self.range is None: - return other - elif other.range is None: - return self - elif self.conflict: - if other.conflict: - r = _r(self) - r.range_ = self.range_ | other.range_ - r.negate_ = (self.negate_ and other.negate_ - and not r.range_.is_any()) - return r - else: - range_ = other.range - self.range - if range_ is None: - return None - else: - r = _r(other) - r.range_ = range_ - return r - elif other.conflict: - range_ = self.range_ - other.range_ - if range_ is None: - return None - else: - r = _r(self) - r.range_ = range_ - return r - else: - range_ = self.range_ & other.range_ - if range_ is None: - return None - else: - r = _r(self) - r.range_ = range_ - return r - - def __eq__(self, other): - return (isinstance(other, Requirement) - and (self.name_ == other.name_) - and (self.range_ == other.range_) - and (self.conflict_ == other.conflict_)) - - def __hash__(self): - return hash(str(self)) - - def __str__(self): - if self._str is None: - pre_str = '~' if self.negate_ else ('!' if self.conflict_ else '') - range_str = '' - sep_str = '' - - range_ = self.range_ - if self.negate_: - range_ = ~range_ if range_ else VersionRange() - - if not range_.is_any(): - range_str = str(range_) - if range_str[0] not in ('=', '<', '>'): - sep_str = self.sep_ - - self._str = pre_str + self.name_ + sep_str + range_str - return self._str - - -class RequirementList(_Common): - """A list of requirements. - - This class takes a Requirement list and reduces it to the equivalent - optimal form, merging any requirements for common objects. Order of objects - is retained. - """ - def __init__(self, requirements): - """Create a RequirementList. - - Args: - requirements: List of Requirement objects. - """ - self.requirements_ = [] - self.conflict_ = None - self.requirements_dict = {} - self.names_ = set() - self.conflict_names_ = set() - - for req in requirements: - existing_req = self.requirements_dict.get(req.name) - - if existing_req is None: - self.requirements_dict[req.name] = req - else: - merged_req = existing_req.merged(req) - if merged_req is None: - self.conflict_ = (existing_req, req) - return - else: - self.requirements_dict[req.name] = merged_req - - seen = set() - - # build optimised list, this intends to match original request order - # as closely as possible - for req in requirements: - if req.name not in seen: - seen.add(req.name) - req_ = self.requirements_dict[req.name] - self.requirements_.append(req_) - - if req_.conflict: - self.conflict_names_.add(req.name) - else: - self.names_.add(req.name) - - @property - def requirements(self): - """Returns optimised list of requirements, or None if there are - conflicts. - """ - return self.requirements_ - - @property - def conflict(self): - """Get the requirement conflict, if any. - - Returns: - None if there is no conflict, otherwise a 2-tuple containing the - conflicting Requirement objects. - """ - return self.conflict_ - - @property - def names(self): - """Set of names of requirements, not including conflict requirements. - """ - return self.names_ - - @property - def conflict_names(self): - """Set of conflict requirement names.""" - return self.conflict_names_ - - def __iter__(self): - for requirement in (self.requirements_ or []): - yield requirement - - def get(self, name): - """Returns the Requirement for the given object, or None. - """ - return self.requirements_dict.get(name) - - def __eq__(self, other): - return (isinstance(other, RequirementList) - and (self.requirements_ == other.requirements_) - and (self.conflict_ == other.conflict_)) - - def __str__(self): - if self.conflict_: - s1 = str(self.conflict_[0]) - s2 = str(self.conflict_[1]) - return "%s <--!--> %s" % (s1, s2) - else: - return ' '.join(str(x) for x in self.requirements_) +from rez.version._requirement import * diff --git a/src/rez/vendor/version/test.py b/src/rez/vendor/version/test.py deleted file mode 100644 index 8f8919fa5..000000000 --- a/src/rez/vendor/version/test.py +++ /dev/null @@ -1,505 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# Copyright Contributors to the Rez Project - - -from rez.vendor.version.version import Version, AlphanumericVersionToken, \ - VersionRange, reverse_sort_key, _ReversedComparable -from rez.vendor.version.requirement import Requirement, RequirementList -from rez.vendor.version.util import VersionError -import random -import textwrap -import unittest - - - -def _print(txt=''): - # uncomment for verbose output - #print txt - pass - - -class TestVersionSchema(unittest.TestCase): - make_token = AlphanumericVersionToken - - def __init__(self, fn): - unittest.TestCase.__init__(self, fn) - - def _test_strict_weak_ordering(self, a, b): - self.assertTrue(a == a) - self.assertTrue(b == b) - - e = (a == b) - ne = (a != b) - lt = (a < b) - lte = (a <= b) - gt = (a > b) - gte = (a >= b) - - _print('\n' + textwrap.dedent( - """ - '%s' '%s' - ==: %s - !=: %s - <: %s - <=: %s - >: %s - >=: %s - """).strip() % (a, b, e, ne, lt, lte, gt, gte)) - - self.assertTrue(e != ne) - if e: - self.assertTrue(not lt) - self.assertTrue(not gt) - self.assertTrue(lte) - self.assertTrue(gte) - else: - self.assertTrue(lt != gt) - self.assertTrue(lte != gte) - self.assertTrue(lt == lte) - self.assertTrue(gt == gte) - - if not isinstance(a, _ReversedComparable): - self._test_strict_weak_ordering(reverse_sort_key(a), - reverse_sort_key(b)) - - def _test_ordered(self, items): - def _test(fn, items_, op_str): - for i, a in enumerate(items_): - for b in items_[i+1:]: - _print("'%s' %s '%s'" % (a, op_str, b)) - self.assertTrue(fn(a, b)) - - _test(lambda a, b: a < b, items, '<') - _test(lambda a, b: a <= b, items, '<=') - _test(lambda a, b: a != b, items, '!=') - _test(lambda a, b: a > b, list(reversed(items)), '>') - _test(lambda a, b: a >= b, list(reversed(items)), '>=') - _test(lambda a, b: a != b, list(reversed(items)), '!=') - - def _create_random_token(self): - s = self.make_token.create_random_token_string() - return self.make_token(s) - - def _create_random_version(self): - ver_str = '.'.join(self.make_token.create_random_token_string() - for i in range(random.randint(0, 6))) - return Version(ver_str, make_token=self.make_token) - - def test_misc(self): - self.assertEqual(Version("1.2.12").as_tuple(), ("1", "2", "12")) - - def test_token_strict_weak_ordering(self): - # test equal tokens - tok = self._create_random_token() - self._test_strict_weak_ordering(tok, tok) - - # test random tokens - for i in range(100): - tok1 = self._create_random_token() - tok2 = self._create_random_token() - self._test_strict_weak_ordering(tok1, tok2) - - def test_version_strict_weak_ordering(self): - # test equal versions - ver = self._create_random_version() - self._test_strict_weak_ordering(ver, ver) - - # test random versions - for i in range(100): - ver1 = self._create_random_version() - ver2 = self._create_random_version() - self._test_strict_weak_ordering(ver1, ver2) - - def test_token_comparisons(self): - def _lt(a, b): - _print("'%s' < '%s'" % (a, b)) - self.assertTrue(self.make_token(a) < self.make_token(b)) - self.assertTrue(Version(a) < Version(b)) - - _print() - _lt("3", "4") - _lt("01", "1") - _lt("beta", "1") - _lt("alpha3", "alpha4") - _lt("alpha", "alpha3") - _lt("gamma33", "33gamma") - - def test_version_comparisons(self): - def _eq(a, b): - _print("'%s' == '%s'" % (a, b)) - self.assertTrue(Version(a) == Version(b)) - - _print() - _eq("", "") - _eq("1", "1") - _eq("1.2", "1-2") - _eq("1.2-3", "1-2.3") - - ascending = ["", - "0.0.0", - "1", - "2", - "2.alpha1", - "2.alpha2", - "2.beta", - "2.0", - "2.0.8.8", - "2.1", - "2.1.0"] - self._test_ordered([Version(x) for x in ascending]) - - def _eq2(a, b): - _print("'%s' == '%s'" % (a, b)) - self.assertTrue(a == b) - - # test behaviour in sets - a = Version("1.0") - b = Version("1.0") - c = Version("1.0alpha") - d = Version("2.0.0") - - _eq2(set([a]) - set([a]), set()) - _eq2(set([a]) - set([b]), set()) - _eq2(set([a, a]) - set([a]), set()) - _eq2(set([b, c, d]) - set([a]), set([c, d])) - _eq2(set([b, c]) | set([c, d]), set([b, c, d])) - _eq2(set([b, c]) & set([c, d]), set([c])) - - def test_version_range(self): - def _eq(a, b): - _print("'%s' == '%s'" % (a, b)) - a_range = VersionRange(a) - b_range = VersionRange(b) - - self.assertTrue(a_range == b_range) - self.assertTrue(a_range.issuperset(a_range)) - self.assertTrue(a_range.issuperset(b_range)) - self.assertTrue(VersionRange(str(a_range)) == a_range) - self.assertTrue(VersionRange(str(b_range)) == a_range) - self.assertTrue(hash(a_range) == hash(b_range)) - - a_ = a.replace('.', '-') - a_ = a_.replace("--", "..") - a_range_ = VersionRange(a_) - self.assertTrue(a_range_ == a_range) - self.assertTrue(hash(a_range_) == hash(a_range)) - - range_strs = a.split('|') - ranges = [VersionRange(x) for x in range_strs] - ranges_ = ranges[0].union(ranges[1:]) - self.assertTrue(ranges_ == a_range) - - self.assertTrue(a_range | b_range == a_range) - self.assertTrue(a_range - b_range is None) - self.assertTrue(b_range - a_range is None) - self.assertTrue(VersionRange() & a_range == a_range) - self.assertTrue(b_range.span() & a_range == a_range) - - a_inv = a_range.inverse() - self.assertTrue(a_inv == ~b_range) - - if a_inv: - self.assertTrue(~a_inv == a_range) - self.assertTrue(a_range | a_inv == VersionRange()) - self.assertTrue(a_range & a_inv is None) - - a_ranges = a_range.split() - a_range_ = a_ranges[0].union(a_ranges[1:]) - self.assertTrue(a_range_ == b_range) - - def _and(a, b, c): - _print("'%s' & '%s' == '%s'" % (a, b, c)) - a_range = VersionRange(a) - b_range = VersionRange(b) - c_range = None if c is None else VersionRange(c) - self.assertTrue(a_range & b_range == c_range) - self.assertTrue(b_range & a_range == c_range) - - a_or_b = a_range | b_range - a_and_b = a_range & b_range - a_sub_b = a_range - b_range - b_sub_a = b_range - a_range - ranges = [a_and_b, a_sub_b, b_sub_a] - ranges = [x for x in ranges if x] - self.assertTrue(ranges[0].union(ranges[1:]) == a_or_b) - - def _inv(a, b): - a_range = VersionRange(a) - b_range = VersionRange(b) - self.assertTrue(~a_range == b_range) - self.assertTrue(~b_range == a_range) - self.assertTrue(a_range | b_range == VersionRange()) - self.assertTrue(a_range & b_range is None) - - # simple cases - _print() - _eq("", "") - _eq("1", "1") - _eq("1.0.0", "1.0.0") - _eq("3+<3_", "3") - _eq("_+<__", "_") - _eq("1.2+<=2.0", "1.2..2.0") - _eq("10+,<20", "10+<20") - _eq("1+<1.0", "1+<1.0") - _eq(">=2", "2+") - _eq(">=1.21.1,<1.23", ">=1.21.1<1.23") - _eq(">1.21.1,<1.23", ">1.21.1<1.23") - _eq(">1.21.1<1.23", ">1.21.1<1.23") - _eq(">1.21.1,<=1.23", ">1.21.1<=1.23") - - # Reverse order which is a syntax pip packages use more often now. - # Only allowed when separated by a comma. - _eq("<1.23,>=1.21.1", ">=1.21.1<1.23") - _eq("<1.23,>1.21.1", ">1.21.1<1.23") - - # optimised cases - _eq("3|3", "3") - _eq("3|1", "1|3") - _eq("5|3|1", "1|3|5") - _eq("1|1_", "1+<1__") - _eq("1|1_|1__", "1+,<1___") - _eq("|", "") - _eq("||", "||||||||") - _eq("1|1_+", "1+") - _eq("<1|1", "<1_") - _eq("1+<3|3+<5", "1+<5") - _eq(">4<6|1+<3", "1+<3|>4,<6") - _eq("4+<6|1+<3|", "") - _eq("4|2+", "2+") - _eq("3|<5", "<5") - _eq("<3|>3", ">3|<3") - _eq("3+|<3", "") - _eq("3+|<4", "") - _eq("2+<=6|3+<5", "2..6") - _eq("3+,<5|2+<=6", "2+<=6") - _eq("2|2+", "2+") - _eq("2|2.1+", "2+") - _eq("2|<2.1", "<2_") - _eq("3..3", "==3") - _eq(">=3,<=3", "==3") - - # AND'ing - _and("3", "3", "3") - _and("1", "==1", "==1") - _and("", "==1", "==1") - _and("3", "4", None) - _and("<3", "5+", None) - _and("4+<6", "6+<8", None) - _and("2+", "<=4", "2..4") - _and("1", "1.0", "1.0") - _and("4..6", "6+<8", "==6") - - # inverse - _inv("3+", "<3") - _inv("<=3", ">3") - _inv("3.5", "<3.5|3.5_+") - self.assertTrue(~VersionRange() is None) - - # odd (but valid) cases - _eq(">", ">") # greater than the empty version - _eq("+", "") # greater or equal to empty version (is all vers) - _eq(">=", "") # equivalent to above - _eq("<=", "==") # less or equal to empty version (is only empty) - _eq("..", "==") # from empty version to empty version - _eq("+<=", "==") # equivalent to above - - invalid_range = [ - "4+<2", # lower bound greater than upper - ">3<3", # both greater and less than same version - ">3<=3", # greater and less or equal to same version - "3+<3" # greater and equal to, and less than, same version - ] - - for s in invalid_range: - self.assertRaises(VersionError, VersionRange, s) - - invalid_syntax = [ - "<", # less than the empty version - "><", # both greater and less than empty version - ">3>4", # both are lower bounds - "<3<4", # both are upper bounds - "<4>3", # upper bound before lower without comma - ",<4", # leading comma - "4+,", # trailing comma - "1>=", # pre-lower-op in post - "+1", # post-lower-op in pre - "4<", # pre-upper-op in post - "1+<2<3" # more than two bounds - ] - - for s in invalid_syntax: - self.assertRaises(VersionError, VersionRange, s) - - # test simple logic - self.assertTrue(VersionRange("").is_any()) - self.assertTrue(VersionRange("2+<4").bounded()) - self.assertTrue(VersionRange("2+").lower_bounded()) - self.assertTrue(not VersionRange("2+").upper_bounded()) - self.assertTrue(not VersionRange("2+").bounded()) - self.assertTrue(VersionRange("<2").upper_bounded()) - self.assertTrue(not VersionRange("<2").lower_bounded()) - self.assertTrue(not VersionRange("<2").bounded()) - - # test range from version(s) - v = Version("3") - self.assertTrue(VersionRange.from_version(v, "eq") == VersionRange("==3")) - self.assertTrue(VersionRange.from_version(v, "gt") == VersionRange(">3")) - self.assertTrue(VersionRange.from_version(v, "gte") == VersionRange("3+")) - self.assertTrue(VersionRange.from_version(v, "lt") == VersionRange("<3")) - self.assertTrue(VersionRange.from_version(v, "lte") == VersionRange("<=3")) - - range1 = VersionRange.from_version(Version("2"), "gte") - range2 = VersionRange.from_version(Version("4"), "lte") - _eq(str(range1 & range2), "2..4") - - v2 = Version("6.0") - v3 = Version("4") - self.assertTrue(VersionRange.from_versions([v, v2, v3]) - == VersionRange("==3|==4|==6.0")) - - # test behaviour in sets - def _eq2(a, b): - _print("'%s' == '%s'" % (a, b)) - self.assertTrue(a == b) - - a = VersionRange("1+<=2.5") - b = VersionRange("1..2.5") - c = VersionRange(">=5") - d = VersionRange(">6.1.0") - e = VersionRange("3.2") - - _eq2(set([a]) - set([a]), set()) - _eq2(set([a]) - set([b]), set()) - _eq2(set([a, a]) - set([a]), set()) - _eq2(set([b, c, d, e]) - set([a]), set([c, d, e])) - _eq2(set([b, c, e]) | set([c, d]), set([b, c, d, e])) - _eq2(set([b, c]) & set([c, d]), set([c])) - - def test_containment(self): - # basic containment - self.assertTrue(Version("3") in VersionRange("3+")) - self.assertTrue(Version("5") in VersionRange("3..5")) - self.assertTrue(Version("5_") not in VersionRange("3..5")) - self.assertTrue(Version("3.0.0") in VersionRange("3+")) - self.assertTrue(Version("3.0.0") not in VersionRange("3.1+")) - self.assertTrue(Version("3") in VersionRange("<1|5|6|8|7|3|60+")) - self.assertTrue(Version("3") in VersionRange("<1|5|6|8|7|==3|60+")) - self.assertTrue(VersionRange("2.1+<4") in VersionRange("<4")) - self.assertTrue(VersionRange("2.1..4") not in VersionRange("<4")) - self.assertTrue(VersionRange("3") in VersionRange("3")) - self.assertTrue(VersionRange("==3") in VersionRange("3")) - self.assertTrue(VersionRange("3.5+<3_") in VersionRange("3")) - self.assertTrue(VersionRange("3") not in VersionRange("4+<6")) - self.assertTrue(VersionRange("3+<10") not in VersionRange("4+<6")) - - # iterating over sorted version list - numbers = [2, 3, 5, 10, 11, 13, 14] - versions = [Version(str(x)) for x in numbers] - rev_versions = list(reversed(versions)) - composite_range = VersionRange.from_versions(versions) - - entries = [(VersionRange(""), 7), - (VersionRange("0+"), 7), - (VersionRange("5+"), 5), - (VersionRange("6+"), 4), - (VersionRange("50+"), 0), - (VersionRange(">5"), 4), - (VersionRange("5"), 1), - (VersionRange("6"), 0), - (VersionRange("<5"), 2), - (VersionRange("<6"), 3), - (VersionRange("<50"), 7), - (VersionRange("<=5"), 3), - (VersionRange("<1"), 0), - (VersionRange("2|9+"), 5), - (VersionRange("3+<6|12+<13.5"), 3), - (VersionRange("<1|20+"), 0), - (VersionRange(">0<20"), 7)] - - for range_, count in entries: - # brute-force containment tests - matches = set(x for x in versions if x in range_) - self.assertEqual(len(matches), count) - - # more optimal containment tests - def _test_it(it): - matches_ = set(version for contains, version in it if contains) - self.assertEqual(matches_, matches) - - _test_it(range_.iter_intersect_test(versions)) - _test_it(range_.iter_intersect_test(rev_versions, descending=True)) - - # throw in an intersection test - self.assertEqual(composite_range.intersects(range_), (count != 0)) - int_range = composite_range & range_ - versions_ = [] if int_range is None else int_range.to_versions() - self.assertEqual(set(versions_), matches) - - # throw in a superset test as well - self.assertEqual(range_.issuperset(composite_range), (count == 7)) - if count: - self.assertTrue(composite_range.issuperset(int_range)) - - def test_requirement_list(self): - def _eq(reqs, expected_reqs): - _print("requirements(%s) == requirements(%s)" - % (' '.join(reqs), ' '.join(expected_reqs))) - reqs_ = [Requirement(x) for x in reqs] - reqlist = RequirementList(reqs_) - _print("result: %s" % str(reqlist)) - - exp_reqs_ = [Requirement(x) for x in expected_reqs] - self.assertTrue(reqlist.requirements == exp_reqs_) - - exp_names = set(x.name for x in exp_reqs_ if not x.conflict) - self.assertTrue(reqlist.names == exp_names) - - exp_confl_names = set(x.name for x in exp_reqs_ if x.conflict) - self.assertTrue(reqlist.conflict_names == exp_confl_names) - - def _confl(reqs, a, b): - _print("requirements(%s) == %s <--!--> %s" % (' '.join(reqs), a, b)) - reqs_ = [Requirement(x) for x in reqs] - reqlist = RequirementList(reqs_) - _print("result: %s" % str(reqlist)) - - a_req = Requirement(a) - b_req = Requirement(b) - self.assertTrue(reqlist.conflict == (a_req, b_req)) - - _print() - _eq(["foo"], - ["foo"]) - _eq(["foo", "bah"], - ["foo", "bah"]) - _eq(["bah", "foo"], - ["bah", "foo"]) - _eq(["foo-4+", "foo-4.5"], - ["foo-4.5"]) - _eq(["bah-2.4", "foo", "bah-2.4.1+"], - ["bah-2.4.1+<2.4_", "foo"]) - _eq(["foo-2+", "!foo-4+"], - ["foo-2+<4"]) - _eq(["!bah-1", "!bah-3"], - ["!bah-1|3"]) - _eq(["!bah-5", "foo-2.3", "!bah-5.6+"], - ["!bah-5+", "foo-2.3"]) - _eq(["~bah-4", "foo", "bah<4.2"], - ["bah-4+<4.2", "foo"]) - _eq(["~bah", "!foo", "bah<4.2"], - ["bah<4.2", "!foo"]) - _eq(["~bah-3+", "~bah-5"], - ["~bah-5"]) - - _confl(["foo-1", "foo-2"], - "foo-1", "foo-2") - _confl(["foo-2", "foo-1"], - "foo-2", "foo-1") - _confl(["foo", "~bah-5+", "bah-2"], - "~bah-5+", "bah-2") - _confl(["foo", "~bah-5+", "bah-7..12", "bah-2"], - "bah-7..12", "bah-2") - - -if __name__ == '__main__': - unittest.main() diff --git a/src/rez/vendor/version/util.py b/src/rez/vendor/version/util.py index 47da793b6..49dddd77c 100644 --- a/src/rez/vendor/version/util.py +++ b/src/rez/vendor/version/util.py @@ -2,29 +2,4 @@ # Copyright Contributors to the Rez Project -from itertools import groupby - - -class VersionError(Exception): - pass - - -class ParseException(Exception): - pass - - -class _Common(object): - def __str__(self): - raise NotImplementedError - - def __ne__(self, other): - return not (self == other) - - def __repr__(self): - return "%s(%r)" % (self.__class__.__name__, str(self)) - - -def dedup(iterable): - """Removes duplicates from a sorted sequence.""" - for e in groupby(iterable): - yield e[0] +from rez.version._util import * diff --git a/src/rez/vendor/version/version.py b/src/rez/vendor/version/version.py index fb7a85c09..f3b7c54d7 100644 --- a/src/rez/vendor/version/version.py +++ b/src/rez/vendor/version/version.py @@ -2,1491 +2,4 @@ # Copyright Contributors to the Rez Project -""" -Implements a well defined versioning schema. - -There are three class types - VersionToken, Version and VersionRange. A Version -is a set of zero or more VersionTokens, separate by '.'s or '-'s (eg "1.2-3"). -A VersionToken is a string containing alphanumerics, and default implemenations -'NumericToken' and 'AlphanumericVersionToken' are supplied. You can implement -your own if you want stricter tokens or different sorting behaviour. - -A VersionRange is a set of one or more contiguous version ranges - for example, -"3+<5" contains any version >=3 but less than 5. Version ranges can be used to -define dependency requirements between objects. They can be OR'd together, AND'd -and inverted. - -The empty version '', and empty version range '', are also handled. The empty -version is used to denote unversioned objects. The empty version range, also -known as the 'any' range, is used to refer to any version of an object. -""" -from __future__ import print_function -from .util import VersionError, ParseException, _Common, \ - dedup -import rez.vendor.pyparsing.pyparsing as pp -from bisect import bisect_left -import copy -import string -import re - - -re_token = re.compile(r"[a-zA-Z0-9_]+") - - -class _Comparable(_Common): - def __gt__(self, other): - return not (self < other or self == other) - - def __le__(self, other): - return self < other or self == other - - def __ge__(self, other): - return not self < other - - -class _ReversedComparable(_Common): - def __init__(self, value): - self.value = value - - def __lt__(self, other): - return not (self.value < other.value) - - def __gt__(self, other): - return not (self < other or self == other) - - def __le__(self, other): - return self < other or self == other - - def __ge__(self, other): - return not self < other - - def __str__(self): - return "reverse(%s)" % str(self.value) - - def __repr__(self): - return "reverse(%r)" % self.value - - -class VersionToken(_Comparable): - """Token within a version number. - - A version token is that part of a version number that appears between a - delimiter, typically '.' or '-'. For example, the version number '2.3.07b' - contains the tokens '2', '3' and '07b' respectively. - - Version tokens are only allowed to contain alphanumerics (any case) and - underscores. - """ - def __init__(self, token): - """Create a VersionToken. - - Args: - token: Token string, eg "rc02" - """ - raise NotImplementedError - - @classmethod - def create_random_token_string(cls): - """Create a random token string. For testing purposes only.""" - raise NotImplementedError - - def less_than(self, other): - """Compare to another VersionToken. - - Args: - other: The VersionToken object to compare against. - - Returns: - True if this token is less than other, False otherwise. - """ - raise NotImplementedError - - def next(self): - """Returns the next largest token.""" - raise NotImplementedError - - def __str__(self): - raise NotImplementedError - - def __lt__(self, other): - return self.less_than(other) - - def __eq__(self, other): - return (not self < other) and (not other < self) - - -class NumericToken(VersionToken): - """Numeric version token. - - Version token supporting numbers only. Padding is ignored. - """ - def __init__(self, token): - if not token.isdigit(): - raise VersionError("Invalid version token: '%s'" % token) - else: - self.n = int(token) - - @classmethod - def create_random_token_string(cls): - import random - chars = string.digits - return ''.join([chars[random.randint(0, len(chars) - 1)] - for _ in range(8)]) - - def __str__(self): - return str(self.n) - - def __eq__(self, other): - return (self.n == other.n) - - def less_than(self, other): - return (self.n < other.n) - - def __next__(self): - other = copy.copy(self) - other.n = self.n = 1 - return other - - def next(self): - return self.__next__() - - -class _SubToken(_Comparable): - """Used internally by AlphanumericVersionToken.""" - def __init__(self, s): - self.s = s - self.n = int(s) if s.isdigit() else None - - def __lt__(self, other): - if self.n is None: - return (self.s < other.s) if other.n is None else True - else: - return False if other.n is None \ - else ((self.n, self.s) < (other.n, other.s)) - - def __eq__(self, other): - return (self.s == other.s) and (self.n == other.n) - - def __str__(self): - return self.s - - -class AlphanumericVersionToken(VersionToken): - """Alphanumeric version token. - - These tokens compare as follows: - - each token is split into alpha and numeric groups (subtokens); - - the resulting subtoken list is compared. - - alpha comparison is case-sensitive, numeric comparison is padding-sensitive. - - Subtokens compare as follows: - - alphas come before numbers; - - alphas are compared alphabetically (_, then A-Z, then a-z); - - numbers are compared numerically. If numbers are equivalent but zero- - padded differently, they are then compared alphabetically. Thus "01" < "1". - - Some example comparisons that equate to true: - - "3" < "4" - - "01" < "1" - - "beta" < "1" - - "alpha3" < "alpha4" - - "alpha" < "alpha3" - - "gamma33" < "33gamma" - """ - numeric_regex = re.compile("[0-9]+") - regex = re.compile(r"[a-zA-Z0-9_]+\Z") - - def __init__(self, token): - if token is None: - self.subtokens = None - elif not self.regex.match(token): - raise VersionError("Invalid version token: '%s'" % token) - else: - self.subtokens = self._parse(token) - - @classmethod - def create_random_token_string(cls): - import random - chars = string.digits + string.ascii_letters - return ''.join([chars[random.randint(0, len(chars) - 1)] - for _ in range(8)]) - - def __str__(self): - return ''.join(map(str, self.subtokens)) - - def __eq__(self, other): - return (self.subtokens == other.subtokens) - - def less_than(self, other): - return (self.subtokens < other.subtokens) - - def __next__(self): - other = AlphanumericVersionToken(None) - other.subtokens = self.subtokens[:] - subtok = other.subtokens[-1] - if subtok.n is None: - other.subtokens[-1] = _SubToken(subtok.s + '_') - else: - other.subtokens.append(_SubToken('_')) - return other - - def next(self): - return self.__next__() - - @classmethod - def _parse(cls, s): - subtokens = [] - alphas = cls.numeric_regex.split(s) - numerics = cls.numeric_regex.findall(s) - b = True - - while alphas or numerics: - if b: - alpha = alphas[0] - alphas = alphas[1:] - if alpha: - subtokens.append(_SubToken(alpha)) - else: - numeric = numerics[0] - numerics = numerics[1:] - subtokens.append(_SubToken(numeric)) - b = not b - - return subtokens - - -def reverse_sort_key(comparable): - """Key that gives reverse sort order on versions and version ranges. - - Example: - - >>> Version("1.0") < Version("2.0") - True - >>> reverse_sort_key(Version("1.0")) < reverse_sort_key(Version("2.0")) - False - - Args: - comparable (`Version` or `VesionRange`): Object to wrap. - - Returns: - `_ReversedComparable`: Wrapper object that reverses comparisons. - """ - return _ReversedComparable(comparable) - - -class Version(_Comparable): - """Version object. - - A Version is a sequence of zero or more version tokens, separated by either - a dot '.' or hyphen '-' delimiters. Note that separators only affect Version - objects cosmetically - in other words, the version '1.0.0' is equivalent to - '1-0-0'. - - The empty version '' is the smallest possible version, and can be used to - represent an unversioned resource. - """ - inf = None - - def __init__(self, ver_str='', make_token=AlphanumericVersionToken): - """Create a Version object. - - Args: - ver_str: Version string. - make_token: Callable that creates a VersionToken subclass from a - string. - """ - self.tokens = [] - self.seps = [] - self._str = None - self._hash = None - - if ver_str: - toks = re_token.findall(ver_str) - if not toks: - raise VersionError(ver_str) - - seps = re_token.split(ver_str) - if seps[0] or seps[-1] or max(len(x) for x in seps) > 1: - raise VersionError("Invalid version syntax: '%s'" % ver_str) - - for tok in toks: - try: - self.tokens.append(make_token(tok)) - except VersionError as e: - raise VersionError("Invalid version '%s': %s" - % (ver_str, str(e))) - - self.seps = seps[1:-1] - - def copy(self): - """Returns a copy of the version.""" - other = Version(None) - other.tokens = self.tokens[:] - other.seps = self.seps[:] - return other - - def trim(self, len_): - """Return a copy of the version, possibly with less tokens. - - Args: - len_ (int): New version length. If >= current length, an - unchanged copy of the version is returned. - """ - other = Version(None) - other.tokens = self.tokens[:len_] - other.seps = self.seps[:len_ - 1] - return other - - def __next__(self): - """Return 'next' version. Eg, next(1.2) is 1.2_""" - if self.tokens: - other = self.copy() - tok = other.tokens.pop() - other.tokens.append(tok.next()) - return other - else: - return Version.inf - - def next(self): - return self.__next__() - - @property - def major(self): - """Semantic versioning major version.""" - return self[0] - - @property - def minor(self): - """Semantic versioning minor version.""" - return self[1] - - @property - def patch(self): - """Semantic versioning patch version.""" - return self[2] - - def as_tuple(self): - """Convert to a tuple of strings. - - Example: - - >>> print Version("1.2.12").as_tuple() - ('1', '2', '12') - """ - return tuple(map(str, self.tokens)) - - def __len__(self): - return len(self.tokens or []) - - def __getitem__(self, index): - try: - return (self.tokens or [])[index] - except IndexError: - raise IndexError("version token index out of range") - - def __nonzero__(self): - """The empty version equates to False.""" - return bool(self.tokens) - - __bool__ = __nonzero__ # py3 compat - - def __eq__(self, other): - return isinstance(other, Version) and self.tokens == other.tokens - - def __lt__(self, other): - if self.tokens is None: - return False - elif other.tokens is None: - return True - else: - return (self.tokens < other.tokens) - - def __hash__(self): - if self._hash is None: - self._hash = hash(None) if self.tokens is None \ - else hash(tuple(map(str, self.tokens))) - return self._hash - - def __str__(self): - if self._str is None: - self._str = "[INF]" if self.tokens is None \ - else ''.join(str(x) + y for x, y in zip(self.tokens, self.seps + [''])) - return self._str - - -# internal use only -Version.inf = Version() -Version.inf.tokens = None - - -class _LowerBound(_Comparable): - min = None - - def __init__(self, version, inclusive): - self.version = version - self.inclusive = inclusive - - def __str__(self): - if self.version: - s = "%s+" if self.inclusive else ">%s" - return s % self.version - else: - return '' if self.inclusive else ">" - - def __eq__(self, other): - return (self.version == other.version) \ - and (self.inclusive == other.inclusive) - - def __lt__(self, other): - return (self.version < other.version) \ - or ((self.version == other.version) - and (self.inclusive and not other.inclusive)) - - def __hash__(self): - return hash((self.version, self.inclusive)) - - def contains_version(self, version): - return (version > self.version) \ - or (self.inclusive and (version == self.version)) - -_LowerBound.min = _LowerBound(Version(), True) - - -class _UpperBound(_Comparable): - inf = None - - def __init__(self, version, inclusive): - self.version = version - self.inclusive = inclusive - if not version and not inclusive: - raise VersionError("Invalid upper bound: '%s'" % str(self)) - - def __str__(self): - s = "<=%s" if self.inclusive else "<%s" - return s % self.version - - def __eq__(self, other): - return (self.version == other.version) \ - and (self.inclusive == other.inclusive) - - def __lt__(self, other): - return (self.version < other.version) \ - or ((self.version == other.version) - and (not self.inclusive and other.inclusive)) - - def __hash__(self): - return hash((self.version, self.inclusive)) - - def contains_version(self, version): - return (version < self.version) \ - or (self.inclusive and (version == self.version)) - -_UpperBound.inf = _UpperBound(Version.inf, True) - - -class _Bound(_Comparable): - any = None - - def __init__(self, lower=None, upper=None, invalid_bound_error=True): - self.lower = lower or _LowerBound.min - self.upper = upper or _UpperBound.inf - - if (invalid_bound_error and - (self.lower.version > self.upper.version - or ((self.lower.version == self.upper.version) - and not (self.lower.inclusive and self.upper.inclusive)))): - raise VersionError("Invalid bound") - - def __str__(self): - if self.upper.version == Version.inf: - return str(self.lower) - elif self.lower.version == self.upper.version: - return "==%s" % str(self.lower.version) - elif self.lower.inclusive and self.upper.inclusive: - if self.lower.version: - return "%s..%s" % (self.lower.version, self.upper.version) - else: - return "<=%s" % self.upper.version - elif (self.lower.inclusive and not self.upper.inclusive) \ - and (self.lower.version.next() == self.upper.version): - return str(self.lower.version) - else: - return "%s%s" % (self.lower, self.upper) - - def __eq__(self, other): - return (self.lower == other.lower) and (self.upper == other.upper) - - def __lt__(self, other): - return (self.lower, self.upper) < (other.lower, other.upper) - - def __hash__(self): - return hash((self.lower, self.upper)) - - def lower_bounded(self): - return (self.lower != _LowerBound.min) - - def upper_bounded(self): - return (self.upper != _UpperBound.inf) - - def contains_version(self, version): - return (self.version_containment(version) == 0) - - def version_containment(self, version): - if not self.lower.contains_version(version): - return -1 - if not self.upper.contains_version(version): - return 1 - return 0 - - def contains_bound(self, bound): - return (self.lower <= bound.lower) and (self.upper >= bound.upper) - - def intersects(self, other): - lower = max(self.lower, other.lower) - upper = min(self.upper, other.upper) - - return (lower.version < upper.version) or \ - ((lower.version == upper.version) and - (lower.inclusive and upper.inclusive)) - - def intersection(self, other): - lower = max(self.lower, other.lower) - upper = min(self.upper, other.upper) - - if (lower.version < upper.version) or \ - ((lower.version == upper.version) and - (lower.inclusive and upper.inclusive)): - return _Bound(lower, upper) - else: - return None - -_Bound.any = _Bound() - - -class _VersionRangeParser(object): - debug = False # set to True to enable parser debugging - - re_flags = (re.VERBOSE | re.DEBUG) if debug else re.VERBOSE - - # The regular expression for a version - one or more version tokens - # followed by a non-capturing group of version separator followed by - # one or more version tokens. - # - # Note that this assumes AlphanumericVersionToken-based versions! - # - # TODO - Would be better to have `VersionRange` keep a static dict of - # parser instances, per token class type. We would add a 'regex' static - # string to each token class, and that could be used to construct - # `version_group` as below. We need to keep a dict of these parser instances, - # to avoid recompiling the large regex every time a version range is - # instantiated. In the cpp port this would be simpler - VersionRange could - # just have a static parser that is instantiated when the version range - # template class is instantiated. - # - version_group = r"([0-9a-zA-Z_]+(?:[.-][0-9a-zA-Z_]+)*)" - - version_range_regex = ( - # Match a version number (e.g. 1.0.0) - r" ^(?P{version_group})$" - "|" - # Or match an exact version number (e.g. ==1.0.0) - " ^(?P" - " ==" # Required == operator - " (?P{version_group})?" - " )$" - "|" - # Or match an inclusive bound (e.g. 1.0.0..2.0.0) - " ^(?P" - " (?P{version_group})?" - " \.\." # Required .. operator - " (?P{version_group})?" - " )$" - "|" - # Or match a lower bound (e.g. 1.0.0+) - " ^(?P" - " (?P>|>=)?" # Bound is exclusive? - " (?P{version_group})?" - " (?(lower_bound_prefix)|\+)" # + only if bound is not exclusive - " )$" - "|" - # Or match an upper bound (e.g. <=1.0.0) - " ^(?P" - " (?P<(?={version_group})|<=)?" # Bound is exclusive? - " (?P{version_group})?" - " )$" - "|" - # Or match a range in ascending order (e.g. 1.0.0+<2.0.0) - " ^(?P" - " (?P" - " (?P>|>=)?" # Lower bound is exclusive? - " (?P{version_group})?" - " (?(range_lower_asc_prefix)|\+)?" # + only if lower bound is not exclusive - " )(?P" - " (?(range_lower_asc_version),?|)" # , only if lower bound is found - " (?P<(?={version_group})|<=)" # <= only if followed by a version group - " (?P{version_group})?" - " )" - " )$" - "|" - # Or match a range in descending order (e.g. <=2.0.0,1.0.0+) - " ^(?P" - " (?P" - " (?P<|<=)?" # Upper bound is exclusive? - " (?P{version_group})?" - " (?(range_upper_desc_prefix)|\+)?" # + only if upper bound is not exclusive - " )(?P" - " (?(range_upper_desc_version),|)" # Comma is not optional because we don't want to recognize something like "<4>3" - " (?P<(?={version_group})|>=?)" # >= or > only if followed by a version group - " (?P{version_group})?" - " )" - " )$" - ).format(version_group=version_group) - - regex = re.compile(version_range_regex, re_flags) - - def __init__(self, input_string, make_token, invalid_bound_error=True): - self.make_token = make_token - self._groups = {} - self._input_string = input_string - self.bounds = [] - self.invalid_bound_error = invalid_bound_error - - is_any = False - - for part in input_string.split("|"): - if part == '': - # OR'ing anthing with the 'any' version range ('') will also - # give the any range. Note that we can't early out here, as we - # need to validate that the rest of the string is syntactically - # correct - # - is_any = True - self.bounds = [] - continue - - match = re.search(self.regex, part) - if not match: - raise ParseException("Syntax error in version range '%s'" % part) - - if is_any: - # we're already the 'any' range regardless, so avoid more work - continue - - self._groups = match.groupdict() - - # Note: the following is ordered by approx likelihood of use - - if self._groups['range_asc']: - self._act_lower_and_upper_bound_asc() - - elif self._groups['version']: - self._act_version() - - elif self._groups['lower_bound']: - self._act_lower_bound() - - elif self._groups['exact_version']: - self._act_exact_version() - - elif self._groups['range_desc']: - self._act_lower_and_upper_bound_desc() - - elif self._groups['inclusive_bound']: - self._act_bound() - - elif self._groups['upper_bound']: - self._act_upper_bound() - - def _is_lower_bound_exclusive(self, token): - return (token == ">") - - def _is_upper_bound_exclusive(self, token): - return (token == "<") - - def _create_version_from_token(self, token): - return Version(token, make_token=self.make_token) - - def action(fn): - def fn_(self): - result = fn(self) - if self.debug: - label = fn.__name__.replace("_act_", "") - print("%-21s: %s" % (label, self._input_string)) - for key, value in self._groups.items(): - print(" %-17s= %s" % (key, value)) - print(" %-17s= %s" % ("bounds", self.bounds)) - return result - return fn_ - - @action - def _act_version(self): - version = self._create_version_from_token(self._groups['version']) - lower_bound = _LowerBound(version, True) - upper_bound = _UpperBound(version.next(), False) if version else None - - self.bounds.append(_Bound(lower_bound, upper_bound)) - - @action - def _act_exact_version(self): - version = self._create_version_from_token(self._groups['exact_version_group']) - lower_bound = _LowerBound(version, True) - upper_bound = _UpperBound(version, True) - - self.bounds.append(_Bound(lower_bound, upper_bound)) - - @action - def _act_bound(self): - lower_version = self._create_version_from_token(self._groups['inclusive_lower_version']) - lower_bound = _LowerBound(lower_version, True) - - upper_version = self._create_version_from_token(self._groups['inclusive_upper_version']) - upper_bound = _UpperBound(upper_version, True) - - self.bounds.append(_Bound(lower_bound, upper_bound, self.invalid_bound_error)) - - @action - def _act_lower_bound(self): - version = self._create_version_from_token(self._groups['lower_version']) - exclusive = self._is_lower_bound_exclusive(self._groups['lower_bound_prefix']) - lower_bound = _LowerBound(version, not exclusive) - - self.bounds.append(_Bound(lower_bound, None)) - - @action - def _act_upper_bound(self): - version = self._create_version_from_token(self._groups['upper_version']) - exclusive = self._is_upper_bound_exclusive(self._groups['upper_bound_prefix']) - upper_bound = _UpperBound(version, not exclusive) - - self.bounds.append(_Bound(None, upper_bound)) - - @action - def _act_lower_and_upper_bound_asc(self): - lower_bound = None - upper_bound = None - - if self._groups['range_lower_asc']: - version = self._create_version_from_token(self._groups['range_lower_asc_version']) - exclusive = self._is_lower_bound_exclusive(self._groups['range_lower_asc_prefix']) - lower_bound = _LowerBound(version, not exclusive) - - if self._groups['range_upper_asc']: - version = self._create_version_from_token(self._groups['range_upper_asc_version']) - exclusive = self._is_upper_bound_exclusive(self._groups['range_upper_asc_prefix']) - upper_bound = _UpperBound(version, not exclusive) - - self.bounds.append(_Bound(lower_bound, upper_bound, self.invalid_bound_error)) - - @action - def _act_lower_and_upper_bound_desc(self): - lower_bound = None - upper_bound = None - - if self._groups['range_lower_desc']: - version = self._create_version_from_token(self._groups['range_lower_desc_version']) - exclusive = self._is_lower_bound_exclusive(self._groups['range_lower_desc_prefix']) - lower_bound = _LowerBound(version, not exclusive) - - if self._groups['range_upper_desc']: - version = self._create_version_from_token(self._groups['range_upper_desc_version']) - exclusive = self._is_upper_bound_exclusive(self._groups['range_upper_desc_prefix']) - upper_bound = _UpperBound(version, not exclusive) - - self.bounds.append(_Bound(lower_bound, upper_bound, self.invalid_bound_error)) - - -class VersionRange(_Comparable): - """Version range. - - A version range is a set of one or more contiguous ranges of versions. For - example, "3.0 or greater, but less than 4" is a contiguous range that contains - versions such as "3.0", "3.1.0", "3.99" etc. Version ranges behave something - like sets - they can be intersected, added and subtracted, but can also be - inverted. You can test to see if a Version is contained within a VersionRange. - - A VersionRange "3" (for example) is the superset of any version "3[.X.X...]". - The version "3" itself is also within this range, and is smaller than "3.0" - - any version with common leading tokens, but with a larger token count, is - the larger version of the two. - - VersionRange objects have a flexible syntax that let you describe any - combination of contiguous ranges, including inclusive and exclusive upper - and lower bounds. This is best explained by example (those listed on the - same line are equivalent): - - "3": 'superset' syntax, contains "3", "3.0", "3.1.4" etc; - "2+", ">=2": inclusive lower bound syntax, contains "2", "2.1", "5.0.0" etc; - ">2": exclusive lower bound; - "<5": exclusive upper bound; - "<=5": inclusive upper bound; - "==2": a range that contains only the exact single version "2". - - "1+<5", ">=1<5": inclusive lower, exclusive upper. The most common form of - a 'bounded' version range (ie, one with a lower and upper bound); - ">1<5": exclusive lower, exclusive upper; - ">1<=5": exclusive lower, inclusive upper; - "1+<=5", "1..5": inclusive lower, inclusive upper; - - "<=4,>2", "<4,>2", "<4,>=2": Reverse pip syntax (note comma) - - To help with readability, bounded ranges can also have their bounds separated - with a comma, eg ">=2,<=6". The comma is purely cosmetic and is dropped in - the string representation. - - To describe more than one contiguous range, seperate ranges with the or '|' - symbol. For example, the version range "4|6+" contains versions such as "4", - "4.0", "4.3.1", "6", "6.1", "10.0.0", but does not contain any version - "5[.X.X...X]". If you provide multiple ranges that overlap, they will be - automatically optimised - for example, the version range "3+<6|4+<8" - becomes "3+<8". - - Note that the empty string version range represents the superset of all - possible versions - this is called the "any" range. The empty version can - also be used as an upper or lower bound, leading to some odd but perfectly - valid version range syntax. For example, ">" is a valid range - read like - ">''", it means "any version greater than the empty version". - """ - def __init__(self, range_str='', make_token=AlphanumericVersionToken, - invalid_bound_error=True): - """Create a VersionRange object. - - Args: - range_str: Range string, such as "3", "3+<4.5", "2|6+". The range - will be optimised, so the string representation of this instance - may not match range_str. For example, "3+<6|4+<8" == "3+<8". - make_token: Version token class to use. - invalid_bound_error (bool): If True, raise an exception if an - impossible range is given, such as '3+<2'. - """ - self._str = None - self.bounds = [] # note: kept in ascending order - if range_str is None: - return - - try: - parser = _VersionRangeParser(range_str, make_token, - invalid_bound_error=invalid_bound_error) - bounds = parser.bounds - except ParseException as e: - raise VersionError("Syntax error in version range '%s': %s" - % (range_str, str(e))) - except VersionError as e: - raise VersionError("Invalid version range '%s': %s" - % (range_str, str(e))) - - if bounds: - self.bounds = self._union(bounds) - else: - self.bounds.append(_Bound.any) - - def is_any(self): - """Returns True if this is the "any" range, ie the empty string range - that contains all versions.""" - return (len(self.bounds) == 1) and (self.bounds[0] == _Bound.any) - - def lower_bounded(self): - """Returns True if the range has a lower bound (that is not the empty - version).""" - return self.bounds[0].lower_bounded() - - def upper_bounded(self): - """Returns True if the range has an upper bound.""" - return self.bounds[-1].upper_bounded() - - def bounded(self): - """Returns True if the range has a lower and upper bound.""" - return (self.lower_bounded() and self.upper_bounded()) - - def issuperset(self, range): - """Returns True if the VersionRange is contained within this range. - """ - return self._issuperset(self.bounds, range.bounds) - - def issubset(self, range): - """Returns True if we are contained within the version range. - """ - return range.issuperset(self) - - def union(self, other): - """OR together version ranges. - - Calculates the union of this range with one or more other ranges. - - Args: - other: VersionRange object (or list of) to OR with. - - Returns: - New VersionRange object representing the union. - """ - if not hasattr(other, "__iter__"): - other = [other] - bounds = self.bounds[:] - for range in other: - bounds += range.bounds - - bounds = self._union(bounds) - range = VersionRange(None) - range.bounds = bounds - return range - - def intersection(self, other): - """AND together version ranges. - - Calculates the intersection of this range with one or more other ranges. - - Args: - other: VersionRange object (or list of) to AND with. - - Returns: - New VersionRange object representing the intersection, or None if - no ranges intersect. - """ - if not hasattr(other, "__iter__"): - other = [other] - - bounds = self.bounds - for range in other: - bounds = self._intersection(bounds, range.bounds) - if not bounds: - return None - - range = VersionRange(None) - range.bounds = bounds - return range - - def inverse(self): - """Calculate the inverse of the range. - - Returns: - New VersionRange object representing the inverse of this range, or - None if there is no inverse (ie, this range is the any range). - """ - if self.is_any(): - return None - else: - bounds = self._inverse(self.bounds) - range = VersionRange(None) - range.bounds = bounds - return range - - def intersects(self, other): - """Determine if we intersect with another range. - - Args: - other: VersionRange object. - - Returns: - True if the ranges intersect, False otherwise. - """ - return self._intersects(self.bounds, other.bounds) - - def split(self): - """Split into separate contiguous ranges. - - Returns: - A list of VersionRange objects. For example, the range "3|5+" will - be split into ["3", "5+"]. - """ - ranges = [] - for bound in self.bounds: - range = VersionRange(None) - range.bounds = [bound] - ranges.append(range) - return ranges - - @classmethod - def as_span(cls, lower_version=None, upper_version=None, - lower_inclusive=True, upper_inclusive=True): - """Create a range from lower_version..upper_version. - - Args: - lower_version: Version object representing lower bound of the range. - upper_version: Version object representing upper bound of the range. - - Returns: - `VersionRange` object. - """ - lower = (None if lower_version is None - else _LowerBound(lower_version, lower_inclusive)) - upper = (None if upper_version is None - else _UpperBound(upper_version, upper_inclusive)) - bound = _Bound(lower, upper) - - range = cls(None) - range.bounds = [bound] - return range - - @classmethod - def from_version(cls, version, op=None): - """Create a range from a version. - - Args: - version: Version object. This is used as the upper/lower bound of - the range. - op: Operation as a string. One of 'gt'/'>', 'gte'/'>=', lt'/'<', - 'lte'/'<=', 'eq'/'=='. If None, a bounded range will be created - that contains the version superset. - - Returns: - `VersionRange` object. - """ - lower = None - upper = None - - if op is None: - lower = _LowerBound(version, True) - upper = _UpperBound(version.next(), False) - elif op in ("eq", "=="): - lower = _LowerBound(version, True) - upper = _UpperBound(version, True) - elif op in ("gt", ">"): - lower = _LowerBound(version, False) - elif op in ("gte", ">="): - lower = _LowerBound(version, True) - elif op in ("lt", "<"): - upper = _UpperBound(version, False) - elif op in ("lte", "<="): - upper = _UpperBound(version, True) - else: - raise VersionError("Unknown bound operation '%s'" % op) - - bound = _Bound(lower, upper) - range = cls(None) - range.bounds = [bound] - return range - - @classmethod - def from_versions(cls, versions): - """Create a range from a list of versions. - - This method creates a range that contains only the given versions and - no other. Typically the range looks like (for eg) "==3|==4|==5.1". - - Args: - versions: List of Version objects. - - Returns: - `VersionRange` object. - """ - range = cls(None) - range.bounds = [] - for version in dedup(sorted(versions)): - lower = _LowerBound(version, True) - upper = _UpperBound(version, True) - bound = _Bound(lower, upper) - range.bounds.append(bound) - return range - - def to_versions(self): - """Returns exact version ranges as Version objects, or None if there - are no exact version ranges present. - """ - versions = [] - for bound in self.bounds: - if bound.lower.inclusive and bound.upper.inclusive \ - and (bound.lower.version == bound.upper.version): - versions.append(bound.lower.version) - - return versions or None - - def contains_version(self, version): - """Returns True if version is contained in this range.""" - if len(self.bounds) < 5: - # not worth overhead of binary search - for bound in self.bounds: - i = bound.version_containment(version) - if i == 0: - return True - if i == -1: - return False - else: - _, contains = self._contains_version(version) - return contains - - return False - - def iter_intersect_test(self, iterable, key=None, descending=False): - """Performs containment tests on a sorted list of versions. - - This is more optimal than performing separate containment tests on a - list of sorted versions. - - Args: - iterable: An ordered sequence of versioned objects. If the list - is not sorted by version, behaviour is undefined. - key (callable): Function that returns a `Version` given an object - from `iterable`. If None, the identity function is used. - descending (bool): Set to True if `iterable` is in descending - version order. - - Returns: - An iterator that returns (bool, object) tuples, where 'object' is - the original object in `iterable`, and the bool indicates whether - that version is contained in this range. - """ - return _ContainsVersionIterator(self, iterable, key, descending) - - def iter_intersecting(self, iterable, key=None, descending=False): - """Like `iter_intersect_test`, but returns intersections only. - - Returns: - An iterator that returns items from `iterable` that intersect. - """ - return _ContainsVersionIterator(self, iterable, key, descending, - mode=_ContainsVersionIterator.MODE_INTERSECTING) - - def iter_non_intersecting(self, iterable, key=None, descending=False): - """Like `iter_intersect_test`, but returns non-intersections only. - - Returns: - An iterator that returns items from `iterable` that don't intersect. - """ - return _ContainsVersionIterator(self, iterable, key, descending, - mode=_ContainsVersionIterator.MODE_NON_INTERSECTING) - - def span(self): - """Return a contiguous range that is a superset of this range. - - Returns: - A VersionRange object representing the span of this range. For - example, the span of "2+<4|6+<8" would be "2+<8". - """ - other = VersionRange(None) - bound = _Bound(self.bounds[0].lower, self.bounds[-1].upper) - other.bounds = [bound] - return other - - # TODO have this return a new VersionRange instead - this currently breaks - # VersionRange immutability, and could invalidate __str__. - def visit_versions(self, func): - """Visit each version in the range, and apply a function to each. - - This is for advanced usage only. - - If `func` returns a `Version`, this call will change the versions in - place. - - It is possible to change versions in a way that is nonsensical - for - example setting an upper bound to a smaller version than the lower bound. - Use at your own risk. - - Args: - func (callable): Takes a `Version` instance arg, and is applied to - every version in the range. If `func` returns a `Version`, it - will replace the existing version, updating this `VersionRange` - instance in place. - """ - for bound in self.bounds: - if bound.lower is not _LowerBound.min: - result = func(bound.lower.version) - if isinstance(result, Version): - bound.lower.version = result - - if bound.upper is not _UpperBound.inf: - result = func(bound.upper.version) - if isinstance(result, Version): - bound.upper.version = result - - def __contains__(self, version_or_range): - if isinstance(version_or_range, Version): - return self.contains_version(version_or_range) - else: - return self.issuperset(version_or_range) - - def __len__(self): - return len(self.bounds) - - def __invert__(self): - return self.inverse() - - def __and__(self, other): - return self.intersection(other) - - def __or__(self, other): - return self.union(other) - - def __add__(self, other): - return self.union(other) - - def __sub__(self, other): - inv = other.inverse() - return None if inv is None else self.intersection(inv) - - def __str__(self): - if self._str is None: - self._str = '|'.join(map(str, self.bounds)) - return self._str - - def __eq__(self, other): - return isinstance(other, VersionRange) and self.bounds == other.bounds - - def __lt__(self, other): - return (self.bounds < other.bounds) - - def __hash__(self): - return hash(tuple(self.bounds)) - - def _contains_version(self, version): - vbound = _Bound(_LowerBound(version, True)) - i = bisect_left(self.bounds, vbound) - if i and self.bounds[i - 1].contains_version(version): - return i - 1, True - if (i < len(self.bounds)) and self.bounds[i].contains_version(version): - return i, True - return i, False - - @classmethod - def _union(cls, bounds): - if len(bounds) < 2: - return bounds - - bounds_ = list(sorted(bounds)) - new_bounds = [] - prev_bound = None - upper = None - start = 0 - - for i, bound in enumerate(bounds_): - if i and ((bound.lower.version > upper.version) - or ((bound.lower.version == upper.version) - and (not bound.lower.inclusive) - and (not prev_bound.upper.inclusive))): - new_bound = _Bound(bounds_[start].lower, upper) - new_bounds.append(new_bound) - start = i - - prev_bound = bound - upper = bound.upper if upper is None else max(upper, bound.upper) - - new_bound = _Bound(bounds_[start].lower, upper) - new_bounds.append(new_bound) - return new_bounds - - @classmethod - def _intersection(cls, bounds1, bounds2): - new_bounds = [] - for bound1 in bounds1: - for bound2 in bounds2: - b = bound1.intersection(bound2) - if b: - new_bounds.append(b) - return new_bounds - - @classmethod - def _inverse(cls, bounds): - lbounds = [None] - ubounds = [] - - for bound in bounds: - if not bound.lower.version and bound.lower.inclusive: - ubounds.append(None) - else: - b = _UpperBound(bound.lower.version, not bound.lower.inclusive) - ubounds.append(b) - - if bound.upper.version == Version.inf: - lbounds.append(None) - else: - b = _LowerBound(bound.upper.version, not bound.upper.inclusive) - lbounds.append(b) - - ubounds.append(None) - new_bounds = [] - - for lower, upper in zip(lbounds, ubounds): - if not (lower is None and upper is None): - new_bounds.append(_Bound(lower, upper)) - - return new_bounds - - @classmethod - def _issuperset(cls, bounds1, bounds2): - lo = 0 - for bound2 in bounds2: - i = bisect_left(bounds1, bound2, lo=lo) - if i and bounds1[i - 1].contains_bound(bound2): - lo = i - 1 - continue - if (i < len(bounds1)) and bounds1[i].contains_bound(bound2): - lo = i - continue - return False - - return True - - @classmethod - def _intersects(cls, bounds1, bounds2): - # sort so bounds1 is the shorter list - bounds1, bounds2 = sorted((bounds1, bounds2), key=lambda x: len(x)) - - if len(bounds2) < 5: - # not worth overhead of binary search - for bound1 in bounds1: - for bound2 in bounds2: - if bound1.intersects(bound2): - return True - return False - - lo = 0 - for bound1 in bounds1: - i = bisect_left(bounds2, bound1, lo=lo) - if i and bounds2[i - 1].intersects(bound1): - return True - if (i < len(bounds2)) and bounds2[i].intersects(bound1): - return True - lo = max(i - 1, 0) - - return False - - -class _ContainsVersionIterator(object): - MODE_INTERSECTING = 0 - MODE_NON_INTERSECTING = 2 - MODE_ALL = 3 - - def __init__(self, range_, iterable, key=None, descending=False, mode=MODE_ALL): - self.mode = mode - self.range_ = range_ - self.index = None - self.nbounds = len(self.range_.bounds) - self._constant = True if range_.is_any() else None - self.fn = self._descending if descending else self._ascending - self.it = iter(iterable) - if key is None: - key = lambda x: x - self.keyfunc = key - - if mode == self.MODE_ALL: - self.next_fn = self._next - elif mode == self.MODE_INTERSECTING: - self.next_fn = self._next_intersecting - else: - self.next_fn = self._next_non_intersecting - - def __iter__(self): - return self - - def __next__(self): - return self.next_fn() - - def next(self): - return self.next_fn() - - def _next(self): - value = next(self.it) - if self._constant is not None: - return self._constant, value - - version = self.keyfunc(value) - intersects = self.fn(version) - return intersects, value - - def _next_intersecting(self): - while True: - value = next(self.it) - - if self._constant: - return value - elif self._constant is not None: - raise StopIteration - - version = self.keyfunc(value) - intersects = self.fn(version) - if intersects: - return value - - def _next_non_intersecting(self): - while True: - value = next(self.it) - - if self._constant: - raise StopIteration - elif self._constant is not None: - return value - - version = self.keyfunc(value) - intersects = self.fn(version) - if not intersects: - return value - - @property - def _bound(self): - if self.index < self.nbounds: - return self.range_.bounds[self.index] - else: - return None - - def _ascending(self, version): - if self.index is None: - self.index, contains = self.range_._contains_version(version) - bound = self._bound - if contains: - if not bound.upper_bounded(): - self._constant = True - return True - elif bound is None: # past end of last bound - self._constant = False - return False - else: - return False # there are more bound(s) ahead - else: - bound = self._bound - j = bound.version_containment(version) - if j == 0: - return True - elif j == -1: - return False - else: - while True: - self.index += 1 - bound = self._bound - if bound is None: # past end of last bound - self._constant = False - return False - else: - j = bound.version_containment(version) - if j == 0: - if not bound.upper_bounded(): - self._constant = True - return True - elif j == -1: - return False - - def _descending(self, version): - if self.index is None: - self.index, contains = self.range_._contains_version(version) - bound = self._bound - if contains: - if not bound.lower_bounded(): - self._constant = True - return True - elif bound is None: # past end of last bound - self.index = self.nbounds - 1 - return False - elif self.index == 0: # before start of first bound - self._constant = False - return False - else: - self.index -= 1 - return False - else: - bound = self._bound - j = bound.version_containment(version) - if j == 0: - return True - elif j == 1: - return False - else: - while self.index: - self.index -= 1 - bound = self._bound - j = bound.version_containment(version) - if j == 0: - if not bound.lower_bounded(): - self._constant = True - return True - elif j == 1: - return False - - self._constant = False # before start of first bound - return False +from rez.version._version import * diff --git a/src/rez/version/__init__.py b/src/rez/version/__init__.py new file mode 100644 index 000000000..af11036e9 --- /dev/null +++ b/src/rez/version/__init__.py @@ -0,0 +1,50 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright Contributors to the Rez Project + + +""" +Implements everything needed to manipulate versions and requirements. + +There are three class types: :class:`VersionToken`, :class:`Version` and :class:`VersionRange`. +A :class:`Version` is a set of zero or more :class:`VersionToken`\\s, separate by ``.``\\s or ``-``\\s (eg ``1.2-3``). +A :class:`VersionToken` is a string containing alphanumerics, and default implemenations +:class:`NumericToken` and :class:`AlphanumericVersionToken` are supplied. You can implement +your own if you want stricter tokens or different sorting behaviour. + +A :class:`VersionRange` is a set of one or more contiguous version ranges. For example, +``3+<5`` contains any version >=3 but less than 5. Version ranges can be used to +define dependency requirements between objects. They can be OR'd together, AND'd +and inverted. + +The empty version ``''``, and empty version range ``''``, are also handled. The empty +version is used to denote unversioned objects. The empty version range, also +known as the 'any' range, is used to refer to any version of an object. + +Requirements and list of requirements are represented by :class:`Requirement` and +:class:`RequirementList` respectively. +""" + +from rez.version._requirement import Requirement, RequirementList, VersionedObject +from rez.version._util import ParseException, VersionError +from rez.version._version import ( + AlphanumericVersionToken, + NumericToken, + Version, + VersionRange, + VersionToken, + reverse_sort_key, +) + +__all__ = ( + "Version", + "VersionRange", + "Requirement", + "RequirementList", + "VersionedObject", + "VersionToken", + "NumericToken", + "AlphanumericVersionToken", + "reverse_sort_key", + "ParseException", + "VersionError", +) diff --git a/src/rez/version/_requirement.py b/src/rez/version/_requirement.py new file mode 100644 index 000000000..9e72a1133 --- /dev/null +++ b/src/rez/version/_requirement.py @@ -0,0 +1,476 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright Contributors to the Rez Project + + +from rez.version._version import Version, VersionRange +from rez.version._util import _Common +import re + + +class VersionedObject(_Common): + """Definition of a versioned object, eg ``foo-1.0``. + + ``foo`` is also a valid object definiton. When there is no version part, we + are defining an unversioned object. + + .. note:: + Note that ``-``, ``@`` or ``#`` can be used as the seperator between object name + and version, however this is purely cosmetic. ``foo-1`` is the same as ``foo@1``. + """ + sep_regex_str = r'[-@#]' + sep_regex = re.compile(sep_regex_str) + + def __init__(self, s): + """ + Args: + s (str): + """ + self.name_ = None + self.version_ = None + self.sep_ = '-' + if s is None: + return + + m = self.sep_regex.search(s) + if m: + i = m.start() + self.name_ = s[:i] + self.sep_ = s[i] + ver_str = s[i + 1:] + self.version_ = Version(ver_str) + else: + self.name_ = s + self.version_ = Version() + + @classmethod + def construct(cls, name, version=None): + """Create a VersionedObject directly from an object name and version. + + Args: + name (str): Object name string. + version (typing.Optional[Version]): Version object. + """ + other = VersionedObject(None) + other.name_ = name + other.version_ = Version() if version is None else version + return other + + @property + def name(self): + """Name of the object. + + Returns: + str: + """ + return self.name_ + + @property + def version(self): + """Version of the object. + + Returns: + Version: + """ + return self.version_ + + def as_exact_requirement(self): + """Get the versioned object, as an exact requirement string. + + Returns: + str: Equivalent requirement string, eg ``maya==2016.1`` + """ + sep_str = '' + ver_str = '' + if self.version_: + sep_str = "==" + ver_str = str(self.version_) + return self.name_ + sep_str + ver_str + + def __eq__(self, other): + return (isinstance(other, VersionedObject) + and (self.name_ == other.name_) + and (self.version_ == other.version_)) + + def __hash__(self): + return hash((self.name_, self.version_)) + + def __str__(self): + sep_str = '' + ver_str = '' + if self.version_: + sep_str = self.sep_ + ver_str = str(self.version_) + return self.name_ + sep_str + ver_str + + +class Requirement(_Common): + """ + Defines a requirement for an object. For example, ``foo-5+`` means that you + require any version of ``foo``, version 5 or greater. An unversioned + requirement can also be used (``foo``), this means you require any version of + foo. You can drop the hyphen between object name and version range if the + version range starts with a non-alphanumeric character - eg ``foo<2``. + + There are two different prefixes that can be applied to a requirement: + + - ``!``: The conflict requirement. This means that you require this version + range of an object NOT to be present. To conflict with all versions of an + object, use "!foo". + - ``~``: This is known as a "weak reference", and means, "I do not require this + object, but if present, it must be within this range." It is equivalent to + the *conflict of the inverse* of the given version range. + + There is one subtle case to be aware of. ``~foo`` is a requirement that has no + effect. It means "I do not require foo, but if foo is present, it can + be any version." This statement is still valid, but will produce a + Requirement object with a None range. + + Examples of valid requirement strings: + + - ``foo-1.0`` + - ``foo@1.0`` + - ``foo#1.0`` + - ``foo-1+`` + - ``foo-1+<4.3`` + - ``foo<3`` + - ``foo==1.0.1`` + """ + sep_regex = re.compile(r'[-@#=<>]') + + def __init__(self, s, invalid_bound_error=True): + """ + Args: + s (str): Requirement string + invalid_bound_error (bool): If True, raise :exc:`VersionError` if an + impossible range is given, such as ``3+<2``. + """ + self.name_ = None + self.range_ = None + self.negate_ = False + self.conflict_ = False + self._str = None + self.sep_ = '-' + if s is None: + return + + self.conflict_ = s.startswith('!') + if self.conflict_: + s = s[1:] + elif s.startswith('~'): + s = s[1:] + self.negate_ = True + self.conflict_ = True + + m = self.sep_regex.search(s) + if m: + i = m.start() + self.name_ = s[:i] + req_str = s[i:] + if req_str[0] in ('-', '@', '#'): + self.sep_ = req_str[0] + req_str = req_str[1:] + + self.range_ = VersionRange( + req_str, invalid_bound_error=invalid_bound_error) + if self.negate_: + self.range_ = ~self.range_ + elif self.negate_: + self.name_ = s + # rare case - '~foo' equates to no effect + self.range_ = None + else: + self.name_ = s + self.range_ = VersionRange() + + @classmethod + def construct(cls, name, range=None): + """Create a requirement directly from an object name and VersionRange. + + Args: + name (str): Object name string. + range (typing.Optional[VersionRange]): If None, an unversioned requirement is + created. + """ + other = Requirement(None) + other.name_ = name + other.range_ = VersionRange() if range is None else range + return other + + @property + def name(self): + """Name of the required object. + + Returns: + str: + """ + return self.name_ + + @property + def range(self): + """Version range of the requirement. + + Returns: + VersionRange: + """ + return self.range_ + + @property + def conflict(self): + """True if the requirement is a conflict requirement, eg "!foo", "~foo-1". + + Returns: + bool: + """ + return self.conflict_ + + @property + def weak(self): + """True if the requirement is weak, eg "~foo". + + .. note:: + Note that weak requirements are also conflict requirements, but not + necessarily the other way around. + + Returns: + bool: + """ + return self.negate_ + + def safe_str(self): + """Return a string representation that is safe for the current filesystem, + and guarantees that no two different Requirement objects will encode to + the same value. + + Returns: + str: + """ + return str(self) + + def conflicts_with(self, other): + """Returns True if this requirement conflicts with another :class:`Requirement` + or :class:`VersionedObject`. + + Returns: + bool: + """ + if isinstance(other, Requirement): + if (self.name_ != other.name_) or (self.range is None) \ + or (other.range is None): + return False + elif self.conflict: + return False if other.conflict \ + else self.range_.issuperset(other.range_) + elif other.conflict: + return other.range_.issuperset(self.range_) + else: + return not self.range_.intersects(other.range_) + else: # VersionedObject + if (self.name_ != other.name_) or (self.range is None): + return False + if self.conflict: + return (other.version_ in self.range_) + else: + return (other.version_ not in self.range_) + + def merged(self, other): + """Merge two requirements. + + Two requirements can be in conflict and if so, this function returns + None. For example, requests for ``foo-4`` and ``foo-6`` are in conflict, + since both cannot be satisfied with a single version of foo. + + Some example successful requirements merges are: + + - ``foo-3+`` and ``!foo-5+`` == ``foo-3+<5`` + - ``foo-1`` and ``foo-1.5`` == ``foo-1.5`` + - ``!foo-2`` and ``!foo-5`` == ``!foo-2|5`` + + Returns: + Requirement: the merged result of two requirements. + """ + if self.name_ != other.name_: + return None # cannot merge across object names + + def _r(r_): + r = Requirement(None) + r.name_ = r_.name_ + r.negate_ = r_.negate_ + r.conflict_ = r_.conflict_ + r.sep_ = r_.sep_ + return r + + if self.range is None: + return other + elif other.range is None: + return self + elif self.conflict: + if other.conflict: + r = _r(self) + r.range_ = self.range_ | other.range_ + r.negate_ = (self.negate_ and other.negate_ + and not r.range_.is_any()) + return r + else: + range_ = other.range - self.range + if range_ is None: + return None + else: + r = _r(other) + r.range_ = range_ + return r + elif other.conflict: + range_ = self.range_ - other.range_ + if range_ is None: + return None + else: + r = _r(self) + r.range_ = range_ + return r + else: + range_ = self.range_ & other.range_ + if range_ is None: + return None + else: + r = _r(self) + r.range_ = range_ + return r + + def __eq__(self, other): + return (isinstance(other, Requirement) + and (self.name_ == other.name_) + and (self.range_ == other.range_) + and (self.conflict_ == other.conflict_)) + + def __hash__(self): + return hash(str(self)) + + def __str__(self): + if self._str is None: + pre_str = '~' if self.negate_ else ('!' if self.conflict_ else '') + range_str = '' + sep_str = '' + + range_ = self.range_ + if self.negate_: + range_ = ~range_ if range_ else VersionRange() + + if not range_.is_any(): + range_str = str(range_) + if range_str[0] not in ('=', '<', '>'): + sep_str = self.sep_ + + self._str = pre_str + self.name_ + sep_str + range_str + return self._str + + +class RequirementList(_Common): + """A list of requirements. + + This class takes a Requirement list and reduces it to the equivalent + optimal form, merging any requirements for common objects. Order of objects + is retained. + """ + def __init__(self, requirements): + """ + Args: + requirements (list[Requirement]): List of requirements. + """ + self.requirements_ = [] + self.conflict_ = None + self.requirements_dict = {} + self.names_ = set() + self.conflict_names_ = set() + + for req in requirements: + existing_req = self.requirements_dict.get(req.name) + + if existing_req is None: + self.requirements_dict[req.name] = req + else: + merged_req = existing_req.merged(req) + if merged_req is None: + self.conflict_ = (existing_req, req) + return + else: + self.requirements_dict[req.name] = merged_req + + seen = set() + + # build optimised list, this intends to match original request order + # as closely as possible + for req in requirements: + if req.name not in seen: + seen.add(req.name) + req_ = self.requirements_dict[req.name] + self.requirements_.append(req_) + + if req_.conflict: + self.conflict_names_.add(req.name) + else: + self.names_.add(req.name) + + @property + def requirements(self): + """Returns optimised list of requirements, or None if there are + conflicts. + + Returns: + list[Requirement]: + """ + return self.requirements_ + + @property + def conflict(self): + """Get the requirement conflict, if any. + + Returns: + typing.Optional[tuple[Requirement]]: None if there is no conflict, otherwise a + 2-tuple containing the conflicting requirement objects. + """ + return self.conflict_ + + @property + def names(self): + """Set of names of requirements, not including conflict requirements. + + Returns: + set[str]: + """ + return self.names_ + + @property + def conflict_names(self): + """Set of conflict requirement names. + + Returns: + set[str]: + """ + return self.conflict_names_ + + def __iter__(self): + for requirement in (self.requirements_ or []): + yield requirement + + def get(self, name): + """Returns the requirement for the given object, or None. + + Args: + name (str): requirement to get. + + Returns: + Requirement: + """ + return self.requirements_dict.get(name) + + def __eq__(self, other): + return (isinstance(other, RequirementList) + and (self.requirements_ == other.requirements_) + and (self.conflict_ == other.conflict_)) + + def __str__(self): + if self.conflict_: + s1 = str(self.conflict_[0]) + s2 = str(self.conflict_[1]) + return "%s <--!--> %s" % (s1, s2) + else: + return ' '.join(str(x) for x in self.requirements_) diff --git a/src/rez/version/_util.py b/src/rez/version/_util.py new file mode 100644 index 000000000..47da793b6 --- /dev/null +++ b/src/rez/version/_util.py @@ -0,0 +1,30 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright Contributors to the Rez Project + + +from itertools import groupby + + +class VersionError(Exception): + pass + + +class ParseException(Exception): + pass + + +class _Common(object): + def __str__(self): + raise NotImplementedError + + def __ne__(self, other): + return not (self == other) + + def __repr__(self): + return "%s(%r)" % (self.__class__.__name__, str(self)) + + +def dedup(iterable): + """Removes duplicates from a sorted sequence.""" + for e in groupby(iterable): + yield e[0] diff --git a/src/rez/version/_version.py b/src/rez/version/_version.py new file mode 100644 index 000000000..8658980a2 --- /dev/null +++ b/src/rez/version/_version.py @@ -0,0 +1,1541 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright Contributors to the Rez Project + + +from __future__ import print_function +from rez.version._util import VersionError, ParseException, _Common, \ + dedup +from bisect import bisect_left +import copy +import string +import re + + +re_token = re.compile(r"[a-zA-Z0-9_]+") + + +class _Comparable(_Common): + def __gt__(self, other): + return not (self < other or self == other) + + def __le__(self, other): + return self < other or self == other + + def __ge__(self, other): + return not self < other + + +class _ReversedComparable(_Common): + def __init__(self, value): + self.value = value + + def __lt__(self, other): + return not (self.value < other.value) + + def __gt__(self, other): + return not (self < other or self == other) + + def __le__(self, other): + return self < other or self == other + + def __ge__(self, other): + return not self < other + + def __str__(self): + return "reverse(%s)" % str(self.value) + + def __repr__(self): + return "reverse(%r)" % self.value + + +class VersionToken(_Comparable): + """Token within a version number. + + A version token is that part of a version number that appears between a + delimiter, typically ``.`` or ``-``. For example, the version number ``2.3.07b`` + contains the tokens ``2``, ``3`` and ``07b`` respectively. + + Version tokens are only allowed to contain alphanumerics (any case) and + underscores. + """ + def __init__(self, token): + """ + Args: + token (str): Token string, eg "rc02" + """ + raise NotImplementedError + + @classmethod + def create_random_token_string(cls): + """Create a random token string. For testing purposes only. + + :meta private: + """ + raise NotImplementedError + + def less_than(self, other): + """Compare to another :class:`VersionToken`. + + Args: + other (VersionToken): The VersionToken object to compare against. + + Returns: + bool: True if this token is less than other, False otherwise. + """ + raise NotImplementedError + + def next(self): + """Returns the next largest token.""" + raise NotImplementedError + + def __str__(self): + raise NotImplementedError + + def __lt__(self, other): + return self.less_than(other) + + def __eq__(self, other): + return (not self < other) and (not other < self) + + +class NumericToken(VersionToken): + """Numeric version token. + + Version token supporting numbers only. Padding is ignored. + """ + def __init__(self, token): + if not token.isdigit(): + raise VersionError("Invalid version token: '%s'" % token) + else: + self.n = int(token) + + @classmethod + def create_random_token_string(cls): + import random + chars = string.digits + return ''.join([chars[random.randint(0, len(chars) - 1)] + for _ in range(8)]) + + def __str__(self): + return str(self.n) + + def __eq__(self, other): + return (self.n == other.n) + + def less_than(self, other): + return (self.n < other.n) + + def __next__(self): + other = copy.copy(self) + other.n = self.n = 1 + return other + + def next(self): + return self.__next__() + + +class _SubToken(_Comparable): + """Used internally by AlphanumericVersionToken.""" + def __init__(self, s): + self.s = s + self.n = int(s) if s.isdigit() else None + + def __lt__(self, other): + if self.n is None: + return (self.s < other.s) if other.n is None else True + else: + return False if other.n is None \ + else ((self.n, self.s) < (other.n, other.s)) + + def __eq__(self, other): + return (self.s == other.s) and (self.n == other.n) + + def __str__(self): + return self.s + + +class AlphanumericVersionToken(VersionToken): + """Alphanumeric version token. + + These tokens compare as follows: + + - each token is split into alpha and numeric groups (subtokens); + - the resulting subtoken list is compared. + - alpha comparison is case-sensitive, numeric comparison is padding-sensitive. + + Subtokens compare as follows: + + - alphas come before numbers; + - alphas are compared alphabetically (``_``, then A-Z, then a-z); + - numbers are compared numerically. If numbers are equivalent but zero-padded + differently, they are then compared alphabetically. Thus ``01`` < ``1``. + + Some example comparisons that equate to true: + + - ``3`` < ``4`` + - ``01`` < ``1`` + - ``beta`` < ``1`` + - ``alpha3`` < ``alpha4`` + - ``alpha`` < ``alpha3`` + - ``gamma33`` < ``33gamma`` + """ + numeric_regex = re.compile("[0-9]+") + regex = re.compile(r"[a-zA-Z0-9_]+\Z") + + def __init__(self, token): + if token is None: + self.subtokens = None + elif not self.regex.match(token): + raise VersionError("Invalid version token: '%s'" % token) + else: + self.subtokens = self._parse(token) + + @classmethod + def create_random_token_string(cls): + import random + chars = string.digits + string.ascii_letters + return ''.join([chars[random.randint(0, len(chars) - 1)] + for _ in range(8)]) + + def __str__(self): + return ''.join(map(str, self.subtokens)) + + def __eq__(self, other): + return (self.subtokens == other.subtokens) + + def less_than(self, other): + return (self.subtokens < other.subtokens) + + def __next__(self): + other = AlphanumericVersionToken(None) + other.subtokens = self.subtokens[:] + subtok = other.subtokens[-1] + if subtok.n is None: + other.subtokens[-1] = _SubToken(subtok.s + '_') + else: + other.subtokens.append(_SubToken('_')) + return other + + def next(self): + return self.__next__() + + @classmethod + def _parse(cls, s): + subtokens = [] + alphas = cls.numeric_regex.split(s) + numerics = cls.numeric_regex.findall(s) + b = True + + while alphas or numerics: + if b: + alpha = alphas[0] + alphas = alphas[1:] + if alpha: + subtokens.append(_SubToken(alpha)) + else: + numeric = numerics[0] + numerics = numerics[1:] + subtokens.append(_SubToken(numeric)) + b = not b + + return subtokens + + +def reverse_sort_key(comparable): + """Key that gives reverse sort order on versions and version ranges. + + Example: + + >>> Version("1.0") < Version("2.0") + True + >>> reverse_sort_key(Version("1.0")) < reverse_sort_key(Version("2.0")) + False + + Args: + comparable (Version or VersionRange): Object to wrap. + + Returns: + _ReversedComparable: Wrapper object that reverses comparisons. + """ + return _ReversedComparable(comparable) + + +class Version(_Comparable): + """ + A Version is a sequence of zero or more version tokens, separated by either + a dot ``.`` or hyphen ``-`` delimiters. Note that separators only affect Version + objects cosmetically. In other words, the version ``1.0.0`` is equivalent to + ``1-0-0``. + + The empty version ``''`` is the smallest possible version, and can be used to + represent an unversioned resource. + """ + inf = None + + def __init__(self, ver_str='', make_token=AlphanumericVersionToken): + """ + Args: + ver_str (str): Version string. + make_token (typing.Callable[[str], None]): Callable that creates a VersionToken subclass from a + string. + """ + self.tokens = [] + self.seps = [] + self._str = None + self._hash = None + + if ver_str: + toks = re_token.findall(ver_str) + if not toks: + raise VersionError(ver_str) + + seps = re_token.split(ver_str) + if seps[0] or seps[-1] or max(len(x) for x in seps) > 1: + raise VersionError("Invalid version syntax: '%s'" % ver_str) + + for tok in toks: + try: + self.tokens.append(make_token(tok)) + except VersionError as e: + raise VersionError("Invalid version '%s': %s" + % (ver_str, str(e))) + + self.seps = seps[1:-1] + + def copy(self): + """ + Returns a copy of the version. + + Returns: + Version: + """ + other = Version(None) + other.tokens = self.tokens[:] + other.seps = self.seps[:] + return other + + def trim(self, len_): + """Return a copy of the version, possibly with less tokens. + + Args: + len_ (int): New version length. If >= current length, an + unchanged copy of the version is returned. + + Returns: + Version: + """ + other = Version(None) + other.tokens = self.tokens[:len_] + other.seps = self.seps[:len_ - 1] + return other + + def __next__(self): + """Return :meth:`next` version. Eg, ``next(1.2)`` is ``1.2_``""" + if self.tokens: + other = self.copy() + tok = other.tokens.pop() + other.tokens.append(tok.next()) + return other + else: + return Version.inf + + def next(self): + return self.__next__() + + @property + def major(self): + """Semantic versioning major version. + + Returns: + VersionToken: A VersionToken or a subclass of a VersionToken. + """ + return self[0] + + @property + def minor(self): + """Semantic versioning minor version. + + Returns: + VersionToken: A VersionToken or a subclass of a VersionToken. + """ + return self[1] + + @property + def patch(self): + """Semantic versioning patch version. + + Returns: + VersionToken: A VersionToken or a subclass of a VersionToken. + """ + return self[2] + + def as_tuple(self): + """Convert to a tuple of strings. + + Example: + + >>> print Version("1.2.12").as_tuple() + ('1', '2', '12') + + Returns: + tuple[str]: + """ + return tuple(map(str, self.tokens)) + + def __len__(self): + return len(self.tokens or []) + + def __getitem__(self, index): + try: + return (self.tokens or [])[index] + except IndexError: + raise IndexError("version token index out of range") + + def __nonzero__(self): + """The empty version equates to False.""" + return bool(self.tokens) + + __bool__ = __nonzero__ # py3 compat + + def __eq__(self, other): + return isinstance(other, Version) and self.tokens == other.tokens + + def __lt__(self, other): + if self.tokens is None: + return False + elif other.tokens is None: + return True + else: + return (self.tokens < other.tokens) + + def __hash__(self): + if self._hash is None: + self._hash = hash(None) if self.tokens is None \ + else hash(tuple(map(str, self.tokens))) + return self._hash + + def __str__(self): + if self._str is None: + self._str = "[INF]" if self.tokens is None \ + else ''.join(str(x) + y for x, y in zip(self.tokens, self.seps + [''])) + return self._str + + +# internal use only +Version.inf = Version() +Version.inf.tokens = None + + +class _LowerBound(_Comparable): + min = None + + def __init__(self, version, inclusive): + self.version = version + self.inclusive = inclusive + + def __str__(self): + if self.version: + s = "%s+" if self.inclusive else ">%s" + return s % self.version + else: + return '' if self.inclusive else ">" + + def __eq__(self, other): + return (self.version == other.version) \ + and (self.inclusive == other.inclusive) + + def __lt__(self, other): + return (self.version < other.version) \ + or ((self.version == other.version) + and (self.inclusive and not other.inclusive)) + + def __hash__(self): + return hash((self.version, self.inclusive)) + + def contains_version(self, version): + return (version > self.version) \ + or (self.inclusive and (version == self.version)) + + +_LowerBound.min = _LowerBound(Version(), True) + + +class _UpperBound(_Comparable): + inf = None + + def __init__(self, version, inclusive): + self.version = version + self.inclusive = inclusive + if not version and not inclusive: + raise VersionError("Invalid upper bound: '%s'" % str(self)) + + def __str__(self): + s = "<=%s" if self.inclusive else "<%s" + return s % self.version + + def __eq__(self, other): + return (self.version == other.version) \ + and (self.inclusive == other.inclusive) + + def __lt__(self, other): + return (self.version < other.version) \ + or ((self.version == other.version) + and (not self.inclusive and other.inclusive)) + + def __hash__(self): + return hash((self.version, self.inclusive)) + + def contains_version(self, version): + return (version < self.version) \ + or (self.inclusive and (version == self.version)) + + +_UpperBound.inf = _UpperBound(Version.inf, True) + + +class _Bound(_Comparable): + any = None + + def __init__(self, lower=None, upper=None, invalid_bound_error=True): + self.lower = lower or _LowerBound.min + self.upper = upper or _UpperBound.inf + + if invalid_bound_error and ( + self.lower.version > self.upper.version + or ( + (self.lower.version == self.upper.version) + and not (self.lower.inclusive and self.upper.inclusive) + ) + ): + raise VersionError("Invalid bound") + + def __str__(self): + if self.upper.version == Version.inf: + return str(self.lower) + elif self.lower.version == self.upper.version: + return "==%s" % str(self.lower.version) + elif self.lower.inclusive and self.upper.inclusive: + if self.lower.version: + return "%s..%s" % (self.lower.version, self.upper.version) + else: + return "<=%s" % self.upper.version + elif (self.lower.inclusive and not self.upper.inclusive) \ + and (self.lower.version.next() == self.upper.version): + return str(self.lower.version) + else: + return "%s%s" % (self.lower, self.upper) + + def __eq__(self, other): + return (self.lower == other.lower) and (self.upper == other.upper) + + def __lt__(self, other): + return (self.lower, self.upper) < (other.lower, other.upper) + + def __hash__(self): + return hash((self.lower, self.upper)) + + def lower_bounded(self): + return (self.lower != _LowerBound.min) + + def upper_bounded(self): + return (self.upper != _UpperBound.inf) + + def contains_version(self, version): + return (self.version_containment(version) == 0) + + def version_containment(self, version): + if not self.lower.contains_version(version): + return -1 + if not self.upper.contains_version(version): + return 1 + return 0 + + def contains_bound(self, bound): + return (self.lower <= bound.lower) and (self.upper >= bound.upper) + + def intersects(self, other): + lower = max(self.lower, other.lower) + upper = min(self.upper, other.upper) + + return (lower.version < upper.version) or ( + (lower.version == upper.version) and (lower.inclusive and upper.inclusive) + ) + + def intersection(self, other): + lower = max(self.lower, other.lower) + upper = min(self.upper, other.upper) + + if (lower.version < upper.version) or ( + (lower.version == upper.version) and (lower.inclusive and upper.inclusive) + ): + return _Bound(lower, upper) + else: + return None + + +_Bound.any = _Bound() + + +class _VersionRangeParser(object): + debug = False # set to True to enable parser debugging + + re_flags = (re.VERBOSE | re.DEBUG) if debug else re.VERBOSE + + # The regular expression for a version - one or more version tokens + # followed by a non-capturing group of version separator followed by + # one or more version tokens. + # + # Note that this assumes AlphanumericVersionToken-based versions! + # + # TODO - Would be better to have `VersionRange` keep a static dict of + # parser instances, per token class type. We would add a 'regex' static + # string to each token class, and that could be used to construct + # `version_group` as below. We need to keep a dict of these parser instances, + # to avoid recompiling the large regex every time a version range is + # instantiated. In the cpp port this would be simpler - VersionRange could + # just have a static parser that is instantiated when the version range + # template class is instantiated. + # + version_group = r"([0-9a-zA-Z_]+(?:[.-][0-9a-zA-Z_]+)*)" + + version_range_regex = ( + # Match a version number (e.g. 1.0.0) + r" ^(?P{version_group})$" + "|" + # Or match an exact version number (e.g. ==1.0.0) + " ^(?P" + " ==" # Required == operator + " (?P{version_group})?" + " )$" + "|" + # Or match an inclusive bound (e.g. 1.0.0..2.0.0) + " ^(?P" + " (?P{version_group})?" + r" \.\." # Required .. operator + " (?P{version_group})?" + " )$" + "|" + # Or match a lower bound (e.g. 1.0.0+) + " ^(?P" + " (?P>|>=)?" # Bound is exclusive? + " (?P{version_group})?" + r" (?(lower_bound_prefix)|\+)" # + only if bound is not exclusive + " )$" + "|" + # Or match an upper bound (e.g. <=1.0.0) + " ^(?P" + " (?P<(?={version_group})|<=)?" # Bound is exclusive? + " (?P{version_group})?" + " )$" + "|" + # Or match a range in ascending order (e.g. 1.0.0+<2.0.0) + " ^(?P" + " (?P" + " (?P>|>=)?" # Lower bound is exclusive? + " (?P{version_group})?" + r" (?(range_lower_asc_prefix)|\+)?" # + only if lower bound is not exclusive + " )(?P" + " (?(range_lower_asc_version),?|)" # , only if lower bound is found + " (?P<(?={version_group})|<=)" # <= only if followed by a version group + " (?P{version_group})?" + " )" + " )$" + "|" + # Or match a range in descending order (e.g. <=2.0.0,1.0.0+) + " ^(?P" + " (?P" + " (?P<|<=)?" # Upper bound is exclusive? + " (?P{version_group})?" + r" (?(range_upper_desc_prefix)|\+)?" # + only if upper bound is not exclusive + " )(?P" + " (?(range_upper_desc_version),|)" # Comma is not optional because we don't want + # to recognize something like "<4>3" + " (?P<(?={version_group})|>=?)" # >= or > only if followed + # by a version group + " (?P{version_group})?" + " )" + " )$" + ).format(version_group=version_group) + + regex = re.compile(version_range_regex, re_flags) + + def __init__(self, input_string, make_token, invalid_bound_error=True): + self.make_token = make_token + self._groups = {} + self._input_string = input_string + self.bounds = [] + self.invalid_bound_error = invalid_bound_error + + is_any = False + + for part in input_string.split("|"): + if part == '': + # OR'ing anthing with the 'any' version range ('') will also + # give the any range. Note that we can't early out here, as we + # need to validate that the rest of the string is syntactically + # correct + # + is_any = True + self.bounds = [] + continue + + match = re.search(self.regex, part) + if not match: + raise ParseException("Syntax error in version range '%s'" % part) + + if is_any: + # we're already the 'any' range regardless, so avoid more work + continue + + self._groups = match.groupdict() + + # Note: the following is ordered by approx likelihood of use + + if self._groups['range_asc']: + self._act_lower_and_upper_bound_asc() + + elif self._groups['version']: + self._act_version() + + elif self._groups['lower_bound']: + self._act_lower_bound() + + elif self._groups['exact_version']: + self._act_exact_version() + + elif self._groups['range_desc']: + self._act_lower_and_upper_bound_desc() + + elif self._groups['inclusive_bound']: + self._act_bound() + + elif self._groups['upper_bound']: + self._act_upper_bound() + + def _is_lower_bound_exclusive(self, token): + return (token == ">") + + def _is_upper_bound_exclusive(self, token): + return (token == "<") + + def _create_version_from_token(self, token): + return Version(token, make_token=self.make_token) + + def action(fn): + def fn_(self): + result = fn(self) + if self.debug: + label = fn.__name__.replace("_act_", "") + print("%-21s: %s" % (label, self._input_string)) + for key, value in self._groups.items(): + print(" %-17s= %s" % (key, value)) + print(" %-17s= %s" % ("bounds", self.bounds)) + return result + return fn_ + + @action + def _act_version(self): + version = self._create_version_from_token(self._groups['version']) + lower_bound = _LowerBound(version, True) + upper_bound = _UpperBound(version.next(), False) if version else None + + self.bounds.append(_Bound(lower_bound, upper_bound)) + + @action + def _act_exact_version(self): + version = self._create_version_from_token(self._groups['exact_version_group']) + lower_bound = _LowerBound(version, True) + upper_bound = _UpperBound(version, True) + + self.bounds.append(_Bound(lower_bound, upper_bound)) + + @action + def _act_bound(self): + lower_version = self._create_version_from_token(self._groups['inclusive_lower_version']) + lower_bound = _LowerBound(lower_version, True) + + upper_version = self._create_version_from_token(self._groups['inclusive_upper_version']) + upper_bound = _UpperBound(upper_version, True) + + self.bounds.append(_Bound(lower_bound, upper_bound, self.invalid_bound_error)) + + @action + def _act_lower_bound(self): + version = self._create_version_from_token(self._groups['lower_version']) + exclusive = self._is_lower_bound_exclusive(self._groups['lower_bound_prefix']) + lower_bound = _LowerBound(version, not exclusive) + + self.bounds.append(_Bound(lower_bound, None)) + + @action + def _act_upper_bound(self): + version = self._create_version_from_token(self._groups['upper_version']) + exclusive = self._is_upper_bound_exclusive(self._groups['upper_bound_prefix']) + upper_bound = _UpperBound(version, not exclusive) + + self.bounds.append(_Bound(None, upper_bound)) + + @action + def _act_lower_and_upper_bound_asc(self): + lower_bound = None + upper_bound = None + + if self._groups['range_lower_asc']: + version = self._create_version_from_token(self._groups['range_lower_asc_version']) + exclusive = self._is_lower_bound_exclusive(self._groups['range_lower_asc_prefix']) + lower_bound = _LowerBound(version, not exclusive) + + if self._groups['range_upper_asc']: + version = self._create_version_from_token(self._groups['range_upper_asc_version']) + exclusive = self._is_upper_bound_exclusive(self._groups['range_upper_asc_prefix']) + upper_bound = _UpperBound(version, not exclusive) + + self.bounds.append(_Bound(lower_bound, upper_bound, self.invalid_bound_error)) + + @action + def _act_lower_and_upper_bound_desc(self): + lower_bound = None + upper_bound = None + + if self._groups['range_lower_desc']: + version = self._create_version_from_token(self._groups['range_lower_desc_version']) + exclusive = self._is_lower_bound_exclusive(self._groups['range_lower_desc_prefix']) + lower_bound = _LowerBound(version, not exclusive) + + if self._groups['range_upper_desc']: + version = self._create_version_from_token(self._groups['range_upper_desc_version']) + exclusive = self._is_upper_bound_exclusive(self._groups['range_upper_desc_prefix']) + upper_bound = _UpperBound(version, not exclusive) + + self.bounds.append(_Bound(lower_bound, upper_bound, self.invalid_bound_error)) + + +class VersionRange(_Comparable): + """ + A version range is a set of one or more contiguous ranges of versions. For + example, "3.0 or greater, but less than 4" is a contiguous range that contains + versions such as ``3.0``, ``3.1.0``, ``3.99`` etc. Version ranges behave something + like sets. They can be intersected, added and subtracted, but can also be + inverted. You can test to see if a :class:`Version` is contained within a :class:`VersionRange`. + + A VersionRange ``3`` (for example) is the superset of any version ``3[.X.X...]``. + The version ``3`` itself is also within this range, and is smaller than ``3.0``. + Any version with common leading tokens, but with a larger token count, is + the larger version of the two. + + VersionRange objects have a flexible syntax that let you describe any + combination of contiguous ranges, including inclusive and exclusive upper + and lower bounds. This is best explained by example (those listed on the + same line are equivalent): + + - ``3``: 'superset' syntax, contains ``3``, ``3.0``, ``3.1.4`` etc; + - ``2+``, ``>=2``: inclusive lower bound syntax, contains ``2``, ``2.1``, ``5.0.0`` etc; + - ``>2``: exclusive lower bound; + - ``<5``: exclusive upper bound; + - ``<=5``: inclusive upper bound; + - ``==2``: a range that contains only the exact single version ``2``. + + .. + + - ``1+<5``, ``>=1<5``: inclusive lower, exclusive upper. The most common form of + a 'bounded' version range (ie, one with a lower and upper bound); + + .. + + - ``>1<5``: exclusive lower, exclusive upper; + - ``>1<=5``: exclusive lower, inclusive upper; + - ``1+<=5``, ``1..5``: inclusive lower, inclusive upper; + + .. + + - ``<=4,>2``, ``<4,>2``, ``<4,>=2``: Reverse pip syntax (note comma) + + To help with readability, bounded ranges can also have their bounds separated + with a comma, eg ``>=2,<=6``. The comma is purely cosmetic and is dropped in + the string representation. + + To describe more than one contiguous range, seperate ranges with the or ``|`` + symbol. For example, the version range ``4|6+`` contains versions such as ``4``, + ``4.0``, ``4.3.1``, ``6``, ``6.1``, ``10.0.0``, but does not contain any version + ``5[.X.X...X]``. If you provide multiple ranges that overlap, they will be + automatically optimised. For example, the version range ``3+<6|4+<8`` + becomes ``3+<8``. + + Note that the empty string version range represents the superset of all + possible versions. This is called the "any" range. The empty version can + also be used as an upper or lower bound, leading to some odd but perfectly + valid version range syntax. For example, ``>`` is a valid range - read like + ``>''``, it means ``any version greater than the empty version``. + """ + def __init__(self, range_str='', make_token=AlphanumericVersionToken, + invalid_bound_error=True): + """ + Args: + range_str (str): Range string, such as "3", "3+<4.5", "2|6+". The range + will be optimised, so the string representation of this instance + may not match range_str. For example, "3+<6|4+<8" == "3+<8". + make_token (typing.Type[VersionToken]): Version token class to use. + invalid_bound_error (bool): If True, raise an exception if an + impossible range is given, such as '3+<2'. + """ + self._str = None + self.bounds = [] # note: kept in ascending order + if range_str is None: + return + + try: + parser = _VersionRangeParser(range_str, make_token, + invalid_bound_error=invalid_bound_error) + bounds = parser.bounds + except ParseException as e: + raise VersionError("Syntax error in version range '%s': %s" + % (range_str, str(e))) + except VersionError as e: + raise VersionError("Invalid version range '%s': %s" + % (range_str, str(e))) + + if bounds: + self.bounds = self._union(bounds) + else: + self.bounds.append(_Bound.any) + + def is_any(self): + """ + Returns: + bool: True if this is the "any" range, ie the empty string range + that contains all versions. + """ + return (len(self.bounds) == 1) and (self.bounds[0] == _Bound.any) + + def lower_bounded(self): + """ + Returns: + bool: True if the range has a lower bound (that is not the empty + version). + """ + return self.bounds[0].lower_bounded() + + def upper_bounded(self): + """ + Returns: + bool: True if the range has an upper bound. + """ + return self.bounds[-1].upper_bounded() + + def bounded(self): + """ + Returns: + bool: True if the range has a lower and upper bound. + """ + return (self.lower_bounded() and self.upper_bounded()) + + def issuperset(self, range): + """ + Returns: + bool: True if the VersionRange is contained within this range. + """ + return self._issuperset(self.bounds, range.bounds) + + def issubset(self, range): + """ + Returns: + bool: True if we are contained within the version range. + """ + return range.issuperset(self) + + def union(self, other): + """OR together version ranges. + + Calculates the union of this range with one or more other ranges. + + Args: + other (VersionRange or list[VersionRange]): Version range object(s) to OR with. + + Returns: + VersionRange: Range object representing the union. + """ + if not hasattr(other, "__iter__"): + other = [other] + bounds = self.bounds[:] + for range in other: + bounds += range.bounds + + bounds = self._union(bounds) + range = VersionRange(None) + range.bounds = bounds + return range + + def intersection(self, other): + """AND together version ranges. + + Calculates the intersection of this range with one or more other ranges. + + Args: + other (VersionRange or list[VersionRange]): Version range object(s) to AND with. + + Returns: + typing.Optional[VersionRange]: New VersionRange object representing the intersection, or None if + no ranges intersect. + """ + if not hasattr(other, "__iter__"): + other = [other] + + bounds = self.bounds + for range in other: + bounds = self._intersection(bounds, range.bounds) + if not bounds: + return None + + range = VersionRange(None) + range.bounds = bounds + return range + + def inverse(self): + """Calculate the inverse of the range. + + Returns: + typing.Optional[VersionRange]: New VersionRange object representing the inverse of this range, or + None if there is no inverse (ie, this range is the any range). + """ + if self.is_any(): + return None + else: + bounds = self._inverse(self.bounds) + range = VersionRange(None) + range.bounds = bounds + return range + + def intersects(self, other): + """Determine if we intersect with another range. + + Args: + other (VersionRange): Version range object. + + Returns: + bool: True if the ranges intersect, False otherwise. + """ + return self._intersects(self.bounds, other.bounds) + + def split(self): + """Split into separate contiguous ranges. + + Returns: + list[VersionRange]: A list of VersionRange objects. For example, the range ``3|5+`` will + be split into ``["3", "5+"]``. + """ + ranges = [] + for bound in self.bounds: + range = VersionRange(None) + range.bounds = [bound] + ranges.append(range) + return ranges + + @classmethod + def as_span(cls, lower_version=None, upper_version=None, + lower_inclusive=True, upper_inclusive=True): + """Create a range from lower_version..upper_version. + + Args: + lower_version (Version): Version object representing lower bound of the range. + upper_version (Version): Version object representing upper bound of the range. + lower_inclusive (bool): Include lower_version into the span. + upper_inclusive (bool): Include upper_inclusive into the span. + Returns: + VersionRange: + """ + lower = (None if lower_version is None + else _LowerBound(lower_version, lower_inclusive)) + upper = (None if upper_version is None + else _UpperBound(upper_version, upper_inclusive)) + bound = _Bound(lower, upper) + + range = cls(None) + range.bounds = [bound] + return range + + @classmethod + def from_version(cls, version, op=None): + """Create a range from a version. + + Args: + version (Version): This is used as the upper/lower bound of + the range. + op (typing.Optional[str]): Operation as a string. One of: gt, >, gte, >=, lt, <, + lte, <=, eq, ==. If None, a bounded range will be created + that contains the version superset. + + Returns: + VersionRange: + """ + lower = None + upper = None + + if op is None: + lower = _LowerBound(version, True) + upper = _UpperBound(version.next(), False) + elif op in ("eq", "=="): + lower = _LowerBound(version, True) + upper = _UpperBound(version, True) + elif op in ("gt", ">"): + lower = _LowerBound(version, False) + elif op in ("gte", ">="): + lower = _LowerBound(version, True) + elif op in ("lt", "<"): + upper = _UpperBound(version, False) + elif op in ("lte", "<="): + upper = _UpperBound(version, True) + else: + raise VersionError("Unknown bound operation '%s'" % op) + + bound = _Bound(lower, upper) + range = cls(None) + range.bounds = [bound] + return range + + @classmethod + def from_versions(cls, versions): + """Create a range from a list of versions. + + This method creates a range that contains only the given versions and + no other. Typically the range looks like (for eg) ``==3|==4|==5.1``. + + Args: + versions (list[Version]): List of Version objects. + + Returns: + VersionRange: + """ + range = cls(None) + range.bounds = [] + for version in dedup(sorted(versions)): + lower = _LowerBound(version, True) + upper = _UpperBound(version, True) + bound = _Bound(lower, upper) + range.bounds.append(bound) + return range + + def to_versions(self): + """Returns exact version ranges as Version objects, or None if there + are no exact version ranges present. + + Returns: + typing.Optional[list[Version]]: + """ + versions = [] + for bound in self.bounds: + if bound.lower.inclusive and bound.upper.inclusive \ + and (bound.lower.version == bound.upper.version): + versions.append(bound.lower.version) + + return versions or None + + def contains_version(self, version): + """Returns True if version is contained in this range. + + Returns: + bool: + """ + if len(self.bounds) < 5: + # not worth overhead of binary search + for bound in self.bounds: + i = bound.version_containment(version) + if i == 0: + return True + if i == -1: + return False + else: + _, contains = self._contains_version(version) + return contains + + return False + + def iter_intersect_test(self, iterable, key=None, descending=False): + """Performs containment tests on a sorted list of versions. + + This is more optimal than performing separate containment tests on a + list of sorted versions. + + Args: + iterable: An ordered sequence of versioned objects. If the list + is not sorted by version, behaviour is undefined. + key (typing.Callable[typing.Any]): Function that returns a :class:`Version` given an object + from ``iterable``. If None, the identity function is used. + descending (bool): Set to True if ``iterable`` is in descending + version order. + + Returns: + ~collections.abc.Iterator[tuple[bool, typing.Any]]: An iterator that returns (bool, object) tuples, + where 'object' is the original object in ``iterable``, and the bool indicates whether + that version is contained in this range. + """ + return _ContainsVersionIterator(self, iterable, key, descending) + + def iter_intersecting(self, iterable, key=None, descending=False): + """Like :meth:iter_intersect_test`, but returns intersections only. + + Returns: + An iterator that returns items from `iterable` that intersect. + """ + return _ContainsVersionIterator( + self, iterable, key, descending, mode=_ContainsVersionIterator.MODE_INTERSECTING + ) + + def iter_non_intersecting(self, iterable, key=None, descending=False): + """Like :meth:`iter_intersect_test`, but returns non-intersections only. + + Returns: + An iterator that returns items from `iterable` that don't intersect. + """ + return _ContainsVersionIterator( + self, iterable, key, descending, mode=_ContainsVersionIterator.MODE_NON_INTERSECTING + ) + + def span(self): + """Return a contiguous range that is a superset of this range. + + Returns: + VersionRange: A range object representing the span of this range. For + example, the span of ``2+<4|6+<8`` would be ``2+<8``. + """ + other = VersionRange(None) + bound = _Bound(self.bounds[0].lower, self.bounds[-1].upper) + other.bounds = [bound] + return other + + # TODO have this return a new VersionRange instead - this currently breaks + # VersionRange immutability, and could invalidate __str__. + def visit_versions(self, func): + """Visit each version in the range, and apply a function to each. + + This is for advanced usage only. + + If ``func`` returns a :class:`Version`, this call will change the versions in + place. + + It is possible to change versions in a way that is nonsensical - for + example setting an upper bound to a smaller version than the lower bound. + Use at your own risk. + + Args: + func (typing.Callable[[Version], typing.Optional[Version]]): Takes a + version, and is applied to every version in the range. + If ``func`` returns a :class:`Version`, it will replace the existing version, + updating this :class:`VersionRange` instance in place. + + Returns: + None: + """ + for bound in self.bounds: + if bound.lower is not _LowerBound.min: + result = func(bound.lower.version) + if isinstance(result, Version): + bound.lower.version = result + + if bound.upper is not _UpperBound.inf: + result = func(bound.upper.version) + if isinstance(result, Version): + bound.upper.version = result + + def __contains__(self, version_or_range): + if isinstance(version_or_range, Version): + return self.contains_version(version_or_range) + else: + return self.issuperset(version_or_range) + + def __len__(self): + return len(self.bounds) + + def __invert__(self): + return self.inverse() + + def __and__(self, other): + return self.intersection(other) + + def __or__(self, other): + return self.union(other) + + def __add__(self, other): + return self.union(other) + + def __sub__(self, other): + inv = other.inverse() + return None if inv is None else self.intersection(inv) + + def __str__(self): + if self._str is None: + self._str = '|'.join(map(str, self.bounds)) + return self._str + + def __eq__(self, other): + return isinstance(other, VersionRange) and self.bounds == other.bounds + + def __lt__(self, other): + return (self.bounds < other.bounds) + + def __hash__(self): + return hash(tuple(self.bounds)) + + def _contains_version(self, version): + vbound = _Bound(_LowerBound(version, True)) + i = bisect_left(self.bounds, vbound) + if i and self.bounds[i - 1].contains_version(version): + return i - 1, True + if (i < len(self.bounds)) and self.bounds[i].contains_version(version): + return i, True + return i, False + + @classmethod + def _union(cls, bounds): + if len(bounds) < 2: + return bounds + + bounds_ = list(sorted(bounds)) + new_bounds = [] + prev_bound = None + upper = None + start = 0 + + for i, bound in enumerate(bounds_): + if i and ((bound.lower.version > upper.version) + or ((bound.lower.version == upper.version) + and (not bound.lower.inclusive) + and (not prev_bound.upper.inclusive))): + new_bound = _Bound(bounds_[start].lower, upper) + new_bounds.append(new_bound) + start = i + + prev_bound = bound + upper = bound.upper if upper is None else max(upper, bound.upper) + + new_bound = _Bound(bounds_[start].lower, upper) + new_bounds.append(new_bound) + return new_bounds + + @classmethod + def _intersection(cls, bounds1, bounds2): + new_bounds = [] + for bound1 in bounds1: + for bound2 in bounds2: + b = bound1.intersection(bound2) + if b: + new_bounds.append(b) + return new_bounds + + @classmethod + def _inverse(cls, bounds): + lbounds = [None] + ubounds = [] + + for bound in bounds: + if not bound.lower.version and bound.lower.inclusive: + ubounds.append(None) + else: + b = _UpperBound(bound.lower.version, not bound.lower.inclusive) + ubounds.append(b) + + if bound.upper.version == Version.inf: + lbounds.append(None) + else: + b = _LowerBound(bound.upper.version, not bound.upper.inclusive) + lbounds.append(b) + + ubounds.append(None) + new_bounds = [] + + for lower, upper in zip(lbounds, ubounds): + if not (lower is None and upper is None): + new_bounds.append(_Bound(lower, upper)) + + return new_bounds + + @classmethod + def _issuperset(cls, bounds1, bounds2): + lo = 0 + for bound2 in bounds2: + i = bisect_left(bounds1, bound2, lo=lo) + if i and bounds1[i - 1].contains_bound(bound2): + lo = i - 1 + continue + if (i < len(bounds1)) and bounds1[i].contains_bound(bound2): + lo = i + continue + return False + + return True + + @classmethod + def _intersects(cls, bounds1, bounds2): + # sort so bounds1 is the shorter list + bounds1, bounds2 = sorted((bounds1, bounds2), key=lambda x: len(x)) + + if len(bounds2) < 5: + # not worth overhead of binary search + for bound1 in bounds1: + for bound2 in bounds2: + if bound1.intersects(bound2): + return True + return False + + lo = 0 + for bound1 in bounds1: + i = bisect_left(bounds2, bound1, lo=lo) + if i and bounds2[i - 1].intersects(bound1): + return True + if (i < len(bounds2)) and bounds2[i].intersects(bound1): + return True + lo = max(i - 1, 0) + + return False + + +class _ContainsVersionIterator(object): + MODE_INTERSECTING = 0 + MODE_NON_INTERSECTING = 2 + MODE_ALL = 3 + + def __init__(self, range_, iterable, key=None, descending=False, mode=MODE_ALL): + self.mode = mode + self.range_ = range_ + self.index = None + self.nbounds = len(self.range_.bounds) + self._constant = True if range_.is_any() else None + self.fn = self._descending if descending else self._ascending + self.it = iter(iterable) + if key is None: + key = lambda x: x # noqa: E731 + self.keyfunc = key + + if mode == self.MODE_ALL: + self.next_fn = self._next + elif mode == self.MODE_INTERSECTING: + self.next_fn = self._next_intersecting + else: + self.next_fn = self._next_non_intersecting + + def __iter__(self): + return self + + def __next__(self): + return self.next_fn() + + def next(self): + return self.next_fn() + + def _next(self): + value = next(self.it) + if self._constant is not None: + return self._constant, value + + version = self.keyfunc(value) + intersects = self.fn(version) + return intersects, value + + def _next_intersecting(self): + while True: + value = next(self.it) + + if self._constant: + return value + elif self._constant is not None: + raise StopIteration + + version = self.keyfunc(value) + intersects = self.fn(version) + if intersects: + return value + + def _next_non_intersecting(self): + while True: + value = next(self.it) + + if self._constant: + raise StopIteration + elif self._constant is not None: + return value + + version = self.keyfunc(value) + intersects = self.fn(version) + if not intersects: + return value + + @property + def _bound(self): + if self.index < self.nbounds: + return self.range_.bounds[self.index] + else: + return None + + def _ascending(self, version): + if self.index is None: + self.index, contains = self.range_._contains_version(version) + bound = self._bound + if contains: + if not bound.upper_bounded(): + self._constant = True + return True + elif bound is None: # past end of last bound + self._constant = False + return False + else: + return False # there are more bound(s) ahead + else: + bound = self._bound + j = bound.version_containment(version) + if j == 0: + return True + elif j == -1: + return False + else: + while True: + self.index += 1 + bound = self._bound + if bound is None: # past end of last bound + self._constant = False + return False + else: + j = bound.version_containment(version) + if j == 0: + if not bound.upper_bounded(): + self._constant = True + return True + elif j == -1: + return False + + def _descending(self, version): + if self.index is None: + self.index, contains = self.range_._contains_version(version) + bound = self._bound + if contains: + if not bound.lower_bounded(): + self._constant = True + return True + elif bound is None: # past end of last bound + self.index = self.nbounds - 1 + return False + elif self.index == 0: # before start of first bound + self._constant = False + return False + else: + self.index -= 1 + return False + else: + bound = self._bound + j = bound.version_containment(version) + if j == 0: + return True + elif j == 1: + return False + else: + while self.index: + self.index -= 1 + bound = self._bound + j = bound.version_containment(version) + if j == 0: + if not bound.lower_bounded(): + self._constant = True + return True + elif j == 1: + return False + + self._constant = False # before start of first bound + return False diff --git a/src/rezgui/dialogs/AboutDialog.py b/src/rezgui/dialogs/AboutDialog.py index e94cb311b..2e90c9d72 100644 --- a/src/rezgui/dialogs/AboutDialog.py +++ b/src/rezgui/dialogs/AboutDialog.py @@ -5,7 +5,7 @@ from Qt import QtCore, QtWidgets from rezgui.util import create_pane, get_icon from rez import __version__ -from rez.vendor.version.version import Version +from rez.version import Version class AboutDialog(QtWidgets.QDialog): diff --git a/src/rezgui/dialogs/ResolveDialog.py b/src/rezgui/dialogs/ResolveDialog.py index 9ca54e82d..ad0f66dd2 100644 --- a/src/rezgui/dialogs/ResolveDialog.py +++ b/src/rezgui/dialogs/ResolveDialog.py @@ -11,7 +11,7 @@ from rezgui.objects.ResolveThread import ResolveThread from rezgui.objects.App import app from rez.vendor.six.six import StringIO -from rez.vendor.version.requirement import Requirement +from rez.version import Requirement from rez.config import config diff --git a/src/rezgui/widgets/BrowsePackageWidget.py b/src/rezgui/widgets/BrowsePackageWidget.py index 334d1c263..39eff4dfe 100644 --- a/src/rezgui/widgets/BrowsePackageWidget.py +++ b/src/rezgui/widgets/BrowsePackageWidget.py @@ -9,7 +9,7 @@ from rezgui.widgets.PackageTabWidget import PackageTabWidget from rezgui.mixins.ContextViewMixin import ContextViewMixin from rezgui.objects.App import app -from rez.vendor.version.requirement import Requirement +from rez.version import Requirement class BrowsePackageWidget(QtWidgets.QWidget, ContextViewMixin): diff --git a/src/rezgui/widgets/ChangelogEdit.py b/src/rezgui/widgets/ChangelogEdit.py index 2d4d0a7aa..8ecf28cba 100644 --- a/src/rezgui/widgets/ChangelogEdit.py +++ b/src/rezgui/widgets/ChangelogEdit.py @@ -3,11 +3,10 @@ from Qt import QtCore, QtWidgets, QtGui -import cgi - +import rez.utils.py23 def plaintext_to_html(txt): - out = cgi.escape(txt) + out = rez.utils.py23.escape(txt) out = out.replace('\t', " ") out = out.replace(' ', " ") out = out.replace('\n', "
") diff --git a/src/rezgui/widgets/ContextTableWidget.py b/src/rezgui/widgets/ContextTableWidget.py index 3b96b748e..c786af516 100644 --- a/src/rezgui/widgets/ContextTableWidget.py +++ b/src/rezgui/widgets/ContextTableWidget.py @@ -12,8 +12,8 @@ from rezgui.models.ContextModel import ContextModel from rezgui.objects.App import app from rez.packages import iter_packages -from rez.vendor.version.requirement import Requirement -from rez.vendor.version.version import VersionRange +from rez.version import Requirement +from rez.version import VersionRange from functools import partial import os.path diff --git a/src/rezgui/widgets/PackageLineEdit.py b/src/rezgui/widgets/PackageLineEdit.py index 9fc692c2a..82cf34841 100644 --- a/src/rezgui/widgets/PackageLineEdit.py +++ b/src/rezgui/widgets/PackageLineEdit.py @@ -6,7 +6,7 @@ from rezgui.models.ContextModel import ContextModel from rezgui.mixins.ContextViewMixin import ContextViewMixin from rez.packages import get_completions, iter_packages -from rez.vendor.version.requirement import Requirement +from rez.version import Requirement class PackageLineEdit(QtWidgets.QLineEdit, ContextViewMixin): diff --git a/src/rezgui/widgets/VariantCellWidget.py b/src/rezgui/widgets/VariantCellWidget.py index c2a86c6f1..3447c1ea0 100644 --- a/src/rezgui/widgets/VariantCellWidget.py +++ b/src/rezgui/widgets/VariantCellWidget.py @@ -9,8 +9,8 @@ from rez.packages import PackageSearchPath from rez.package_filter import PackageFilterList from rez.resolved_context import PatchLock, get_lock_request -from rez.vendor.version.requirement import RequirementList -from rez.vendor.version.version import VersionRange +from rez.version import RequirementList +from rez.version import VersionRange from functools import partial diff --git a/src/rezgui/widgets/VariantHelpWidget.py b/src/rezgui/widgets/VariantHelpWidget.py index 1bd63be00..baa803cc9 100644 --- a/src/rezgui/widgets/VariantHelpWidget.py +++ b/src/rezgui/widgets/VariantHelpWidget.py @@ -6,7 +6,7 @@ from rezgui.util import create_pane, get_icon_widget from rezgui.mixins.ContextViewMixin import ContextViewMixin from rezgui.widgets.PackageLoadingWidget import PackageLoadingWidget -from rez.vendor.version.version import VersionRange +from rez.version import VersionRange from rez.package_help import PackageHelp from functools import partial diff --git a/src/rezgui/widgets/VariantVersionsTable.py b/src/rezgui/widgets/VariantVersionsTable.py index 1b5fb00a3..2f323b7a5 100644 --- a/src/rezgui/widgets/VariantVersionsTable.py +++ b/src/rezgui/widgets/VariantVersionsTable.py @@ -7,7 +7,7 @@ from rez.package_filter import PackageFilterList from rezgui.util import get_timestamp_str, update_font, get_icon_widget, create_pane from rez.packages import iter_packages -from rez.vendor.version.version import VersionRange +from rez.version import VersionRange class VariantVersionsTable(QtWidgets.QTableWidget, ContextViewMixin): diff --git a/src/rezgui/widgets/VariantVersionsWidget.py b/src/rezgui/widgets/VariantVersionsWidget.py index d8c4b8287..2557f6f7b 100644 --- a/src/rezgui/widgets/VariantVersionsWidget.py +++ b/src/rezgui/widgets/VariantVersionsWidget.py @@ -9,7 +9,7 @@ from rezgui.widgets.ChangelogEdit import ChangelogEdit from rezgui.mixins.ContextViewMixin import ContextViewMixin from rez.utils.formatting import positional_number_string -from rez.vendor.version.version import VersionRange +from rez.version import VersionRange class VariantVersionsWidget(PackageLoadingWidget, ContextViewMixin): diff --git a/src/rezplugins/build_system/custom.py b/src/rezplugins/build_system/custom.py index 165a10423..44f602856 100644 --- a/src/rezplugins/build_system/custom.py +++ b/src/rezplugins/build_system/custom.py @@ -10,7 +10,7 @@ from builtins import map except ImportError: pass -from pipes import quote +from rez.utils.py23 import quote import functools import os.path import sys diff --git a/src/rezplugins/package_repository/filesystem.py b/src/rezplugins/package_repository/filesystem.py index 9652f1e24..f506ad038 100644 --- a/src/rezplugins/package_repository/filesystem.py +++ b/src/rezplugins/package_repository/filesystem.py @@ -35,7 +35,7 @@ from rez.backport.lru_cache import lru_cache from rez.vendor.schema.schema import Schema, Optional, And, Use, Or from rez.vendor.six import six -from rez.vendor.version.version import Version, VersionRange +from rez.version import Version, VersionRange basestring = six.string_types[0] @@ -208,6 +208,7 @@ def _load(self): return data + # TODO: Deprecate def _load_old_formats(self): data = None @@ -245,6 +246,7 @@ def _update_changelog(file_format, data): if not maxlen: return data + # TODO: Deprecate if file_format == FileFormat.yaml: changelog = data.get("changelog") if changelog: @@ -334,6 +336,7 @@ def iter_packages(self): yield package def _load(self): + # TODO: Deprecate: What is self.ext? format_ = FileFormat[self.ext] data = load_from_file( self.filepath, @@ -433,6 +436,7 @@ def _root(self): class FileSystemPackageRepository(PackageRepository): """A filesystem-based package repository. + TODO: Deprecate YAML Packages are stored on disk, in either 'package.yaml' or 'package.py' files. These files are stored into an organised directory structure like so: @@ -1182,6 +1186,7 @@ def _get_file(self, path, package_filename=None): package_filenames = _settings.package_filenames for name in package_filenames: + # TODO: Deprecate YAML for format_ in (FileFormat.py, FileFormat.yaml): filename = "%s.%s" % (name, format_.extension) filepath = os.path.join(path, filename) diff --git a/src/rezplugins/package_repository/memory.py b/src/rezplugins/package_repository/memory.py index e6b018173..74d50731e 100644 --- a/src/rezplugins/package_repository/memory.py +++ b/src/rezplugins/package_repository/memory.py @@ -10,7 +10,7 @@ PackageResourceHelper, package_pod_schema from rez.utils.formatting import is_valid_package_name from rez.utils.resources import ResourcePool, cached_property -from rez.vendor.version.requirement import VersionedObject +from rez.version import VersionedObject # This repository type is used when loading 'developer' packages (a package.yaml diff --git a/src/rezplugins/shell/csh.py b/src/rezplugins/shell/csh.py index 1de586d81..b50d34033 100644 --- a/src/rezplugins/shell/csh.py +++ b/src/rezplugins/shell/csh.py @@ -5,7 +5,6 @@ """ CSH shell """ -import pipes import os.path import subprocess import re @@ -16,6 +15,7 @@ from rez.utils.platform_ import platform_ from rez.shells import UnixShell from rez.rex import EscapedString +from rez.utils.py23 import quote class CSH(UnixShell): @@ -105,7 +105,7 @@ def escape_string(self, value, is_path=False, is_shell_path=False): for is_literal, txt in value.strings: if is_literal: - txt = pipes.quote(txt) + txt = quote(txt) if not txt.startswith("'"): txt = "'%s'" % txt else: diff --git a/src/rezplugins/shell/sh.py b/src/rezplugins/shell/sh.py index 9e898ce11..9b4730acd 100644 --- a/src/rezplugins/shell/sh.py +++ b/src/rezplugins/shell/sh.py @@ -7,13 +7,13 @@ """ import os import os.path -import pipes import subprocess from rez.config import config from rez.utils.execution import Popen from rez.utils.platform_ import platform_ from rez.shells import UnixShell from rez.rex import EscapedString +from rez.utils.py23 import quote class SH(UnixShell): @@ -135,7 +135,7 @@ def escape_string( for is_literal, txt in value.strings: if is_literal: - txt = pipes.quote(txt) + txt = quote(txt) if not txt.startswith("'"): txt = "'%s'" % txt else: diff --git a/src/rezplugins/shell/tcsh.py b/src/rezplugins/shell/tcsh.py index da5f1c27c..58b44f7a6 100644 --- a/src/rezplugins/shell/tcsh.py +++ b/src/rezplugins/shell/tcsh.py @@ -9,8 +9,8 @@ from rezplugins.shell.csh import CSH from rez import module_root_path from rez.rex import EscapedString +from rez.utils.py23 import quote import os.path -import pipes class TCSH(CSH): @@ -26,7 +26,7 @@ def escape_string(self, value, is_path=False): for is_literal, txt in value.strings: if is_literal: - txt = pipes.quote(txt) + txt = quote(txt) if not txt.startswith("'"): txt = "'%s'" % txt else: diff --git a/wiki/README.md b/wiki/README.md deleted file mode 100644 index 69aeab1f8..000000000 --- a/wiki/README.md +++ /dev/null @@ -1,16 +0,0 @@ -# rez-wiki - -This directory holds the content used to produce the Rez Wiki documentation -found [here](https://github.com/AcademySoftwareFoundation/rez/wiki). The wiki is updated by the -`Wiki` Github workflow, on release. - -You should include relevant wiki updates with your code PRs. - -## Testing - -To test wiki updates locally: - -* Make your changes to any pages/* or media/* files; -* Run `python ./generate-wiki.py` -* View the resulting content in the `out` directory using your markdown viewer - of choice (we suggest [grip](https://github.com/joeyespo/grip)). diff --git a/wiki/generate-wiki.py b/wiki/generate-wiki.py deleted file mode 100644 index 9a82c113c..000000000 --- a/wiki/generate-wiki.py +++ /dev/null @@ -1,435 +0,0 @@ -""" -Script to generate wiki content. -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -import argparse -from collections import defaultdict -import errno -from io import open -import os -import re -import subprocess -import shutil -import sys - - -# py3.7+ only -if sys.version_info[:2] < (3, 7): - print("update-wiki.py: use python-3.7 or greater", file=sys.stderr) - sys.exit(1) - - -THIS_FILE = os.path.abspath(__file__) -THIS_DIR = os.path.dirname(THIS_FILE) -REZ_SOURCE_DIR = os.getenv("REZ_SOURCE_DIR", os.path.dirname(THIS_DIR)) - -OUT_DIR = "out" -GITHUB_REPO = "unknown/rez" -GITHUB_BRANCH = "main" -GITHUB_WORKFLOW = "Wiki" - - -def add_toc(txt): - """Add github-style ToC to start of md content. - """ - lines = txt.split('\n') - toc_lines = [] - mindepth = None - - for line in lines: - if not line.startswith('#'): - continue - - parts = line.split() - - hashblock = parts[0] - if set(hashblock) != set(["#"]): - continue - - depth = len(hashblock) - if mindepth is None: - mindepth = depth - depth -= mindepth - - toc_lines.append("%s- [%s](#%s)" % ( - ' ' * 4 * depth, - ' '.join(parts[1:]), - '-'.join(x.lower() for x in parts[1:]) - )) - - if not toc_lines: - return txt - - return '\n'.join(toc_lines) + "\n\n" + txt - - -def creating_configuring_rez_md(txt): - lines = txt.split('\n') - - start = None - end = None - for i, line in enumerate(lines): - if "__DOC_START__" in line: - start = i - elif "__DOC_END__" in line: - end = i - - lines = lines[start:end + 1] - assign_regex = re.compile("^([a-z0-9_]+) =") - settings = {} - - # parse out settings and their comment - for i, line in enumerate(lines): - m = assign_regex.match(line) - if not m: - continue - - start_defn = i - end_defn = i - while lines[end_defn].strip() and not lines[end_defn].startswith('#'): - end_defn += 1 - - defn_lines = lines[start_defn:end_defn] - defn_lines = [(" " + x) for x in defn_lines] # turn into code block - defn = '\n'.join(defn_lines) - - end_comment = i - while not lines[end_comment].startswith('#'): - end_comment -= 1 - - start_comment = end_comment - while lines[start_comment].startswith('#'): - start_comment -= 1 - start_comment += 1 - - comments = lines[start_comment:end_comment + 1] - comments = [x[2:] for x in comments] # drop leading '# ' - comment = '\n'.join(comments) - - varname = m.groups()[0] - settings[varname] = (defn, comment) - - # generate md text - md = [] - - for varname, (defn, comment) in sorted(settings.items()): - md.append("### %s" % varname) - md.append("") - md.append(defn) - md.append("") - md.append(comment) - md.append("") - - md = '\n'.join(md) - return md - - -def create_contributors_md(src_path): - # Make sure aliases KEY is fully lowercase to match correctly! - aliases = { - "allan.johns": "Allan Johns", - "allan johns": "Allan Johns", - "ajohns": "Allan Johns", - "nerdvegas": "Allan Johns", - "nerdvegas@gmail.com": "Allan Johns", - "method": "Allan Johns", - "rachel johns": "Allan Johns", - "root": "Allan Johns", - "(no author)": "Allan Johns", - - "mylene pepe": "Mylene Pepe", - "michael.morehouse": "Michael Morehouse", - "phunter.nz": "Philip Hunter", - "joe yu": "Joseph Yu", - "j0yu": "Joseph Yu", - "fpiparo": "Fabio Piparo" - } - out = subprocess.check_output( - ["git", "shortlog", "-sn", "HEAD"], - encoding="utf-8", - cwd=src_path, - ) - contributors = defaultdict(int) - regex = re.compile( - r'^\s*(?P\d+)\s+(?P.+)\s*$', - flags=re.MULTILINE | re.UNICODE - ) - - for match in regex.finditer(out): - author = match.group('author') - author_html = "%s
" % aliases.get(author.lower(), author) - contributors[author_html] += int(match.group('commits')) - - return '\n'.join( - author_html for author_html, commit_count in - sorted(contributors.items(), key=lambda x: x[1], reverse=True) - ) - - -def process_markdown_files(): - pagespath = os.path.join(THIS_DIR, "pages") - user, repo_name = GITHUB_REPO.split('/') - processed_files = {} - - src_path = REZ_SOURCE_DIR - if src_path is None: - print( - "Must provide REZ_SOURCE_DIR which points at root of " - "rez source clone", file=sys.stderr, - ) - sys.exit(1) - - def apply_replacements(filename, replacements=None): - srcfile = os.path.join(pagespath, filename) - - with open(srcfile, encoding='utf-8') as f: - txt = f.read() - - # add standard replacements - repls = { - "__GITHUB_REPO__": GITHUB_REPO, - "__GITHUB_USER__": user, - "__GITHUB_BRANCH__": GITHUB_BRANCH, - "__REPO_NAME__": repo_name - } - - repls.update(replacements or {}) - - for src_txt, repl_txt in repls.items(): - txt = txt.replace(src_txt, repl_txt) - - return txt - - # generate Configuring-Rez.md - filepath = os.path.join(src_path, "src", "rez", "rezconfig.py") - with open(filepath) as f: - txt = f.read() - - processed_files["Configuring-Rez.md"] = apply_replacements( - "Configuring-Rez.md", - { - "__REZCONFIG_MD__": creating_configuring_rez_md(txt) - } - ) - - # generate Credits.md - md = create_contributors_md(src_path) - processed_files["Credits.md"] = apply_replacements( - "Credits.md", - { - "__CONTRIBUTORS_MD__": md - } - ) - - # generate Command-Line-Tools.md - processed_files["Command-Line-Tools.md"] = apply_replacements( - "Command-Line-Tools.md", - { - "__GENERATED_MD__": create_clitools_markdown(src_path) - } - ) - - # all other markdown files - for name in os.listdir(pagespath): - if name not in processed_files: - processed_files[name] = apply_replacements(name) - - # iterate over every file, add a ToC, and write it out - for name, txt in processed_files.items(): - destfile = os.path.join(OUT_DIR, name) - txt = add_toc(txt) - with open(destfile, 'w', encoding="utf-8") as out: - out.write(txt) - - -def create_clitools_markdown(src_path): - """Generate the formatted markdown for each rez cli tool. - - Hot-import rez cli library to get parsers. - - Args: - src_path (str): Full path to the rez source code repository. - - Returns: - str: Generated markdown text. - """ - sys.path.insert(0, os.path.join(src_path, "src")) - try: - from rez.cli._main import setup_parser - from rez.cli._util import LazySubParsersAction - - main_parser = setup_parser() - command_help = [] - parsers = [main_parser] - for action in main_parser._actions: - if isinstance(action, LazySubParsersAction): - parsers += action.choices.values() - - for arg_parser in parsers: - arg_parser.formatter_class = MarkdownHelpFormatter - command_help.append(arg_parser.format_help()) - finally: - sys.path.pop(0) - - return "\n\n\n".join(command_help) - - -class MarkdownHelpFormatter(argparse.ArgumentDefaultsHelpFormatter): - - def _format_usage(self, usage, actions, groups, prefix): - """Override to produce markdown title and code block formatting. - - Return: - str: Markdown title and code block formatted usage. - """ - prefix_was_none = prefix is None - if prefix_was_none: - prefix = "# {self._prog}\n```\n".format(self=self) - - super_format_usage = super(MarkdownHelpFormatter, self)._format_usage - formatted_usage = super_format_usage(usage, actions, groups, prefix) - - if prefix_was_none: - # Fix extra spaces calculated from old "usage: rez ..." prompt - extra_spaces = "{newline:<{count}} ".format( - newline="\n", - count=len("usage: {self._prog}".format(self=self)) - ) - formatted_usage = formatted_usage[:-1] + "```\n" - formatted_usage = formatted_usage.replace(extra_spaces, "\n") - - return formatted_usage - - def remap_heading(self, heading): - """Remap argparse headings to shorter, markdown formatted headings. - - Args: - heading (str): Original heading to remap and format. - - Returns: - str: Remapped and formatted heading, if any. - """ - if heading == "optional arguments": - return "\n**Flags**\n" - elif heading == "positional arguments": - return "" if self._prog == "rez" else "\n**Arguments**\n" - else: - return heading - - def start_section(self, heading): - """Extend to remap optional/positional arguments headings. - - Args: - heading (str): Section heading to parse. - """ - if self.remap_heading(heading) == heading: - super(MarkdownHelpFormatter, self).start_section(heading) - else: - self._indent() - self._add_item(self.remap_heading, [heading]) - super(MarkdownHelpFormatter, self).start_section(argparse.SUPPRESS) - - def _fill_text(self, text, width, indent): - """No indent for description, keep subsequent indents. - - Return: - str: Description but without leading indents. - """ - super_fill_text = super(MarkdownHelpFormatter, self)._fill_text - return super_fill_text(text, width, indent).lstrip() - - def _format_action(self, action): - """Extend to format rez sub commands as table of links. - - Returns: - str: Formatted help text for an action. - """ - backup_width = self._width - if self._prog == "rez" and action.nargs is None: - self._width = 2000 # Temporary thicc width to avoid wrapping - - try: - super_format = super(MarkdownHelpFormatter, self)._format_action - help_text = super_format(action) - finally: - self._width = backup_width - - if self._prog == "rez": - # Sub commands, format them with links - if action.nargs is None: - help_text = re.sub( - r'^\s+(\S+)(\s+)', - r'[\1](#rez-\1)\2| ', - help_text - ) - - # Sub commands heading, format as table heading - elif action.metavar == "COMMAND": - help_text = re.sub( - r'^\s+COMMAND', - "`COMMAND` | Description\n----|----", - help_text - ) - - return help_text - - -class UpdateWikiParser(argparse.ArgumentParser): - def __init__(self, **kwargs): - super(UpdateWikiParser, self).__init__(**kwargs) - - self.add_argument( - "--github-repo", - default=GITHUB_REPO, - dest="repo", - help="Url to GitHub repository without leading github.com/" - ) - self.add_argument( - "--github-branch", - default=GITHUB_BRANCH, - dest="branch", - help="Name of git branch that is generating the Wiki" - ) - self.add_argument( - "--github-workflow", - default=GITHUB_WORKFLOW, - dest="workflow", - help="Name of GitHub workflow that is generating the Wiki" - ) - self.add_argument( - "--out", - default=OUT_DIR, - dest="dir", - help="Output dir" - ) - - -if __name__ == "__main__": - # Quick check for "git" and throw meaningful error message - try: - subprocess.check_call(["git", "--version"]) - except OSError as error: - if error.errno == errno.ENOENT: - raise OSError(errno.ENOENT, '"git" needed but not found in PATH') - raise - - args = UpdateWikiParser().parse_args() - GITHUB_REPO = args.repo - GITHUB_BRANCH = args.branch - GITHUB_WORKFLOW = args.workflow - OUT_DIR = os.path.abspath(args.dir) - - if not os.path.exists(OUT_DIR): - os.makedirs(OUT_DIR) - - shutil.copytree( - os.path.join(THIS_DIR, 'media'), - os.path.join(OUT_DIR, 'media'), - ) - - os.chdir(OUT_DIR) - process_markdown_files() diff --git a/wiki/media/icons/info.png b/wiki/media/icons/info.png deleted file mode 100644 index 954f5e81c..000000000 Binary files a/wiki/media/icons/info.png and /dev/null differ diff --git a/wiki/media/icons/new.png b/wiki/media/icons/new.png deleted file mode 100644 index f44a9428e..000000000 Binary files a/wiki/media/icons/new.png and /dev/null differ diff --git a/wiki/media/icons/under_construction.png b/wiki/media/icons/under_construction.png deleted file mode 100644 index 24deabc5d..000000000 Binary files a/wiki/media/icons/under_construction.png and /dev/null differ diff --git a/wiki/media/icons/warning.png b/wiki/media/icons/warning.png deleted file mode 100644 index 91677b5f1..000000000 Binary files a/wiki/media/icons/warning.png and /dev/null differ diff --git a/wiki/media/other_pkg_mgr.png b/wiki/media/other_pkg_mgr.png deleted file mode 100644 index f3891daaf..000000000 Binary files a/wiki/media/other_pkg_mgr.png and /dev/null differ diff --git a/wiki/media/pkg_path_anatomy.png b/wiki/media/pkg_path_anatomy.png deleted file mode 100644 index d7b660495..000000000 Binary files a/wiki/media/pkg_path_anatomy.png and /dev/null differ diff --git a/wiki/media/rez_banner_128.png b/wiki/media/rez_banner_128.png deleted file mode 100644 index bc7769aec..000000000 Binary files a/wiki/media/rez_banner_128.png and /dev/null differ diff --git a/wiki/media/rez_deps_simple_eg.png b/wiki/media/rez_deps_simple_eg.png deleted file mode 100644 index 59e2ba485..000000000 Binary files a/wiki/media/rez_deps_simple_eg.png and /dev/null differ diff --git a/wiki/media/rez_env.png b/wiki/media/rez_env.png deleted file mode 100644 index ef6e19c0d..000000000 Binary files a/wiki/media/rez_env.png and /dev/null differ diff --git a/wiki/media/rez_pkg_mgr.png b/wiki/media/rez_pkg_mgr.png deleted file mode 100644 index 87e1b162d..000000000 Binary files a/wiki/media/rez_pkg_mgr.png and /dev/null differ diff --git a/wiki/pages/Basic-Concepts.md b/wiki/pages/Basic-Concepts.md deleted file mode 100644 index 95a8aa977..000000000 --- a/wiki/pages/Basic-Concepts.md +++ /dev/null @@ -1,369 +0,0 @@ -## Overview - -Rez manages packages. You request a list of packages from rez, and it resolves this request, if -possible. If the resolution is not possible, the system supplies you with the relevant information -to determine why this is so. You typically want to resolve a list of packages because you want to -create an environment in which you can use them in combination, without conflicts occurring. A -conflict occurs when there is a request for two or more different versions of the same package - a -version clash. - -Rez lets you describe the environment you want in a natural way. For example, you can say: -“I want an environment with...” - -* “...the latest version of houdini” -* “...maya-2009.1” -* “...the latest rv and the latest maya and houdini-11.something” -* “...rv-3.something or greater” -* “...the latest houdini which works with boost-1.37.0” -* “...PyQt-2.2 or greater, but less than PyQt-4.5.3” - -In many examples in this documentation we will use the -[rez-env](Command-Line-Tools#rez-env) command line tool. This tool takes a list of package -requests and creates the resulting configured environment. It places you in a subshell - simply -exit the shell to return to a non-configured environment. - -## Versions - -Rez version numbers are alphanumeric - they support any combination of numbers, letters and -underscores. A version number is a set of *tokens*, separated by either dot or dash. For example, -here is a list of valid package version numbers: - -* 1 -* 1.0.0 -* 3.2.build_13 -* 4.rc1 -* 10a-5 - -Version number tokens follow a strict ordering schema and are case sensitive. Underscore is the -smallest character, followed by letters (a-z and A-Z), followed by numbers. The ordering rules are -like so: - -* Underscore before everything else; -* Letters alphabetical, and before numbers; -* Lowercase letters before uppercase; -* Zero-padded numbers before equivalent non-padded (or less padded) number ('01' is < '1'); -* If a token contains a combination of numbers and letters, it is internally split into groups -containing only numbers or only letters, and the resulting list is compared using the same rules -as above. - -The following table shows some example version token comparisons: - -smaller token | larger token ---------------|------------- -0 | 1 -a | b -a | A -a | 3 -_5 | 2 -ham | hamster -alpha | beta -alpha | bob -02 | 2 -002 | 02 -13 | 043 -3 | 3a -beta3 | 3beta - -Versions are compared by their token lists. The token delimiter (usually dot, but can also be dash) -is ignored for comparison purposes - thus the versions '1.0.0' and '1-0.0' are equivalent. If two -versions share the same token list prefix, the longer version is greater - thus '1.0.0' is a higher -version than '1.0'. - -Note that no special importance is given to specific characters or letters in Rez version numbers - -the terms 'alpha' and 'beta' for example have no special meaning. Similarly, the number of tokens in -a version number doesn't matter, you can have as many as you like. While you are encouraged to use -semantic versioning (see here), it is not enforced. - -## Packages - -A *package* is a versioned piece of software, that may have dependencies on other packages. Packages -are self-contained - they have a single package definition file (typically *package.py*), which -describes everything we need to know about the package in order to use it. Rez manages any kind of -package, whether it be a python package, compiled package, or simply build code or configuration -data. - -Here is an example package definition file (see [here](Package-Definition-Guide) for further details -of each attribute): - - name = "foo" - - version = "1.0.0" - - description = "Something that does foo-like things." - - requires = [ - "python-2.6", - "utils-1.1+<2" - ] - - tools = [ - "fooify" - ] - - def commands(): - env.PYTHONPATH.append("{root}/python") - env.PATH.append("{root}/bin") - -The *requires* section defines the requirements of the package. The *commands* section describes -what happens when this package is added to an environment. Here, the *bin* directory in the package -installation is appended to *PATH*, and similarly the *python* subdirectory is appended to -*PYTHONPATH*. - -## Package Repositories - -Packages are installed into package repositories. A package repository is a directory on disk, with -packages and their versions laid out in a known structure underneath. Going on with our (foo, bah, -eek) example, here is how the package repository might look: - - /packages/inhouse/foo/1.1 - /1.2 - /1.3 - /packages/inhouse/bah/2 - /3 - /4 - /packages/inhouse/eek/2.5 - /2.6 - /2.7 - - # more detailed example of foo-1.1 - /packages/inhouse/foo/1.1/package.py - /python/ - /bin/ - -Here we have a package repository under the directory */packages/inhouse*. The actual package content -(files, executables etc) is installed into each leaf-node version directory, as shown for *foo-1.1*. -The package definition file, in this case *package.py*, is always stored at the root of the package - -right under the version directory for that package. - -Rez only requires that the package's *package.py* file is at the root of the package installation. The -layout of the rest of the package - for example, the *python* and *bin* directories - is completely -up to the package's own build to determine. You should expect to see a package's *commands* section -match up with its installation though. For example, notice how the path for foo's python files and -binaries match what its package commands specified from earlier - "{root}/python" and "{root}/bin" -will expand to these paths respectively. - -## Package Search Path - -Rez finds packages using a search path in much the same way that python finds python modules using -PYTHONPATH. You can find out what the search path is, using the rez command line tool rez-config -(which you can also use to find any other rez setting): - - ]$ rez-config packages_path - - /home/ajohns/packages - - /packages/inhouse - - /packages/vendor - -If the same package appears in two or more repositories on the search path, the earlier package is -used in preference. This happens at the version level - an earlier package "foo-1.0.0" will hide a -later package "foo-1.0.0", but not "foo-1.2.0". - -The example search path shown is a typical setting. There are some central repositories later in the -search path, where packages are released to so everyone can use them. But there is also a local -package path at the front of the search path. This is where packages go that are being locally -developed by a user. Having this at the start of the searchpath allows developers to resolve -environments that pull in test packages in preference to released ones, so they can test a package -before releasing it for general use. - -You can change the packages search path in several ways. A common way is to set the REZ_PACKAGES_PATH -environment variable; see [Configuring Rez](Configuring-Rez) for more configuration options. - -## Package Commands - -The *commands* section of the package definition determines how the environment is configured in -order to use it. It is a python function, but note that if any imports are used, they must appear -within the body of this function. - -Consider this commands example: - - def commands(): - env.PYTHONPATH.append("{root}/python") - env.PATH.append("{root}/bin") - -This is a typical example, where a package adds its source path to *PYTHONPATH*, and its tools to -*PATH*. See [here](Package-Commands) for details on what can be done within the *commands* section, -as well as details on what order package commands are executed in. - -## Package Requests - -A *package request* is a string with a special syntax which matches a number of possible package -versions. You use package requests in the requires section of a package definition file, and also -when creating your own configured environment directly using tools such as *rez-env*. - -For example, here is a request (using the *rez-env* tool) to create an environment containing -*python* version 2.6 or greater, and *my_py_utils* version 5.4 or greater, but less than 6: - - ]$ rez-env 'python-2.6+' 'my_py_utils-5.4+<6' - -Here are some example package requests: - -package request | description | example versions within request -----------------|-------------------------------------|-------------------------------- -foo | Any version of foo. | foo-1, foo-0.4, foo-5.0, foo-2.0.alpha -foo-1 | Any version of foo-1[.x.x...x]. | foo-1, foo-1.0, foo-1.2.3 -foo-1+ | foo-1 or greater. | foo-1, foo-1.0, foo-1.2.3, foo-7.0.0 -foo-1.2+<2 | foo-1.2 or greater, but less than 2 | foo-1.2.0, foo-1.6.4, foo-1.99 -foo<2 | Any version of foo less than 2 | foo-1, foo-1.0.4 -foo==2.0.0 | Only version 2.0.0 exactly | foo-2.0.0 -foo-1.3\|5+ | OR'd requests | foo-1.3.0, foo-6.0.0 - -### The Conflict Operator - -The '!' operator is called the *conflict* operator, and is used to define an incompatibility -between packages, or to specify that you do *not* want a package version present. For example, -consider the command: - - ]$ rez-env maya_utils '!maya-2015.6' - -This specifies that you require any version of *maya_utils*, but that any version of *maya* within -2015.6 (and this includes 2015.6.1 and so on) is not acceptable. - -### Weak References - -The '~' operator is called the *weak reference* operator. It forces a package version to be within -the specified range if present, but does not actually require the package. For example, consider -the command: - - ]$ rez-env foo '~nuke-9.rc2' - -This request may or may not pull in the *nuke* package, depending on the requirements of *foo*; -however, if nuke *is* present, it must be within the version 9.rc2. - -Weak references are useful in certain cases. For example, applications such as *nuke* and *maya* -sometimes ship with their own version of *python*. Their rez packages don't have a requirement on -*python* (they have their own embedded version already). However often other python libraries are -used both inside and outside of these applications, and those packages *do* have a python -requirement. So, to make sure that they're using a compatible python version when used within the -app, the app may define a *weak package reference* to their relevant python version, like so: - - # in maya's package.py - requires = [ - "~python-2.7.3" - ] - -This example ensures that any package that uses python, will use the version compatible with maya -when maya is present in the environment. - -## Implicit Packages - -The *implicit packages* are a list of package requests that are automatically added to every rez -request (for example, when you use *rez-env*). They are set by the configuration setting -*implicit_packages*. The default setting looks like so: - - implicit_packages = [ - "~platform=={system.platform}", - "~arch=={system.arch}", - "~os=={system.os}", - ] - -Rez models the current system - the platform, architecture and operating systems - as packages -themselves. The default implicits are a set of *weak requirements* on each of *platform*, *arch* and -*os*. This ensures that if any platform-dependent package is requested, the platform, architecture -and/or operating system it depends on, matches the current system. - -The list of implicits that were used in a request are printed by *rez-env* when you enter the newly -configured subshell, and are also printed by the *rez-context* tool. - -## Dependency Resolving - -Rez contains a solving algorithm that takes a *request* - a list of package requests - and produces -a *resolve* - a final list of packages that satisfy the request. The algorithm avoids version -conflicts - two or more different versions of the same package at once. - -When you submit a request to rez, it finds a solution for that request that aims to give you the -latest possible version of each package. If this is not possible, it will give you the next latest -version, and so on. - -Consider the following example (the arrows indicate dependencies): - -

- - -

- -Here we have three packages - 'foo', 'bah' and 'eek', where both foo and bah have dependencies on -eek. For example, package "bah-4" might have a package definition file that looks something like -this (some entries skipped for succinctness): - - name = "bah" - - version = "4" - - requires = [ - "eek-2.6" - ] - -A request for "foo-1.3" is going to result in the resolve ("foo-1.3", "eek-2.7"). A request for -"foo" will give the same result - we are asking for "any version of foo", but rez will prefer the -latest. However, if we request ("foo", "bah"), we are not going to get the latest of both - they -depend on different versions of eek, and that would cause a version conflict. Instead, our resolve -is going to be ("foo-1.2", "bah-4", "eek-2.6"). Rez has given you the latest possible versions of -packages, that do not cause a conflict. - -Sometimes your request is impossible to fulfill. For example, the request ("foo-1.3", "bah-4") is -not possible. In this case, the resolve will fail, and rez will inform you of the conflict. - -## Resolving An Environment - -A user can create a resolved environment using the command line tool *rez-env* (also via the API - -practically everything in rez can be done in python). When you create the environment, the current -environment is not changed - you are placed into a sub-shell instead. Here is an example of using -rez-env, assuming that the package repository is from our earlier (foo, bah, eek) example: - - ]$ rez-env foo bah - - You are now in a rez-configured environment. - - resolved by ajohns@14jun01.methodstudios.com, on Wed Oct 22 12:44:00 2014, - using Rez v2.0.rc1.10 - - requested packages: - foo - bah - - resolved packages: - eek-2.6 /packages/inhouse/eek/2.6 - foo-1.2 /packages/inhouse/foo/1.2 - bah-4 /packages/inhouse/bah/4 - - > ]$ █ - -The output of rez-env shows the original request, along with the matching resolve. It's the resolve -that tells you what actual package versions are present in the newly resolved environment. Notice -the '**>**' character in the prompt - this is a visual cue telling you that you have been placed -into a rez-resolved environment. - -### Putting It All Together - -Let's go through what happens when an environment is resolved, using a new (and slightly more -realistic) example. -Let us assume that the following packages are available: - -* maya-2014.sp2; -* nuke-8.0v3; -* 3 versions of a maya plugin "mplugin"; -* 2 versions of a nuke plugin "nplugin"; -* 3 versions of a common base library "lib". - -The following diagram shows what happens when the command *"rez-env mplugin-1.3.0"* is run: - - -

- - -

- -The digram shows the following operations occurring: - -* Rez takes the user's request, and runs it through the dependency solver. The solver reads packages - from the package repositories in order to complete the solve; -* This results in a list of resolved packages. These are the packages that are used in the - configured environment; -* The commands from each package are concatenated together; -* This total list of commands is then translated into the target shell language (in this example - that is *bash*); -* A sub-shell is created and the translated command code is sourced within this environment, - creating the final configured environment. - -The order of package command execution depends on package dependencies, and the order that packages -were requested in. See -[here](Package-Commands#order-of-command-execution) for more details. diff --git a/wiki/pages/Building-Packages.md b/wiki/pages/Building-Packages.md deleted file mode 100644 index 3bf365659..000000000 --- a/wiki/pages/Building-Packages.md +++ /dev/null @@ -1,255 +0,0 @@ -## Overview - -Rez packages can be built and locally installed using the *rez-build* tool. This -tool performs the following actions: - -* Iterates over a package's [variants](Variants); -* Constructs the build environment; -* Runs the build system within this environment. - -Each build occurs within a *build path* - this is typically either a *build* -subdirectory, or a variant-specific subdirectory under *build*. For example, a -package with two python-based variants might look like this: - - +- package.py - +- CMakeLists.txt (or other build file) - +-build - +-python-2.6 # build dir for python-2.6 variant - +-python-2.7 # build dir for python-2.6 variant - -The current working directory is set to the *build path* during a build. - -## The Build Environment - -The build environment is a rez resolved environment. Its requirement list is -constructed like so: - -* First, the package's [requires](Package-Definition-Guide#requires) list is used; -* Then, the package's [build_requires](Package-Definition-Guide#build_requires) is - appended. This is transitive - the *build_requires* of all other packages in the - environment are also used; -* Then, the package's [private_build_requires](Package-Definition-Guide#private-build_requires) - is appended (unlike *build_requires*, it is not transitive). -* Finally, if the package has variants, the current variant's requirements are - appended. - -A standard list of environment variables is also set (all prefixed with *REZ_BUILD_*) - -you can see the full list [here](Environment-Variables#resolved-build-environment-variables). - -The build system is then invoked within this environment, for each variant. - -## Build Time Dependencies - -Sometimes it is desirable for a package to depend on another package only for the purposes -of building its code, or perhaps generating documentation. Let’s use documentation as an -example - a C++ project may need to builds its docs using doxygen, but once the docs are -generated, doxygen is no longer needed. - -This is achieved by listing build-time dependencies under a -[build_requires](Package-Definition-Guide#build_requires) or -[private_build_requires](Package-Definition-Guide#private-build_requires) -section in the *package.py*. The requirements in *private_build_requires* are only used -from the package being built; requirements from *build_requires* however are transitive - build -requirements from all packages in the build environment are included. - -Some example *private_build_requires* use cases include: - -* Documentation generators such as *doxygen* or *sphinx*; -* Build utilities. For example, you may have a package called *pyqt_cmake_utils*, which - provides cmake macros for converting *ui* files to *py*; -* Statically linked libraries (since the library is linked at build time, the package - is not needed at runtime). - -An example use case of *build_requires* is a header-only (hpp) C++ library - if your own -C++ package includes this library in its own headers, other packages will also need this -library at build time (since they may include your headers, which in turn include the -hpp headers). - -## Package Communication - -Let's say I have two C++ packages, *maya_utils* and the well-known *boost* library. How -does *maya_utils* find *boost*'s header files, or library files? - -The short answer is, that is entirely up to you. Rez is not actually a build system - -it supports various build systems (as the next section describes), and it configures the -build environment, but the details of the build itself are left open for the user. -Having said that, *cmake* has been supported by rez for some time, and rez comes with a -decent amount of utility code to manage cmake builds. - -When a rez environment is configured, each required package's -[commands](Package-Definition-Guide#commands) section configures the environment for the building -package to use. When a build is occurring, a special variable -[building](Package-Commands#building) is set to *True*. Your required packages should use this -variable to communicate build information to the package being built. - -For example, our *boost* package's commands might look like so: - - def commands(): - if building: - # there is a 'FindBoost.cmake' file in this dir.. - env.CMAKE_MODULE_PATH.append("{root}/cmake") - -> [[media/icons/warning.png]] Note that _commands_ is never executed for the package actually -being built - -> if you want to run commands in that case, you can use -[pre_build_commands](Package-Commands#pre-build-commands) instead. - -A (very simple) *FindBoost.cmake* file might look like this: - - set(Boost_INCLUDE_DIRS $ENV{REZ_BOOST_ROOT}/include) - set(Boost_LIBRARY_DIRS $ENV{REZ_BOOST_ROOT}/lib) - set(Boost_LIBRARIES boost-python) - -Then, our *maya_utils* package might have a *CMakeLists.txt* file (cmake's build script) -containing: - - find_package(Boost) - include_directories(${Boost_INCLUDE_DIRS}) - link_directories(${Boost_LIBRARY_DIRS}) - target_link_libraries(maya_utils ${Boost_LIBRARIES}) - -As it happens, the [find_package](https://cmake.org/cmake/help/v3.0/command/find_package.html) -cmake macro searches the paths listed in the *CMAKE_MODULE_PATH* environment variable, -and looks for a file called *FindXXX.cmake*, where *XXX* is the name of the package (in this -case, *Boost*), which it then includes. - -## The Build System - -Rez supports multiple build systems, and new ones can be added as plugins. When a -build is invoked, the build system is detected automatically. For example, if a -*CMakeLists.txt* file is found in the package's root directory, the *cmake* build -system is used. - -### Argument Passing - -There are two ways to pass arguments to the build system. - -First, some build system plugins add extra options to the *rez-build* command directly. -For example, if you are in a CMake-based package, and you run *rez-build -h*, you will -see cmake-specific options listed, such as *--build-target*. - -Second, you can pass arguments directly to the build system - either using the -*rez-build* *--build-args* option; or listing the build system arguments after *--*. - -For example, here we explicitly define a variable in a cmake build: - - ]$ rez-build -- -DMYVAR=YES - -### Custom Build Commands - -As well as detecting the build system from build files, a package can explicitly -specify its own build commands, using the -[build_command](Package-Definition-Guide#build_command) package attribute. If present, -this takes precedence over other detected build systems. - -For example, consider the following *package.py* snippet: - - name = "nuke_utils" - - version = "1.2.3" - - build_command = "bash {root}/build.sh {install}" - -When *rez-build* is run on this package, the given *build.sh* script will be executed -with *bash*. The *{root}* string expands to the root path of the package (the same -directory containing *package.py*). The *{install}* string expands to "*install*" if -an install is occurring, or the empty string otherwise. This is useful for passing the -install target directly to the command (for example, when using *make*) rather than -relying on a build script checking the *REZ_BUILD_INSTALL* environment variable. - -> [[media/icons/warning.png]] The current working directory during a build is set -> to the *build path*, **not** to the package root directory. For this reason, you -> will typically use the *{root}* string to refer to a build script in the package's -> root directory. - -#### Passing Arguments - -You can add arguments for your build script to the *rez-build* command directly, by -providing a *parse_build_args.py* source file in the package root. Here is an example: - - # in parse_build_args.py - parser.add_argument("--foo", action="store_true", help="do some foo") - -Now if you run *rez-build -h* on this package, you will see the option listed: - - $ rez-build -h - usage: rez build [-h] [-c] [-i] [-p PATH] [--fail-graph] [-s] [--view-pre] - [--process {remote,local}] [--foo] - [--variants INDEX [INDEX ...]] [--ba ARGS] [--cba ARGS] [-v] - - Build a package from source. - - optional arguments: - ... - --foo do some foo - -The added arguments are stored into environment variables so that your build script -can access them. They are prefixed with `__PARSE_ARG_`; in our example above, the -variable `__PARSE_ARG_FOO` will be set. Booleans will be set to 0/1, and lists are -space separated, with quotes where necessary. - -#### Make Example - -Following is a very simple C++ example, showing how to use a custom build command to -build and install via *make*: - - # in package.py - build_command = "make -f {root}/Makefile {install}" - - - # in Makefile - hai: ${REZ_BUILD_SOURCE_PATH}/lib/main.cpp - g++ -o hai ${REZ_BUILD_SOURCE_PATH}/lib/main.cpp - - .PHONY: install - install: hai - mkdir -p ${REZ_BUILD_INSTALL_PATH}/bin - cp $< ${REZ_BUILD_INSTALL_PATH}/bin/hai - -## Local Package Installs - -After you've made some code changes, you presumably want to test them. You do this -by *locally installing* the package, then resolving an environment with *rez-env* -to test the package in. The cycle goes like this: - -* Make code changes; -* Run *rez-build --install* to install as a local package; -* Run *rez-env mypackage* in a separate shell. This will pick up your local package, - and your package requirements; -* Test the package. - -A local install builds and installs the package to the *local package repository*, -which is typically the directory *~/packages* (see [here](Configuring-Rez#local_packages_path)). -This directory is listed at the start of the -[package search path](Basic-Concepts#package-search-path), so when you resolve an -environment to test with, the locally installed package will be picked up first. Your -package will typically be installed to *~/packages/packagename/version*, for example -*~/packages/maya_utils/1.0.5*. If you have variants, they will be installed into subdirectories -within this install path (see [here](Variants#disk-structure) for more details). - -> [[media/icons/info.png]] You don't need to run *rez-env* after every install. If your -> package's requirements haven't changed, you can keep using the existing test environment. - -You can make sure you've picked up your local package by checking the output of the -*rez-env* call: - - ]$ rez-env sequence - - You are now in a rez-configured environment. - - resolved by ajohns@turtle, on Thu Mar 09 11:41:06 2017, using Rez v2.7.0 - - requested packages: - sequence - ~platform==linux (implicit) - ~arch==x86_64 (implicit) - ~os==Ubuntu-16.04 (implicit) - - resolved packages: - arch-x86_64 /sw/packages/arch/x86_64 - os-Ubuntu-16.04 /sw/packages/os/Ubuntu-16.04 - platform-linux /sw/packages/platform/linux - python-2.7.12 /sw/packages/python/2.7.12 - sequence-2.1.2 /home/ajohns/packages/sequence/2.1.2 (local) - -Note here that the *sequence* package is a local install, denoted by the *(local)* label. diff --git a/wiki/pages/Bundles.md b/wiki/pages/Bundles.md deleted file mode 100644 index 83f589c92..000000000 --- a/wiki/pages/Bundles.md +++ /dev/null @@ -1,81 +0,0 @@ -## Overview - -A "context bundle" is a directory containing a context (an rxt file), and a -package repository. All packages in the context are stored in the repository, -making the bundle relocatable and standalone. You can copy a bundle onto a -server for example, or into a container, and there are no external references -to shared package repositories. This is in contrast to a typical context, which -contains absolute references to one or more package repositories that are -typically on shared disk storage. - -To create a bundle via command line: - -``` -]$ rez-env foo -o foo.rxt -]$ rez-bundle foo.rxt ./mybundle - -]$ # example of running a command from the bundled context -]$ rez-env -i ./mybundle/context.rxt -- foo-tool -``` - -To create a bundle via API: - -``` ->>> from rez.bundle_context import bundle_context ->>> from rez.resolved_context import ResolvedContext ->>> ->>> c = ResolvedContext(["python-3+", "foo-1.2+<2"]) ->>> bundle_context(c, "./mybundle") -``` - - -## Structure - -A bundle directory looks like this: - -``` -.../mybundle/ - ./context.rxt - ./packages/ - -``` - -Package references in the rxt file are relative (unlike in a standard context, -where they're absolute), and this makes the bundle relocatable. - - -## Patching Libraries - -Depending on how compiled libraries and executables within a rez package were -built, it's possible that the dynamic linker will attempt to resolve them to -libraries found outside of the bundle. For example, this is possible in linux -if an elf contains an absolute searchpath in its rpath/runpath header to a -library in another package. - -Rez bundling performs a library patching step that applies various fixes to -solve this issue (use `--no-lib-patch` if you want to skip this step). This step -is platform-specific and is covered in the following sections. Note that in all -cases, references to libraries outside of the bundle will remain intact, if there -is no equivalent path found within the bundle (for example, if the reference is -to a system library not provided by a rez package). - -### Linux - -On linux, rpath/runpath headers are altered if paths are found that map to a -subdirectory within another package in the bundle. To illustrate what happens, -consider the following example, where packages from `/sw/packages` have been -bundled into the local directory `./mybundle`: - -``` -]$ # a lib in an original non-bundled package -]$ patchelf --print-rpath /sw/packages/foo/1.0.0/bin/foo -/sw/packages/bah/2.1.1/lib -]$ -]$ # the same lib in our bundle. We assume that package 'bah' is in the bundle -]$ # also, since foo links to one of its libs -]$ patchelf --print-rpath ./mybundle/packages/foo/1.0.0/bin/foo -$ORIGIN/../../../bah/2.1.1/lib -``` - -Remapped rpaths make use of the special `$ORIGIN` variable, which refers to -the directory containing the current file. diff --git a/wiki/pages/Command-Line-Tools.md b/wiki/pages/Command-Line-Tools.md deleted file mode 100644 index 434525e91..000000000 --- a/wiki/pages/Command-Line-Tools.md +++ /dev/null @@ -1 +0,0 @@ -__GENERATED_MD__ diff --git a/wiki/pages/Configuring-Rez.md b/wiki/pages/Configuring-Rez.md deleted file mode 100644 index baaa2a719..000000000 --- a/wiki/pages/Configuring-Rez.md +++ /dev/null @@ -1,118 +0,0 @@ -## Overview - -Rez has a good number of configurable settings. The default settings, and -documentation for every setting, can be found -[here](https://github.com/__GITHUB_REPO__/blob/__GITHUB_BRANCH__/src/rez/rezconfig.py). - -Settings are determined in the following way: - -- The setting is first read from the file *rezconfig.py* in the rez installation; -- The setting is then overridden if it is present in another settings file pointed at by the - *REZ_CONFIG_FILE* environment variable. This can also be a path-like variable, to read from - multiple configuration files; -- The setting is further overriden if it is present in *$HOME/.rezconfig*; -- The setting is overridden again if the environment variable *REZ_XXX* is present, where *XXX* is - the uppercase version of the setting key. For example, "image_viewer" will be overriden by - *REZ_IMAGE_VIEWER*. -- This is a special case applied only during a package build or release. In this case, if the - package definition file contains a "config" section, settings in this section will override all - others. See [here](#package-overrides). - -It is fairly typical to provide your site-specific rez settings in a file that the environment -variable *REZ_CONFIG_FILE* is then set to for all your users. Note that you do not need to provide -a copy of all settings in this file - just provide those that are changed from the defaults. - -## Settings Merge Rules - -When multiple configuration sources are present, the settings are merged together - -one config file does not replace the previous one, it overrides it. By default, the -following rules apply: - -* Dicts are recursively merged together; -* Non-dicts override the previous value. - -However, it is also possible to append and/or prepend list-based settings. For example, the -following config entry will append to the `release_hooks` setting value defined by the -previous configuration sources (you can also supply a *prepend* argument): - - release_hooks = ModifyList(append=["custom_release_notify"]) - -## Package Overrides - -Packages themselves can override configuration settings. To show how this is useful, -consider the following example: - - # in package.py - with scope("config") as c: - c.release_packages_path = "/svr/packages/internal" - -Here a package is overriding the default release path - perhaps you're releasing -internally- and externally-developed packages to different locations, for example. - -These config overrides are only applicable during building and releasing of the package. -As such, even though any setting can be overridden, it's only useful to do so for -those that have any effect during the build/install process. These include: - -* Settings that determine where packages are found, such as *packages_path*, - *local_packages_path* and *release_packages_path*; -* Settings in the *build_system*, *release_hook* and *release_vcs* plugin types; -* *package_definition_python_path*; -* *package_filter*. - -## String Expansions - -The following string expansions occur on all configuration settings: - -* Any environment variable reference, in the form *${HOME}*; -* Any property of the *system* object, eg *{system.platform}*. - -The *system* object has the following attributes: - -* platform: The platform, eg 'linux'; -* arch: The architecture, eg 'x86_64'; -* os: The operating system, eg 'Ubuntu-12.04'; -* user: The current user's username; -* home: Current user's home directory; -* fqdn: Fully qualified domain name, eg 'somesvr.somestudio.com'; -* hostname: Host name, eg 'somesvr'; -* domain: Domain name, eg 'somestudio.com'; -* rez_version: Version of rez, eg '2.0.1'. - -## Delay Load - -It is possible to store a config setting in a separate file, which will be loaded -only when that setting is referenced. This can be useful if you have a large value -(such as a dict) that you don't want to pollute the main config with. YAML and -JSON formats are supported: - - # in rezconfig - default_relocatable_per_package = DelayLoad('/svr/configs/rez_relocs.yaml') - -## Commandline Tool - -You can use the *rez-config* command line tool to see what the current configured settings are. -Called with no arguments, it prints all settings; if you specify an argument, it prints out just -that setting: - - ]$ rez-config packages_path - - /home/sclaus/packages - - /home/sclaus/.rez/packages/int - - /home/sclaus/.rez/packages/ext - -Here is an example showing how to override settings using your own configuration file: - - ]$ echo 'packages_path = ["~/packages", "/packages"]' > myrezconfig.py - ]$ export REZ_CONFIG_FILE=${PWD}/myrezconfig.py - ]$ rez-config packages_path - - /home/sclaus/packages - - /packages - -## Configuration Settings - -Following is an alphabetical list of rez settings. - -> [[media/icons/info.png]] Note that this list has been generated automatically -> from the [rez-config.py](https://github.com/__GITHUB_REPO__/blob/main/src/rez/rezconfig.py) -> file in the rez source, so you can also refer to that file for the same information. - -__REZCONFIG_MD__ diff --git a/wiki/pages/Contexts.md b/wiki/pages/Contexts.md deleted file mode 100644 index b960353a6..000000000 --- a/wiki/pages/Contexts.md +++ /dev/null @@ -1,97 +0,0 @@ -## Overview - -When you use *rez-env* to create a resolved environment, you are actually -creating something called a *context*. A context is a store of information -including: - -* The initial [package request](Basic-Concepts#package-requests) list; -* The *resolve* - the list of variants that were chosen; -* A graph which shows the resolve visually. - -The context does not store copies of the packages it resolved to; rather, it -stores a kind of handle for each, which gives enough information to know where -to fetch the full package definition and contents from. - -Contexts themselves are quite small, and are stored in JSON format in a file -with the extension *rxt*. When you use *rez-env*, it actually creates a temporary -context file on disk, which is removed when the shell is exited: - - ]$ rez-env foo bah - - You are now in a rez-configured environment. - - resolved by ajohns@14jun01.methodstudios.com, on Wed Oct 22 12:44:00 2014, - using Rez v2.0.rc1.10 - - requested packages: - foo - bah - - resolved packages: - eek-2.6 /packages/inhouse/eek/2.6 - foo-1.2 /packages/inhouse/foo/1.2 - bah-4 /packages/inhouse/bah/4 - - > ]$ echo $REZ_RXT_FILE - /tmp/rez_context_0tMS4U/context.rxt - -## Baking Resolves - -You can use the *rez-env* flag *--output* to write a resolved context directly -to file, rather than invoking a subshell: - - ]$ rez-env foo bah --output test.rxt - -Later, you can read the context back again, to reconstruct the same environment: - - ]$ rez-env --input test.rxt - - You are now in a rez-configured environment. - - resolved by ajohns@14jun01.methodstudios.com, on Wed Oct 22 12:44:00 2014, - using Rez v2.0.rc1.10 - - requested packages: - foo - bah - - resolved packages: - eek-2.6 /packages/inhouse/eek/2.6 - foo-1.2 /packages/inhouse/foo/1.2 - bah-4 /packages/inhouse/bah/4 - - > ]$ █ - -Contexts do not store a copy of the environment that is configured (that is, the -environment variables exported, for example). A context just stores the resolved -list of packages that need to be applied in order to configure the environment. -When you load a context via *rez-env --input*, each of the packages' *commands* -sections are interpreted once more. - -You can think of package *commands* like fragments of a wrapper script which -configures an environment. By creating a context, you are creating a list of -script fragments which, when run in serial, produce the target environment. So, -if your package added a path to *$PATH* which included a reference to *$USER* -for example, this would work correctly even if Joe created the rxt file, and -Jill read it - the commands are reinterpreted when Jill loads the context. - -## The rez-context Tool - -The *rez-context* tool inspects context files. When you're within a resolved -subshell, *rez-context* inspects the current context, unless one is specified -explicitly. For example, we can inspect the context created in the previous -example, without actually being within it: - - ]$ rez-context test.rxt - - resolved by ajohns@14jun01.methodstudios.com, on Wed Oct 22 12:44:00 2014, - using Rez v2.0.rc1.10 - - requested packages: - foo - bah - - resolved packages: - eek-2.6 /packages/inhouse/eek/2.6 - foo-1.2 /packages/inhouse/foo/1.2 - bah-4 /packages/inhouse/bah/4 diff --git a/wiki/pages/Credits.md b/wiki/pages/Credits.md deleted file mode 100644 index 90494b0cf..000000000 --- a/wiki/pages/Credits.md +++ /dev/null @@ -1,15 +0,0 @@ -## Contributors - -

-__CONTRIBUTORS_MD__ -

- -## Other - -

-[[media/icons/info.png]]
-[[media/icons/warning.png]]
-[[media/icons/under_construction.png]]
-[[media/icons/new.png]] pixel buddha
-Icons made by Freepik. -

diff --git a/wiki/pages/Environment-Variables.md b/wiki/pages/Environment-Variables.md deleted file mode 100644 index 59327f3c4..000000000 --- a/wiki/pages/Environment-Variables.md +++ /dev/null @@ -1,102 +0,0 @@ -This chapter lists the environment variables that rez generates in certain -circumstances, as well as environment variables that you can set which affect -the operation of rez. - -## Context Environment Variables - -These are variables that rez generates within a resolved environment (a "context"). - -* **REZ_RXT_FILE** - Filepath of the current context (an rxt file). -* **REZ_USED** - Path to rez installation that was used to resolve this environment. -* **REZ_USED_IMPLICIT_PACKAGES** - The list of implicit packages used in the resolve. -* **REZ_USED_PACKAGES_PATH** - The package searchpath used for this resolve. -* **REZ_USED_RESOLVE** - The list of resolved packages, eg *"platform-linux utils-1.2.3"*. -* **REZ_USED_EPH_RESOLVE** - The list of resolved ephemerals, eg *".foo.cli-1 .debugging-0"*. -* **REZ_USED_LOCAL_RESOLVE** - The list of resolved local packages, eg *"utils-1.2.3 maya_utils-1.3+"*. - Packages listed here will always be a subset of the packages in *REZ_USED_RESOLVE*. -* **REZ_USED_REQUEST** - The environment request string, eg *"maya-2017 maya_utils-1.3+"*. - Does not include implicit packages. -* **REZ_USED_REQUESTED_TIMESTAMP** - The epoch time of this resolved environment, - explicitly set by the user with (for example) the rez-env '\-\-time' flag; zero otherwise. -* **REZ_USED_TIMESTAMP** - The epoch time when this environment was resolved; OR, - the value of *REZ_USED_REQUESTED_TIMESTAMP*, if non-zero. -* **REZ_USED_VERSION** - The version of rez used to resolve this environment. -* **REZ_SHELL_INIT_TIMESTAMP** - The epoch time when the current shell was instantiated. -* **REZ_SHELL_INTERACTIVE** - Will be '1' if the shell is interactive, and '0' otherwise - (ie, when a command is specified, like `rez-env foo -- mycommand`). - -Specifically, per-package, the following variables are generated. Note that for a given -package name, *"(PKG)"* in the variables below is the uppercased package name, with any -dots replaced with underscore: - -* **REZ_(PKG)_BASE** - The base directory of the package installation, eg - *"/packages/utils/1.0.0"*. -* **REZ_(PKG)_ROOT** - The root directory of the package installation (actually, - the variant), eg *"/packages/utils/1.0.0/python-2.7"*. -* **REZ_(PKG)_VERSION** - The version of the package. -* **REZ_(PKG)_MAJOR_VERSION** - The major version of the package, or ''. -* **REZ_(PKG)_MINOR_VERSION** - The minor version of the package, or ''. -* **REZ_(PKG)_PATCH_VERSION** - The patch version of the package, or ''. - -For every _ephemeral_ package request, the following variables are generated. Note -that for a given ephemeral package name, *"(PKG)"* in the variables below is the -uppercased package name, with dots replaced by underscore, and **the leading dot -removed**: - -* **REZ_EPH_(PKG)_REQUEST** - The resolved ephemeral package request. - -## Build Environment Variables - -These are variables that rez generates within a build environment, in addition -to those listed [here](#context-environment-variables). - -* **REZ_BUILD_ENV** - Always present in a build, has value 1. -* **REZ_BUILD_INSTALL** - Has a value of 1 if an installation is taking place - (either a *rez-build -i* or *rez-release*), otherwise 0. -* **REZ_BUILD_INSTALL_PATH** - Installation path, if an install is taking place. -* **REZ_BUILD_PATH** - Path where build output goes. -* **REZ_BUILD_PROJECT_DESCRIPTION** - Equal to the *description* attribute of the - package being built. -* **REZ_BUILD_PROJECT_FILE** - The filepath of the package being built (typically - a *package.py* file). -* **REZ_BUILD_PROJECT_NAME** - Name of the package being built. -* **REZ_BUILD_PROJECT_VERSION** - Version of the package being built. -* **REZ_BUILD_REQUIRES** - Space-separated list of requirements for the build - - comes from the current package's *requires*, *build_requires* and - *private_build_requires* attributes, including the current variant's requirements. -* **REZ_BUILD_REQUIRES_UNVERSIONED** - Equivalent but unversioned list to - *REZ_BUILD_REQUIRES*. -* **REZ_BUILD_SOURCE_PATH** - Path containing the package.py file. -* **REZ_BUILD_THREAD_COUNT** - Number of threads being used for the build. -* **REZ_BUILD_TYPE** - One of *local* or *central*. Value is *central* if a - release is occurring. -* **REZ_BUILD_VARIANT_INDEX** - Zero-based index of the variant currently being - built. For non-varianted packages, this is "0". -* **REZ_BUILD_VARIANT_REQUIRES** - Space-separated list of runtime requirements - of the current variant. This does not include the common requirements as found - in *REZ_BUILD_REQUIRES*. For non-varianted builds, this is an empty string. -* **REZ_BUILD_VARIANT_SUBPATH** - Subdirectory containing the current variant. - For non-varianted builds, this is an empty string. - -## Runtime Environment Variables - -These are environment variables that the user can set, which affect the -operation of rez. - -* **REZ_(CONFIG_ENTRY)** - For any given rez config entry (see *rezconfig.py*), - you can override the setting with an environment variable, for convenience. Here, - *(CONFIG_ENTRY)* is the uppercased equivalent of the setting name. For example, - a setting commonly overriden this way is *packages_path*, whos equivalent - variable is *REZ_PACKAGES_PATH*. -* **REZ_(CONFIG_ENTRY)_JSON** - Same as the previous env-var, except that the format - is a JSON string. This means that some more complex settings can be overridden, - that aren't supported in the non-JSON case (*package_filter* is an example). -* **REZ_DISABLE_HOME_CONFIG** - If 1/t/true, The default `~/.rezconfig.py` config - file is skipped. -* **EDITOR** - On Linux and OSX systems, this will set the default editor to use - if and when rez requires one (an example is on release if the *prompt_release_message* - config setting is true). -* **REZ_KEEP_TMPDIRS** - If set to a non-empty string, this prevents rez from - cleaning up any temporary directories. This is for debugging purposes. -* **REZ_SIGUSR1_ACTION** - If you set this to *print_stack*, rez will prints its - current stacktrace to stdout if sent a USR1 signal. This is for debugging purposes. diff --git a/wiki/pages/Ephemeral-Packages.md b/wiki/pages/Ephemeral-Packages.md deleted file mode 100644 index d794fd57d..000000000 --- a/wiki/pages/Ephemeral-Packages.md +++ /dev/null @@ -1,156 +0,0 @@ -## Overview - -_Introduced in version 2.71.0_ - -Ephemeral packages (or simply 'ephemerals') are requests for packages that do not -exist. Ephemeral package names always begin with a dot (`.`). Like all package -requests, ephemerals can be requested as part of packages' requires or variants -lists, or directly by the user (via `rez-env` for eg). - -Example: - - ]$ rez-env .foo-1 - You are now in a rez-configured environment. - - resolved by ajohns@turtle, on Tue Dec 22 08:17:00 2020, using Rez v2.70.0 - - requested packages: - .foo-1 (ephemeral) - ~platform==linux (implicit) - ~arch==x86_64 (implicit) - ~os==Ubuntu-16.04 (implicit) - - resolved packages: - .foo-1 (ephemeral) - -Ephemerals will act like real packages during a resolve - ie, their request ranges -will intersect, and conflicts can occur - but they never actually correlate to a -real package, nor do they perform any configuration on the runtime (not directly -in any case). - -Example showing range intersection: - - ]$ rez-env .foo-1 '.foo-1.5+' - - You are now in a rez-configured environment. - - resolved by ajohns@turtle, on Tue Dec 22 08:21:04 2020, using Rez v2.70.0 - - requested packages: - .foo-1 (ephemeral) - .foo-1.5+ (ephemeral) - ~platform==linux (implicit) - ~arch==x86_64 (implicit) - ~os==Ubuntu-16.04 (implicit) - - resolved packages: - .foo-1.5+<1_ (ephemeral) - -Example of conflicting request: - - ]$ rez-env .foo-1 .foo-2 - The context failed to resolve: - The following package conflicts occurred: (.foo-1 <--!--> .foo-2) - -## Environment Variables - -Ephemerals do not affect the runtime in the way that packages can (via their -`commands` section), however some environment variables are set: - -* `REZ_USED_EPH_RESOLVE` lists all resolved ephemeral requests; -* `REZ_EPH_(PKG)_REQUEST` is set for every resolved ephemeral. Here, `(PKG)` is - the ephemeral name, in uppercase, with dots replaced by underscores and with - **the leading dot removed**. - -The following example illustrates: - - ]$ rez-env python .foo-1 .bah-2 - ... - ]$ echo $REZ_EPH_FOO_REQUEST - 1 - ]$ echo $REZ_USED_EPH_RESOLVE - .foo-1 .bah-2 - -## Introspection - -In order for a package to inspect the ephemerals that are present in a runtime, -there is an [ephemerals](Package-Commands#ephemerals) object provided, similar -to the [resolve](Package-Commands#resolve) object. You would typically use the -[intersects](Package-Commands#intersects) function to inspect it, like so: - - # in package.py - def commands() - if intersects(ephemerals.get_range('enable_tracking', '0'), '1'): - env.TRACKING_ENABLED = 1 - -In this example, the given package would set the `TRACKING_ENABLED` environment -variable if an ephemeral such as `.enable_tracking-1` (or `.enable_tracking-1.2+` -etc) is present in the resolve. Note that the leading `.` is implied and not -included when querying the `ephemerals` object. - -> [[media/icons/warning.png]] Since `ephemerals` is a dict-like object, so it has -> a `get` function which will return a full request string if key exists. Hence, -> the default value should also be a full request string, not just a version range -> string like `'0'` in `get_range`. Or `intersects` may not work as expect. - -## Ephemeral Use Cases - -Why would you want to request packages that don't exist? There are two main use -cases. - -### Passing Information to Packages - -Ephemerals can be used as a kind of 'package option', or a way to pass information -to packages in a resolve. For example, consider the following package definition: - - name = 'bah' - - def commands(): - if intersects(ephemerals.get_range('bah.cli', '1'), '1'): - env.PATH.append('{root}/bin') - -This package will disable its command line tools if an ephemeral like `.bah.cli-0` -is present in the runtime. - -> [[media/icons/info.png]] Ephemerals are standard package requests and so can -> have any range, such as `1.2.3`, `2.5+` and so on. However, they're often used -> as boolean package options, as in the example above. In this case, it is -> recommended to use the conventional ranges `1` and `0` to designate true and -> false. - -Since ephemerals can be pretty much anything, you might also decide to use them -as a global package option. Here's another take on our example, but in this case -we introduce a `.cli` ephemeral that acts as a global whitelist: - - name = 'bah' - - def commands(): - if intersects(ephemerals.get_range('cli', ''), 'bah'): - env.PATH.append('{root}/bin') - -Here, all packages' cli will be enabled if `.cli` is not specified, but if it is -specified then it acts as a whitelist: - - # turn on cli for foo and bah only - ]$ rez-env foo-1 bah==2.3.1 eek-2.4 '.cli-foo|bah' - -### Abstract Package Representation - -Sometimes it makes sense for a package to require some form of abstract object or -capability, rather than an actual package. For example, perhaps your package (or -one of its variants) requires a GPU to be present on the host machine. To support -this, you might have something setup that includes a `.gpu-1` ephemeral in the -[implicits](Basic-Concepts#implicit-packages) list on all GPU-enabled hosts. -Then, your package could look like this: - - name = 'pixxelator' - - variants = [ - ['.gpu-0'], # renders via CPU - ['.gpu-1'] # renders via GPU - ] - -> [[media/icons/warning.png]] Be aware that on hosts that do _not_ have a gpu -> implicit, either variant could be selected. You would want to either guarantee -> that every host has the gpu implicit set to 0 or 1, or that the user always -> explicitly specifies `.gpu-0` or `.gpu-1` in their request. diff --git a/wiki/pages/FAQ.md b/wiki/pages/FAQ.md deleted file mode 100644 index d6360a1c1..000000000 --- a/wiki/pages/FAQ.md +++ /dev/null @@ -1 +0,0 @@ -FAQ diff --git a/wiki/pages/Getting-Started.md b/wiki/pages/Getting-Started.md deleted file mode 100644 index 8ac4ae7d0..000000000 --- a/wiki/pages/Getting-Started.md +++ /dev/null @@ -1,158 +0,0 @@ -## Essential Packages - -After installation, you need to create some essential Rez packages. The *rez-bind* -tool creates Rez packages that reference software already installed on your system. -Use the *--quickstart* argument to bind a set of standard packages (note that you -may require administrative privileges for some of them): - - ]$ rez-bind --quickstart - Binding platform into /home/ajohns/packages... - Binding arch into /home/ajohns/packages... - Binding os into /home/ajohns/packages... - Binding python into /home/ajohns/packages... - Binding rez into /home/ajohns/packages... - Binding rezgui into /home/ajohns/packages... - Binding setuptools into /home/ajohns/packages... - Binding pip into /home/ajohns/packages... - - Successfully converted the following software found on the current system into Rez packages: - - PACKAGE URI - ------- --- - arch /home/ajohns/packages/arch/x86_64/package.py - os /home/ajohns/packages/os/osx-10.11.5/package.py - pip /home/ajohns/packages/pip/8.0.2/package.py - platform /home/ajohns/packages/platform/osx/package.py - python /home/ajohns/packages/python/2.7.11/package.py - rez /home/ajohns/packages/rez/2.0.rc1.44/package.py - rezgui /home/ajohns/packages/rezgui/2.0.rc1.44/package.py - setuptools /home/ajohns/packages/setuptools/19.4/package.py - -Now you should be able to create an environment containing Python. Try this: - - ]$ rez-env python -- which python - /home/ajohns/packages/python-2.7.8/platform-linux/arch-x86_64/os-Ubuntu-12.04/bin/python - - -## Building Your First Package - -Before building your first rez package, ensure that: - -* The directory *$HOME/packages* exists and is writable; -* The [cmake](https://cmake.org/) tool is available. - -The *rez-build* tool is used to build packages and install them locally (typically -to *$HOME/packages*). Once you've done that, you can use them via *rez-env*, just -like any other package: - - ]$ cd example_packages/hello_world - ]$ rez-build --install - - -------------------------------------------------------------------------------- - Building hello_world-1.0.0... - -------------------------------------------------------------------------------- - Resolving build environment: python - resolved by ajohns@workstation.local, on Sun Jul 31 14:39:33 2016, using Rez v2.0.rc1.44 - - requested packages: - python - ~platform==osx (implicit) - ~arch==x86_64 (implicit) - ~os==osx-10.11.5 (implicit) - - resolved packages: - arch-x86_64 /home/ajohns/packages/arch/x86_64 (local) - os-osx-10.11.5 /home/ajohns/packages/os/osx-10.11.5 (local) - platform-osx /home/ajohns/packages/platform/osx (local) - python-2.7.11 /home/ajohns/packages/python/2.7.11/platform-osx/arch-x86_64/os-osx-10.11.5 (local) - - Invoking cmake build system... - Executing: /usr/local/bin/cmake -d /home/ajohns/workspace/rez/example_packages/hello_world -Wno-dev -DCMAKE_ECLIPSE_GENERATE_SOURCE_PROJECT=TRUE -D_ECLIPSE_VERSION=4.3 --no-warn-unused-cli -DCMAKE_INSTALL_PREFIX=/home/ajohns/packages/hello_world/1.0.0 -DCMAKE_MODULE_PATH=${CMAKE_MODULE_PATH} -DCMAKE_BUILD_TYPE=Release -DREZ_BUILD_TYPE=local -DREZ_BUILD_INSTALL=1 -G Unix Makefiles - Not searching for unused variables given on the command line. - -- Could NOT find PkgConfig (missing: PKG_CONFIG_EXECUTABLE) - -- Configuring done - -- Generating done - -- Build files have been written to: /home/ajohns/workspace/rez/example_packages/hello_world/build - - Executing: make -j4 - [100%] Built target py - - Executing: make -j4 install - [100%] Built target py - Install the project... - -- Install configuration: "Release" - -- Installing: /home/ajohns/packages/hello_world/1.0.0/./python/hello_world.py - -- Installing: /home/ajohns/packages/hello_world/1.0.0/./python/hello_world.pyc - -- Installing: /home/ajohns/packages/hello_world/1.0.0/./bin/hello - - All 1 build(s) were successful. - -You have just built your first package, and installed it to the *local package path*, which defaults -to (and is usually kept as) *$HOME/packages*. - - -## Testing Your Package - -You can use the *rez-env* tool to request a configured environment containing your package: - - ]$ rez-env hello_world - - You are now in a rez-configured environment. - - resolved by ajohns@workstation.local, on Sun Jul 31 14:43:54 2016, using Rez v2.0.rc1.44 - - requested packages: - hello_world - ~platform==osx (implicit) - ~arch==x86_64 (implicit) - ~os==osx-10.11.5 (implicit) - - resolved packages: - arch-x86_64 /home/ajohns/packages/arch/x86_64 (local) - hello_world-1.0.0 /home/ajohns/packages/hello_world/1.0.0 (local) - os-osx-10.11.5 /home/ajohns/packages/os/osx-10.11.5 (local) - platform-osx /home/ajohns/packages/platform/osx (local) - python-2.7.11 /home/ajohns/packages/python/2.7.11/platform-osx/arch-x86_64/os-osx-10.11.5 (local) - - > ]$ █ - -Now you are within the configured environment. The caret (>) prefixed to your prompt is a visual cue -telling you that you're within a rez-configured subshell. Rez does not update the currect environment, -instead it configures a subshell and puts you within it. - -Now you can run the *hello* tool in our *hello_world* package: - - > ]$ hello - Hello world! - -If you're within a rez shell, and you forget what packages are currently available or want to see the -list again, you can use the *rez-context* tool. It prints the same information you see when you -initially created the environment: - - > ]$ rez-context - resolved by ajohns@workstation.local, on Sun Jul 31 14:43:54 2016, using Rez v2.0.rc1.44 - - requested packages: - hello_world - ~platform==osx (implicit) - ~arch==x86_64 (implicit) - ~os==osx-10.11.5 (implicit) - - resolved packages: - arch-x86_64 /home/ajohns/packages/arch/x86_64 (local) - hello_world-1.0.0 /home/ajohns/packages/hello_world/1.0.0 (local) - os-osx-10.11.5 /home/ajohns/packages/os/osx-10.11.5 (local) - platform-osx /home/ajohns/packages/platform/osx (local) - python-2.7.11 /home/ajohns/packages/python/2.7.11/platform-osx/arch-x86_64/os-osx-10.11.5 (local) - -To exit the configured environment, simply exist the shell using the *exit* command: - - > ]$ exit - ]$ █ - -You can also create a configured environment and run a command inside of it, with a single command. -When you use this form, the shell is immediately exited after the command runs: - - ]$ rez-env hello_world -- hello - Hello world! - ]$ █ diff --git a/wiki/pages/Glossary.md b/wiki/pages/Glossary.md deleted file mode 100644 index c1cf05651..000000000 --- a/wiki/pages/Glossary.md +++ /dev/null @@ -1,50 +0,0 @@ - - -## context -An object containing everything about an environment resolve. Contexts can be stored in - .rxt files, and used to reconstruct the same environment at a later date. - -## local package path -Path to package repository where locally-developed packages are installed to. This path typically -appears at the front of the packages search path. It is typically `~/packages`. - -## package -A versioned piece of software - the things that rez manages. - -## package commands -A block of python code in the package definition file that determines how the package updates the -environment it is used in. - -## package definition file -A file, such as 'package.py', that defines everything we want to know about a package, including -its dependencies. Every package has one. - -## package repository -A place where packages are stored - usually a directory on disk. - -## package request -A string describing a request for a package, such as "python-2.6+", "foo==1.0.0". - -## package search path -Search path that rez uses to find packages. - -## request -A list of package requests, such as ("python-2.6+", "foo-1", "bah==2.3.3"). - -## resolve -A list of packages resulting from resolving a request with the dependency solver. - -## rez-config -A command line tool that shows the current rez configuration settings. - -## rez-env -A command line tool that places the user into a newly resolved environment. - -## version -A version number, such as "1", "2.0", "1.5.3alpha". - -## version conflict -Two requests for the same package that do not overlap. For example, ("python-2.5", "python-2.7"). - -## version range -A string describing a range of possible versions, such as "4+", "<2.1", "3.0", "1.1+<2", "==4.2.2". diff --git a/wiki/pages/Home.md b/wiki/pages/Home.md deleted file mode 100644 index 25517e4c8..000000000 --- a/wiki/pages/Home.md +++ /dev/null @@ -1,105 +0,0 @@ -## What Is Rez? - -Rez is a cross-platform package manager with a difference. Using Rez you can create -standalone environments configured for a given set of packages. However, unlike many -other package managers, packages are not installed into these standalone environments. -Instead, all package versions are installed into a central repository, and standalone -environments reference these existing packages. This means that configured environments -are lightweight, and very fast to create, often taking just a few seconds to configure -despite containing hundreds of packages. - -

- - -
Typical package managers install packages into an environment -

- -
-

- - -
Rez installs packages once, and configures environments dynamically -

- -
-Rez takes a list of package requests, and constructs the target environment, resolving -all the necessary package dependencies. Any type of software package is supported - -compiled, python, applications and libraries. - - -## The Basics - -Packages are stored in repositories on disk. Each package has a single concise -definition file (*package.py*) that defines its dependencies, its commands (how it -configures the environment containing it), and other metadata. For example, the -following is the package definition file for the popular *requests* python module: - - name = "requests" - - version = "2.8.1" - - authors = ["Kenneth Reitz"] - - requires = [ - "python-2.7+" - ] - - def commands(): - env.PYTHONPATH.append("{root}/python") - -This package requires python-2.7 or greater. When used, the 'python' subdirectory -within its install location is appended to the PYTHONPATH environment variable. - -When an environment is created with the rez API or *rez-env* tool, a dependency -resolution algorithm tracks package requirements and resolves to a list of needed -packages. The commands from these packages are concatenated and evaluated, resulting -in a configured environment. Rez is able to configure environments containing -hundreds of packages, often within a few seconds. Resolves can also be saved to file, -and when re-evaluated later will reconstruct the same environment once more. - - -## Examples - -This example places the user into a resolved shell containing the requested packages, -using the [rez-env](Command-Line-Tools#rez-env) tool: - - ]$ rez-env requests-2.2+ python-2.6 'pymongo-0+<2.7' - - You are now in a rez-configured environment. - - resolved by ajohns@nn188.somewhere.com, on Wed Feb 26 15:56:20 2014, using Rez v2.0.0 - - requested packages: - requests-2.2+ - python-2.6 - pymongo-0+<2.7 - - resolved packages: - python-2.6.8 /software/ext/python/2.6.8 - platform-linux /software/ext/platform/linux - requests-2.2.1 /software/ext/requests/2.2.1/python-2.6 - pymongo-2.6.3 /software/ext/pymongo/2.6.3 - arch-x86_64 /software/ext/arch/x86_64 - - > ]$ _ - -This example creates an environment containing the package 'houdini' version 12.5 -or greater, and runs the command 'hescape -h' inside that environment: - - ]$ rez-env houdini-12.5+ -- hescape -h - Usage: hescape [-foreground] [-s editor] [filename ...] - -h: output this usage message - -s: specify starting desktop by name - -foreground: starts process in foreground - -Resolved environments can also be created via the API: - - >>> import subprocess - >>> from rez.resolved_context import ResolvedContext - >>> - >>> r = ResolvedContext(["houdini-12.5+", "houdini-0+<13", "java", "!java-1.8+"]) - >>> p = r.execute_shell(command='which hescape', stdout=subprocess.PIPE) - >>> out, err = p.communicate() - >>> - >>> print out - '/software/ext/houdini/12.5.562/bin/hescape' diff --git a/wiki/pages/Installation.md b/wiki/pages/Installation.md deleted file mode 100644 index 935216fd8..000000000 --- a/wiki/pages/Installation.md +++ /dev/null @@ -1,82 +0,0 @@ -## Installation Script - -To install rez, download the source. Then from the root directory, run: - -``` -]$ python ./install.py -``` - -This installs rez to `/opt/rez`. See `install.py -h` for how to install to a -different location. - -Once the installation is complete, a message tells you how to run it: - -``` -SUCCESS! To activate Rez, add the following path to $PATH: -/opt/rez/bin/rez - -You may also want to source the completion script (for bash): -source /opt/rez/completion/complete.sh -``` - -> [[media/icons/warning.png]] Do _not_ move the installation - re-install to a new -> location if you want to change the install path. If you want to install rez for -> multiple operating systems, perform separate installs for each of those systems. - - -## Installation Via Pip - -It is possible to install rez with pip, like so: - -``` -]$ pip install rez -``` - -However, this comes with a caveat - rez command line tools _are not guaranteed -to work correctly_ once inside a rez environment (ie after using the `rez-env` -command). The reasons are given in the next section. - -Pip installation is adequate however, if all you require is the rez API, or you -don't require its command line tools to be available within a resolved environment. - -Note that running pip-installed rez command line tools will print a warning like so: - -``` -Pip-based rez installation detected. Please be aware that rez command line tools -are not guaranteed to function correctly in this case. See -https://github.com/__GITHUB_REPO__/wiki/Installation#why-not-pip-for-production -for futher details. -``` - - -## Why Not Pip For Production? - -Rez is not a normal python package. Although it can successfully be installed -using standard mechanisms such as pip, this comes with a number of caveats. -Specifically: - -* When within a rez environment (ie after using the `rez-env` command), the rez - command line tools are not guaranteed to function correctly; -* When within a rez environment, other packages' tools (that were also installed - with pip) remain visible, but are not guaranteed to work. - -When you enter a rez environment, the rez packages in the resolve configure -that environment as they see fit. For example, it is not uncommon for a python -package to append to PYTHONPATH. Environment variables such as PYTHONPATH -affect the behaviour of tools, including rez itself, and this can cause it to -crash or behave abnormally. - -When you use the `install.py` script to install rez, some extra steps are taken -to avoid this problem. Specifically: - -* Rez is installed into a virtualenv so that it operates standalone; -* The rez tools are shebanged with `python -E`, in order to protect them from - environment variables that affect python's behaviour; -* The rez tools are stored in their own directory, so that other unrelated tools - are not visible. - -Due to the way standard wheel-based python installations work, it simply is not -possible to perform these extra steps without using a custom installation script. -Wheels do not give the opportunity to run post-installation code; neither do -they provide functionality for specifying interpreter arguments to be added for -any given entry point. diff --git a/wiki/pages/Managing-Packages.md b/wiki/pages/Managing-Packages.md deleted file mode 100644 index 3a2d071c0..000000000 --- a/wiki/pages/Managing-Packages.md +++ /dev/null @@ -1,360 +0,0 @@ -## Ignoring Packages - -Packages can be ignored. When this happens, the package is still present in its -repository, but it will not be visible to the rez API nor to any newly resolved -runtimes. Any runtimes that are currently using an ignored package are unaffected, -since the package's payload has not been removed. - -To ignore a package via comandline: - -``` -]$ # you need to specify the repo, but you'll be shown a list if you don't -]$ rez-pkg-ignore foo-1.2.3 -No action taken. Run again, and set PATH to one of: -filesystem@/home/ajohns/packages - -]$ rez-pkg-ignore foo-1.2.3 filesystem@/home/ajohns/packages -Package is now ignored and will not be visible to resolves -``` - -Via API: - -``` ->>> from rez.package_repository import package_repository_manager ->>> ->>> repo_path = "filesystem@/home/ajohns/packages" ->>> repo = package_repository_manager.get_repository(repo_path) ->>> repo.ignore_package("foo", "1.2.3") -1 # -1: pkg not found; 0: pkg already ignored; 1: pkg ignored -``` - -Both of these options generate a *.ignore\* file (e.g. -*.ignore3.1.2*) next to the package version directory. - -You can also do the reverse (ie unignore a package). Use the `-u` option of -`rez-pkg-ignore`, or the `unignore_package` function on the package repository -object. - - -## Copying Packages - -Packages can be copied from one [package repository](Basic-Concepts#package-repositories) -to another, like so: - -Via commandline: - -``` -]$ rez-cp --dest-path /svr/packages2 my_pkg-1.2.3 -``` - -Via API: - -``` ->>> from rez.package_copy import copy_package ->>> from rez.packages import get_latest_package ->>> ->>> p = get_latest_package("python") ->>> p -Package(FileSystemPackageResource({'location': '/home/ajohns/packages', 'name': 'python', 'repository_type': 'filesystem', 'version': '3.7.4'})) - ->>> r = copy_package(p, "./repo2") ->>> ->>> print(pprint.pformat(r)) -{ - 'copied': [ - ( - Variant(FileSystemVariantResource({'location': '/home/ajohns/packages', 'name': 'python', 'repository_type': 'filesystem', 'index': 0, 'version': '3.7.4'})), - Variant(FileSystemVariantResource({'location': '/home/ajohns/repo2', 'name': 'python', 'repository_type': 'filesystem', 'index': 0, 'version': '3.7.4'})) - ) - ], - 'skipped': [] -} -``` - -Copying packages is actually done one variant at a time, and you can copy some -variants of a package if you want, rather than the entire package. The API call's -return value shows what variants were copied - The 2-tuple in `copied` lists the -source (the variant that was copied from) and destination (the variant that was -created) respectively. - -> [[media/icons/warning.png]] Do not simply copy package directories on disk - -> you should always use `rez-cp`. Copying directly on disk is bypassing rez and -> this can cause problems such as a stale resolve cache. Using `rez-cp` gives -> you more control anyway. - -### Enabling Package Copying - -Copying packages is enabled by default, however you're also able to specify which -packages are and are not _relocatable_, for much the same reasons as given -[here](Managing-Packages#enabling-package-caching). - -You can mark a package as non-relocatable by setting [relocatable](Package-Definition-Guide#relocatable) -to False in its package definition file. There are also config settings that affect relocatability -in the event that relocatable is not defined in a package's definition. For example, -see [default_relocatable](Configuring-Rez#default_relocatable), -[default_relocatable_per_package](Configuring-Rez#default_relocatable_per_package) -and [default_relocatable_per_repository](Configuring-Rez#default_relocatable_per_repository). - -Attempting to copy a non-relocatable package will raise a `PackageCopyError`. -However, note that there is a `force` option that will override this - use at -your own risk. - - -## Moving Packages - -Packages can be moved from one [package repository](Basic-Concepts#package-repositories) -to another. Be aware that moving a package does not actually delete the source -package however. Instead, the source package is hidden (ignored) - it is up to -you to delete it at some later date. - -To move a package via commandline: - -``` -]$ rez-mv --dest-path /packages2 python-3.7.4 /packages -``` - -Via API: - -``` ->>> from rez.package_move import move_package ->>> from rez.packages import get_package_from_repository ->>> ->>> p = get_package_from_repository("python", "3.7.4", "/packages") ->>> p -Package(FileSystemPackageResource({'location': '/packages', 'name': 'python', 'repository_type': 'filesystem', 'version': '3.7.4'})) ->>> ->>> new_p = move_package(p, "/packages2") ->>> new_p -Package(FileSystemPackageResource({'location': '/packages2', 'name': 'python', 'repository_type': 'filesystem', 'version': '3.7.4'})) ->>> ->>> p = get_package_from_repository("python", "3.7.4", "/packages") ->>> p -None -``` - -Be aware that a non-relocatable package is also not movable (see -[here](Package-Definition-Guide#relocatable) for more details). Like package -copying, there is a `force` option to move it regardless. - -A typical reason you might want to move a package is to archive packages that are -no longer in use. In this scenario, you would move the package to some archival -package repository. In case an old runtime needs to be resurrected, you would add -this archival repository to the packages path before performing the resolve. Note -that you will probably want to use the `--keep-timestamp` option when doing this, -otherwise rez will think the package did not exist prior to its archival date. - - -## Removing Packages - -Packages can be removed. This is different from ignoring - the package and its -payload is deleted from storage, whereas ignoring just hides it. It is not -possible to un-remove a package. - -To remove a package via commandline: - -``` -]$ rez-rm --package python-3.7.4 /packages -``` - -Via API: - -``` ->>> from rez.package_remove import remove_package ->>> ->>> remove_package("python", "3.7.4", "/packages") -``` - -During the removal process, package versions will first be ignored so that -partially-deleted versions are not visible. - -It can be useful to ignore packages that you don't want to use anymore, and -actually remove them at a later date. This gives you a safety buffer in case -current runtimes are using the package - they won't be affected if the package is -ignored, but could break if it is removed. - -To facilitate this workflow, `rez-rm` lets you remove all packages that have -been ignored for longer than N days (using the timestamp of the -*.ignore\* file). Here we remove all packages that have been ignored -for 30 days or longer: - -``` -]$ rez-rm --ignored-since=30 -v -14:47:09 INFO Searching filesystem@/home/ajohns/packages... -14:47:09 INFO Removed python-3.7.4 from filesystem@/home/ajohns/packages -1 packages were removed. -``` - -Via API: - -``` ->>> from rez.package_remove import remove_packages_ignored_since ->>> ->>> remove_packages_ignored_since(days=30) -1 -``` - - -## Package Caching - -Package caching is a feature that copies package payloads onto local disk in -order to speed up runtime environments. For example, if your released packages -reside on shared storage (which is common), then running say, a Python process, -will fetch all source from the shared storage across your network. The point of -the cache is to copy that content locally instead, and avoid the network cost. - -> [[media/icons/info.png]] Please note: Package caching does _NOT_ cache package -> definitions - only their payloads (ie, the package root directory). - -### Enabling Package Caching - -Package caching is not enabled by default. To enable it, you need to configure -[cache_packages_path](Configuring-Rez#cache_packages_path) to specify a path to -store the cache in. - -You also have granular control over whether an individual package will or will -not be cached. To make a package cachable, you can set [cachable](Package-Definition-Guide#cachable) - to False in its package definition file. Reasons you may _not_ want to do this include -packages that are large, or that aren't relocatable because other compiled packages are -linked to them in a way that doesn't support library relocation. - -There are also config settings that affect cachability in the event that `cachable` -is not defined in a package's definition. For example, see -[default_cachable](Configuring-Rez#default_cachable), -[default_cachable_per_package](Configuring-Rez#default_cachable_per_package) -and [default_cachable_per_repository](Configuring-Rez#default_cachable_per_repository). - -Note that you can also disable package caching on the command line, using -`rez-env --no-pkg-caching`. - -### Verifying - -When you resolve an environment, you can see which variants have been cached by -noting the `cached` label in the righthand column of the `rez-context` output, -as shown below: - -``` -]$ rez-env Flask - -You are now in a rez-configured environment. - -requested packages: -Flask -~platform==linux (implicit) -~arch==x86_64 (implicit) -~os==Ubuntu-16.04 (implicit) - -resolved packages: -Flask-1.1.2 /home/ajohns/package_cache/Flask/1.1.2/d998/a (cached) -Jinja2-2.11.2 /home/ajohns/package_cache/Jinja2/2.11.2/6087/a (cached) -MarkupSafe-1.1.1 /svr/packages/MarkupSafe/1.1.1/d9e9d80193dcd9578844ec4c2c22c9366ef0b88a -Werkzeug-1.0.1 /home/ajohns/package_cache/Werkzeug/1.0.1/fe76/a (cached) -arch-x86_64 /home/ajohns/package_cache/arch/x86_64/6450/a (cached) -click-7.1.2 /home/ajohns/package_cache/click/7.1.2/0da2/a (cached) -itsdangerous-1.1.0 /home/ajohns/package_cache/itsdangerous/1.1.0/b23f/a (cached) -platform-linux /home/ajohns/package_cache/platform/linux/9d4d/a (cached) -python-3.7.4 /home/ajohns/package_cache/python/3.7.4/ce1c/a (cached) -``` - -For reference, cached packages also have their original payload location stored to -an environment variable like so: - -``` -]$ echo $REZ_FLASK_ORIG_ROOT -/svr/packages/Flask/1.1.2/88a70aca30cb79a278872594adf043dc6c40af99 -``` - -### How it Works - -Package caching actually caches _variants_, not entire packages. When you perform -a resolve, or source an existing context, the variants required are copied to -local disk asynchronously (if they are cachable), in a separate process called -`rez-pkg-cache`. This means that a resolve will not necessarily use the cached -variants that it should, the first time around. Package caching is intended to have -a cumulative effect, so that more cached variants will be used over time. This is -a tradeoff to avoid blocking resolves while variant payloads are copied across -your network (and that can be a slow process). - -Note that a package cache is **not** a package repository. It is simply a store -of variant payloads, structured in such a way as to be able to store variants from -any package repository, into the one shared cache. - -Variants that are cached are assumed to be immutable - no check is done to see if -a variant's payload has changed, and needs to replace an existing cache entry. So -you should _not_ enable caching on package repositories where packages may get -overwritten. It is for this reason that caching is disabled for local packages by -default (see [package_cache_local](Configuring-Rez#package_cache_local)). - -### Commandline Tool - -#### Inspection - -Use the `rez-pkg-cache` tool to view the state of the cache, and to perform -warming and deletion operations. Example output follows: - -``` -]$ rez-pkg-cache -Package cache at /home/ajohns/package_cache: - -status package variant uri cache path ------- ------- ----------- ---------- -cached Flask-1.1.2 /svr/packages/Flask/1.1.2/package.py[0] /home/ajohns/package_cache/Flask/1.1.2/d998/a -cached Jinja2-2.11.2 /svr/packages/Jinja2/2.11.2/package.py[0] /home/ajohns/package_cache/Jinja2/2.11.2/6087/a -cached Werkzeug-1.0.1 /svr/packages/Werkzeug/1.0.1/package.py[0] /home/ajohns/package_cache/Werkzeug/1.0.1/fe76/a -cached arch-x86_64 /svr/packages/arch/x86_64/package.py[] /home/ajohns/package_cache/arch/x86_64/6450/a -cached click-7.1.2 /svr/packages/click/7.1.2/package.py[0] /home/ajohns/package_cache/click/7.1.2/0da2/a -cached itsdangerous-1.1.0 /svr/packages/itsdangerous/1.1.0/package.py[0] /home/ajohns/package_cache/itsdangerous/1.1.0/b23f/a -cached platform-linux /svr/packages/platform/linux/package.py[] /home/ajohns/package_cache/platform/linux/9d4d/a -copying python-3.7.4 /svr/packages/python/3.7.4/package.py[0] /home/ajohns/package_cache/python/3.7.4/ce1c/a -stalled MarkupSafe-1.1.1 /svr/packages/MarkupSafe/1.1.1/package.py[1] /home/ajohns/package_cache/MarkupSafe/1.1.1/724c/a - -``` - -Each variant is stored into a directory based on a partial hash of that variant's -unique identifier (its "handle"). The package cache is thread- and multiprocess- -proof, and uses a file lock to control access where necessary. - -Cached variants have one of the following statuses at any given time: - -* **copying**: The variant is in the process of being copied into the cache, and is not - yet available for use; -* **cached**: The variant has been cached and is ready for use; -* **stalled**: The variant was getting copied, but something went wrong and there is - now a partial copy present (but unused) in the cache. - -#### Logging - -Caching operations are stored into logfiles within the cache directory. To view: - -``` -]$ rez-pkg-cache --logs -rez-pkg-cache 2020-05-23 16:17:45,194 PID-29827 INFO Started daemon -rez-pkg-cache 2020-05-23 16:17:45,201 PID-29827 INFO Started caching of variant /home/ajohns/packages/Werkzeug/1.0.1/package.py[0]... -rez-pkg-cache 2020-05-23 16:17:45,404 PID-29827 INFO Cached variant to /home/ajohns/package_cache/Werkzeug/1.0.1/fe76/a in 0.202576 seconds -rez-pkg-cache 2020-05-23 16:17:45,404 PID-29827 INFO Started caching of variant /home/ajohns/packages/python/3.7.4/package.py[0]... -rez-pkg-cache 2020-05-23 16:17:46,006 PID-29827 INFO Cached variant to /home/ajohns/package_cache/python/3.7.4/ce1c/a in 0.602037 seconds -``` - -#### Cleaning The Cache - -Cleaning the cache refers to deleting variants that are stalled or no longer in use. -It isn't really possible to know whether a variant is in use, so there is a -configurable [package_cache_max_variant_days](Configuring-Rez#package_cache_max_variant_days) -setting, that will delete variants that have not been used (ie that have not appeared -in a created or sourced context) for more than N days. - -You can also manually remove variants from the cache using `rez-pkg-cache -r`. -Note that when you do this, the variant is no longer available in the cache, -however it is still stored on disk. You must perform a clean (`rez-pkg-cache --clean`) -to purge unused cache files from disk. - -You can use the [package_cache_clean_limit](Configuring-Rez#package_cache_clean_limit) -setting to asynchronously perform some cleanup every time the cache is updated. If -you do not use this setting, it is recommended that you set up a cron or other form -of execution scheduler, to run `rez-pkg-cache --clean` periodically. Otherwise, -your cache will grow indefinitely. - -Lastly, note that a stalled variant will not attempt to be re-cached until it is -removed by a clean operation. Using `package_cache_clean_limit` will not clean -stalled variants either, as that could result in a problematic variant getting -cached, then stalled, then deleted, then cached again and so on. You must run -`rez-pkg-cache --clean` to delete stalled variants. diff --git a/wiki/pages/Notes.md b/wiki/pages/Notes.md deleted file mode 100644 index 5efbbdc0a..000000000 --- a/wiki/pages/Notes.md +++ /dev/null @@ -1,4 +0,0 @@ - - -## Bez Deprecation, awaiting PR merge - diff --git a/wiki/pages/Package-Commands.md b/wiki/pages/Package-Commands.md deleted file mode 100644 index 914a2f1b3..000000000 --- a/wiki/pages/Package-Commands.md +++ /dev/null @@ -1,612 +0,0 @@ -## Overview - -Package definition files (*package.py*) usually define a *commands* section. This is a python -function that determines how the environment is configured in order to include the package. - -Consider the simple example: - - def commands(): - env.PYTHONPATH.append("{root}/python") - env.PATH.append("{root}/bin") - -This is a typical case, where a package adds its source path to PYTHONPATH, and its tools to -PATH. The "{root}" string expands to the installation directory of the package. - -When a rez environment is configured, every package in the resolve list has its *commands* section -interpreted and converted into shell code (the language - bash or other - depends on the platform -and is extensible). The resulting shell code is sourced, and this configures the environment. -Within a configured environment, the variable *REZ_CONTEXT_FILE* points at this shell code, and the -command *rez-context --interpet* prints it. - -The python API that you use in the *commands* section is called *rex* (Rez EXecution language). It -is an API for performing shell operations in a shell-agnostic way. Some common operations you would -perform with this API include setting environment variables, and appending/prepending path-like -environment variables. - -> [[media/icons/info.png]] By default, environment variables that are not referenced by any package -> are left unaltered. There will typically be many system variables that are left unchanged. - -> [[media/icons/warning.png]] If you need to import any python modules to use in a *commands* -> section, the import statements **must** appear inline to that function. - -## Order Of Command Execution - -The order in which package commands are interpreted depends on two factors - the order in which -the packages were requested, and dependencies between packages. This order can be defined as: - -* If package *A* was requested before package *B*, then *A*'s commands are interpreted before *B*'s; -* Unless package *A* requires (depends on) *B*, in which case *B* will be interpreted before *A*. - -Consider a package *maya_anim_tool*. Let us say this is a maya plugin. Naturally it has a dependency -on *maya*, therefore *maya*'s commands will be interpreted first. This is because the maya plugin -may depend on certain environment variables that *maya* sets. For example, *maya* might initialize -the *MAYA_PLUG_IN_PATH* environment variable, and *maya_anim_tool* may then append to this -variable. - -For example, consider the request: - - ]$ rez-env maya_anim_tool-1.3+ PyYAML-3.10 maya-2015 - -Assuming that *PyYAML* depends on *python*, and *maya_anim_tool* depends on *maya*, then the -resulting *commands* execution order would be: - -* maya; -* maya_anim_tool; -* python; -* PyYAML. - -## Variable Appending And Prepending - -Path-like environment variables can be appended and prepended like so: - - env.PATH.append("{root}/bin") - -However, the first append/prepend operation on any given variable actually **overwrites** the -variable, rather than appending. Why does this happen? Consider *PYTHONPATH* - if an initial -overwrite did not happen, then any modules visible on *PYTHONPATH* before the rez environment was -configured would still be there. This would mean you may not have a properly configured -environment. If your system *PyQt* were on *PYTHONPATH* for example, and you used *rez-env* to set -a different *PyQt* version, an attempt to import it within the configured environment would still, -incorrectly, import the system version. - -> [[media/icons/info.png]] *PATH* is a special case. It is not simply overwritten, because if that -> happened you would lose important system paths and thus utilities like *ls* and *cd*. In this -> case the system paths are appended back to *PATH* after all commands are interpreted. The system -> paths are defined as the default value of *PATH* in a non-interactive shell. - -> [[media/icons/under_construction.png]] Better control over environment variable initialization is -> coming. Specifically, you will be able to specify various modes for variables. For example, one -> mode will append the original (pre-rez) value back to the resulting value. - -## String Expansion - -### Object Expansion - -Any of the objects available to you in a *commands* section can be referred to in formatted strings -that are passed to rex functions such as *setenv* and so on. For example, consider the code: - - appendenv("PATH", "{root}/bin") - -Here, "{root}" will expand out to the value of [root](#root), which is the installation path of the -package ("this.root" could also have been used). - -You don't *have* to use this feature; it is provided as a convenience. For example, the following -code is equivalent to the previous example, and is just as valid (but more verbose): - - import os.path - appendenv("PATH", os.path.join(root, "bin")) - -Object string expansion is also supported when setting an environment variable via the *env* object: - - env.FOO_LIC = "{this.root}/lic" - -### Environment Variable Expansion - -Environment variable expansion is also supported when passed to rex functions. The syntaxes *$FOO* -and *${FOO}* are supported, regardless of the syntax supported by the target shell. - -### Literal Strings - -You can use the [literal](#literal) function to inhibit object- and environment variable- string -expansion. For example, the following code will set the environment variable to the literal string: - - env.TEST = literal("this {root} will not expand") - -There is also an expandable function, which matches the default behavior. You wouldn't typically -use this function; however, you can define a string containing literal and expandable parts by -chaining together *literal* and *expandable*: - - env.DESC = literal("the value of {root} is").expandable("{root}") - -### Explicit String Expansion - -Object string expansion usually occurs **only** when a string is passed to a rex function, or to -the *env* object. For example the simple statement *var = "{root}/bin"* would not expand "{root}" -into *var*. However, you can use the [expandvars](#expandvars) function to enable this behavior -explicitly: - - var = expandvars("{root}/bin") - -The *expandvars* and *expandable* functions are slightly different - *expandable* will generate a -shell variable assignment that will expand out; *expandvars* will expand the value immediately. - -This table illustrates the difference between *literal*, *expandable* and *expandvars*: - -package command | equivalent bash command ---------------------------------|------------------------ -env.FOO = literal("${USER}") | export FOO='${USER}' -env.FOO = expandable("${USER}") | export FOO="${USER}" -env.FOO = expandvars("${USER}") | export FOO="jbloggs" - -## Filepaths - -Rez expects POSIX-style filepath syntax in package commands, regardless of the shell or platform. -Thus, even if you're on Windows, you should do this: - - def commands(): - env.PATH.append("{root}/bin") # note the forward slash - -Where necessary, filepaths will be automatically normalized for you - that is, converted into -the syntax expected by the shell. In order for this to work correctly however, rez needs to know -what environment variables are actually paths. You determine this with the -[pathed_env_vars](Configuring-Rez#pathed_env_vars) config setting. By default, any environment -variable ending in `PATH` will be treated as a filepath or list of filepaths, and any -set/append/prepend operation on it will cause those values to be path-normalized automatically. - -> [[media/icons/warning.png]] Avoid using `os.pathsep` or hardcoded lists of paths such as -> `{root}/foo:{root}/bah`. Doing so can cause your package to be incompatible with some shells or -> platforms. Even the seemingly innocuous `os.pathsep` is an issue, because there are some cases -> (eg Git for Windows, aka git-bash) where the shell's path separator does not match the underlying -> system's. - -## Pre And Post Commands - -Occasionally it's useful for a package to run commands either before or after all other packages, -regardless of the command execution order rules. This can be achieved by defining a *pre_commands* -or *post_commands* function. A package can have any, all or none of *pre_commands*, *commands* and -*post_commands* defined, although it is very common for a package to define just *commands*. - -The order of command execution is: - -* All package *pre_commands* are executed, in standard execution order; -* Then, all package *commands* are executed, in standard execution order; -* Then, all package *post_commands* are executed, in standard execution order. - -## Pre Build Commands - -If a package is being built, that package's commands are not run, simply because that package is -not present in its own build environment! However, sometimes there is a need to run commands -specifically for the package being built. For example, you may wish to set some environment -variables to pass information along to the build system. - -The *pre_build_commands* function does just this. It is called prior to the build. Note that info -about the current build (such as the installation path) is available in a -[build](#build) object (other commands functions do not have this object visible). - -## Pre Test Commands - -Sometimes it's useful to perform some extra configuration in the environment that a package's test -will run in. You can define the *pre_test_commands* function to do this. It will be invoked just -before the test is run. As well as the standard [this](#this) object, a [test](#test) object is also -provided to distinguish which test is about to run. - -## A Largish Example - -Here is an example of a package definition with a fairly lengthy *commands* section: - - name = "foo" - - version = "1.0.0" - - requires = [ - "python-2.7", - "~maya-2015" - ] - - def commands(): - import os.path # imports MUST be inline to the function - - # add python module, executables - env.PYTHONPATH.append("{this.root}/python") - env.PATH.append("{this.root}/bin") - - # show include path if a build is occurring - if building: - env.FOO_INCLUDE_PATH = "{this.root}/include" - - # debug support to point at local config - if defined("DEBUG_FOO"): - conf_file = os.path.expanduser("~/.foo/config") - else: - conf_file = "{this.root}/config" - env.FOO_CONFIG_FILE = conf_file - - # if maya is in use then include the maya plugin part of this package - if "maya" in resolve: - env.MAYA_PLUG_IN_PATH.append("{this.root}/maya/plugins") - - if resolve.maya.version.minor == "sp3": - error("known issue with GL renderer in service pack 3, beware") - - # license file per major version - env.FOO_LIC = "/lic/foo_{this.version.major}.lic" - -## Objects - -Various objects and functions are available to use in the *commands* function (as well as -*pre_commands* and *post_commands*). For example, *env* is a dict-like object that represents all -the environment variables being constructed in the target environment. - -Following is a list of the objects and functions available. - -### alias -*Function* - - alias("nukex", "Nuke -x") - -Create a command alias. - -> [[media/icons/info.png]] In *bash*, aliases are implemented as bash functions. - -### base -*String* - -See [this.base](#thisbase). - -### build -*Dict-like object* - - if build.install: - info("An installation is taking place") - -This object is only available in the [pre_build_commands](#pre-build-commands) -function. It has the following fields: - -#### build.build_type -*String* - -One of 'local', 'central'. The type is _central_ if a package _release_ is occurring, and _local_ -otherwise. - -#### build.install -*Boolean* - -True if an installation is taking place, False otherwise. - -#### build.build_path -*String* - -Path to the build directory (not the installation path). This will typically reside somewhere -within the `./build` subdirectory of the package being built. - -#### build.install_path -Installation directory. Note that this will be set, even if an installation is _not_ taking place. -Do not check this variable to detect if an installation is occurring - see `build.install` instead. - -### building -*Boolean* - - if building: - env.FOO_INCLUDE_PATH = "{root}/include" - -This boolean variable is *True* if a build is occurring (typically done via the *rez-build* tool), -and *False* otherwise, however, the `commands` block is only executed when the package is brought -into a resolved environment, so this is not used when the package itself is building. Typically a -package will use this variable to set environment variables that are only useful during when other -packages are being built - C++ header include paths are a good example. - -### command -*Function* - - command("rm -rf ~/.foo_plugin") - -Run an arbitrary shell command. Note that you cannot return a value from this function call, because -*the command has not yet run*. All of the packages in a resolve only have their commands executed -after all packages have been interpreted and converted to the target shell language. Therefore any -value returned from the command, or any side effect the command has, is not visible to any package. - -You should prefer to perform simple operations (such as file manipulations and so on) in python -where possible instead. Not only does that take effect immediately, but it's also more cross -platform. For example, instead of running the command above, we could have done this: - - def commands(): - import shutil - import os.path - path = os.path.expanduser("~/.foo_plugin") - if os.path.exists(path): - shutil.rmtree(path) - -### comment -*Function* - - if "nuke" in resolve: - comment("note: taking over 'nuke' binary!") - alias("nuke", "foo_nuke_replacer") - -Creates a comment line in the converted shell script code. This is only visible if the user views -the current shell's code using the command *"rez-context --interpret"* or looks at the file -referenced by the environment variable *REZ_CONTEXT_FILE*. You would create a comment for debugging -purposes. - -### defined -*Function* - - if defined("REZ_MAYA_VERSION"): - env.FOO_MAYA = 1 - -Use this boolean function to determine whether or not an environment variable is set. - -### env -*Dict-like object* - - env.FOO_DEBUG = 1 - env["BAH_LICENSE"] = "/lic/bah.lic" - -The *env* object represents the environment dict of the configured environment. Note that this is -different from the standard python *os.environ* dict, which represents the current environment, -not the one being configured. If a prior package's *commands* set a variable via the *env* object, -it will be visible only via *env*, not *os*. The *os* dict hasn't been updated because the target -configured environment does not yet exist! - -The *env* object also provides the following functions: - -#### env.append -*Function* - - env.PATH.append("{root}/bin") - -Appends a value to an environment variable. By default this will use the *os.pathsep* delimiter -between list items, but this can be overridden using the config setting *env_var_separators*. See -[here](#variable-appending-and-prepending) for further information on the behavior of this function. - -#### env.prepend -*Function* - - env.PYTHONPATH.prepend("{root}/python") - -like *env.append*, but prepends the environment variable instead. - -### ephemerals -*Dict-like object* - - if "foo.cli" in ephemerals: - info("Foo cli option is being specified!") - -A dict representing the list of ephemerals in the resolved environment. Each item is a -string (the full request, eg `.foo.cli-1`), keyed by the ephemeral package name. Note -that you do **not** include the leading `.` when getting items from the `ephemerals` -object. - -Use `get_range` to test with the [intersects](Package-Commands#intersects) function. -Here, we enable foo's commandline tools by default, unless explicitly disabled via -a request for `.foo.cli-0`: - - if intersects(ephemerals.get_range("foo.cli", "1"), "1"): - info("Enabling foo cli tools") - env.PATH.append("{root}/bin") - -### error -*Function* - - if "PyQt" in resolve: - error("The floob package has problems running in combo with PyQt") - -Prints to standard error. - -> [[media/icons/info.png]] This function just prints the error, it does not prevent the target -> environment from being constructed (use the [stop](#stop) command for that). - -### getenv -*Function* - - if getenv("REZ_MAYA_VERSION") == "2016.sp1": - pass - -Gets the value of an environment variable; raises *RexUndefinedVariableError* if not set. - -### implicits -*Dict-like object* - - if "platform" in implicits: - pass - -This is similar to the [request](#request) object, but it contains only the package requests as -defined by the [implicit_packages](Configuring-Rez#implicit_packages) configuration setting. - -### info -*Function* - - info("floob version is %s" % resolve.floob.version) - -Prints to standard out. - -### intersects -*Function* - - if intersects(resolve.maya, "2019+"): - info("Maya 2019 or greater is present") - -A boolean function that returns True if the version or version range of the given -object, intersects with the given version range. Valid objects to query include: - -* A resolved package, eg `resolve.maya`; -* A package request, eg `request.foo`; -* A version of a resolved package, eg `resolve.maya.version`; -* A resolved ephemeral, eg `ephemerals.foo`; -* A version range object, eg `ephemerals.get_range('foo.cli', '1')` - -> [[media/icons/warning.png]] Do **not** do this: -> `if intersects(ephemerals.get("foo.cli", "0"), "1"): ...` -> If 'foo.cli' is not present, this will unexpectedly compare the unversioned -> package named "0" against the version range "1", which will succeed! Use -> `get_range` when testing intersections on the _request_ and _ephemerals_ -> objects instead: -> `if intersects(ephemerals.get_range("foo.cli", "0"), "1"): ...` - -### literal -*Function* - - env.FOO = literal("this {root} will not expand") - -Inhibits expansion of object and environment variable references. You can also chain together -*literal* and *expandable* functions like so: - - env.FOO = literal("the value of {root} is").expandable("{root}") - -### optionvars -*Function* - -A `dict.get` like function for package accessing arbitrary data from `optionvars` in rez config. See the [optionvars](Configuring-Rez#optionvars) configuration setting for more details. - -### request -*Dict-like object* - - if "maya" in request: - info("maya was asked for!") - -A dict representing the list of package requests. Each item is a request string keyed by the -package name. For example, consider the package request: - - ]$ rez-env maya-2015 maya_utils-1.2+<2 !corelib-1.4.4 - -This request would yield the following *request* object: - - { - "maya": "maya-2015", - "maya_utils": "maya_utils-1.2+<2", - "corelib": "!corelib-1.4.4" - } - -Use `get_range` to test with the [intersects](Package-Commands#intersects) function: - - if intersects(request.get_range("maya", "0"), "2019"): - info("maya 2019.* was asked for!") - -> [[media/icons/info.png]] If multiple requests are present that refer to the same package, the -> request is combined ahead of time. In other words, if requests *foo-4+* and *foo-<6* were both -> present, the single request *foo-4+<6* would be present in the *request* object. - -### resolve -*Dict-like object* - - if "maya" in resolve: - info("Maya version is %s", resolve.maya.version) - # ..or resolve["maya"].version - -A dict representing the list of packages in the resolved environment. Each item is a -[Package](Package-Definition-Guide) object, keyed by the package name. - -### root -*String* - -See [this.root](#thisroot). - -### setenv -*Function* - - setenv("FOO_PLUGIN_PATH", "{root}/plugins") - -This function sets an environment variable to the given value. It is equivalent to setting a -variable via the *env* object (eg, "env.FOO = 'BAH'"). - -### source -*Function* - - source("{root}/scripts/init.sh") - -Source a shell script. Note that, similarly to *commands*, this function cannot return a value, and -any side effects that the script sourcing has is not visible to any packages. For example, if the -*init.sh* script above contained *"export FOO=BAH"*, a subsequent test for this variable on the -*env* object would yield nothing. - -### stop -*Function* - - stop("The value should be %s", expected_value) - -Raises an exception and stops a resolve from completing. You should use this when an unrecoverable -error is detected and it is not possible to configure a valid environment. - -### system -*System object* - - if system.platform == "windows": - ... - -This object provided system information, such as current platform, arch and os. See -[the source](https://github.com/__GITHUB_REPO__/blob/__GITHUB_BRANCH__/src/rez/system.py) for more info. - -### test -*Dict-like object* - - if test.name == "unit": - info("My unit test is about to run yay") - -This object is only available in the [pre_test_commands](#pre-test-commands) function. It has the -following fields: - -#### test.name -*String* - -Name of the test about to run. - -### this -*Package object* - - import os.path - env.PATH.append(os.path.join(this.root, "bin")) - -The *this* object represents the current package. The following attributes are most commonly used -in a *commands* section (though you have access to all package attributes - see -[here](Package-Definition-Guide)): - -#### this.base -*String* - -Similar to *this.root*, but does not include the variant subpath, if there is one. Different -variants of the same package share the same *base* directory. See [here](Variants) for more -information on package structure in relation to variants. - -#### this.name -*String* - -The name of the package, eg 'houdini'. - -#### this.root -*String* - -The installation directory of the package. If the package contains variants, this path will include -the variant subpath. This is the directory that contains the installed package payload. See -[here](Variants) for more information on package structure in relation to variants. - -#### this.version -*Version object* - -The package version. It can be used as a string, however you can also access specific tokens in the -version (such as major version number and so on), as this code snippet demonstrates: - - env.FOO_MAJOR = this.version.major # or, this.version[0] - -The available token references are *this.version.major*, *this.version.minor* and -*this.version.patch*, but you can also use a standard list index to reference any version token. - -### undefined -*Function* - - if undefined("REZ_MAYA_VERSION"): - info("maya is not present") - -Use this boolean function to determine whether or not an environment variable is set. This is the -opposite of [defined](#defined). - -### unsetenv -*Function* - - unsetenv("FOO_LIC_SERVER") - -Unsets an environment variable. This function does nothing if the environment variable was not set. - -### version -*Version object* - -See [this.version](#thisversion). diff --git a/wiki/pages/Package-Definition-Guide.md b/wiki/pages/Package-Definition-Guide.md deleted file mode 100644 index b39ea66b0..000000000 --- a/wiki/pages/Package-Definition-Guide.md +++ /dev/null @@ -1,925 +0,0 @@ -## Overview - -Packages are defined by a *package definition file*. This is typically a file named *package.py* -that is located in the root directory of each package install. For example, given package -repository location */packages/inhouse*, the package definition file for package "foo-1.0.0" would -be */packages/inhouse/foo/1.0.0/package.py*. - -Here is an example package definition file: - - name = 'sequence' - - version = '2.1.2' - - description = 'Sequence detection library.' - - authors = ['ajohns'] - - tools = [ - 'lsq', - 'cpq' - ] - - requires = [ - 'python-2.6+<3', - 'argparse' - ] - - def commands(): - env.PATH.append("{root}/bin") - env.PYTHONPATH.append("{root}/python") - - uuid = '6c43d533-92bb-4f8b-b812-7020bf54d3f1' - -## Package Attributes - -Every variable defined in the package definition file becomes an attribute on the built or -installed package. This includes attributes that are not in the -[standard list](#standard-package-attributes) - you can add any custom attribute to a package. - -Some variables are not, however, added as package attributes. Consider the following package -definition snippet: - - import sys - - description = "This package was built on %s" % sys.platform - -Here we do not want *sys* to become a package attribute, because providing a python module as a -package attribute is nonsensical. - -Python variables that do **not** become package attributes include: - -* Python modules; -* Functions, not include early- and late- binding functions (see next), and not including the - *commands* and related functions; -* Any variable with a leading double underscore; -* Any variable that is a [build-time package attribute](#build-time-package-attributes). - -### Function Attributes - -Package attributes can be implemented as functions - the return value of the function becomes -the attribute value. There are two types of attribute functions - *early binding* functions, -and *late binding* functions - and these are decorated using *@early* and *@late* respectively. - -> [[media/icons/warning.png]] The *commands* functions are an exception to the rule. They are -> late bound, but are not the same as a standard function attribute, and are *never* decorated -> with the early or late decorators. - -#### Early Binding Functions - -Early binding functions use the *@early* decorator. They are evaluated at *build time*, hence the -'early' in 'early binding'. Any package attribute can be implemented as an early binding function. - -Here is an example of an *authors* attribute that is automatically set to the contributors of the -package's git project: - - @early() - def authors(): - import subprocess - p = subprocess.Popen("git shortlog -sn | cut -f2", - shell=True, stdout=subprocess.PIPE) - out, _ = p.communicate() - return out.strip().split('\n') - -> [[media/icons/info.png]] You can assume that during evaluation of early binding functions, the -> current working directory is the root directory containing your *package.py*. - -An early bound function can also have access to other package attributes. To do this, use the -implicit *this* object: - - @early() - def description(): - # a not very useful description - return "%s version %s" % (this.name, this.version) - -> [[media/icons/warning.png]] Do not reference other early bound or late bound attributes in -> your early bound function - an error will be raised if you do. - -Early binding functions are a convenience - you can always use an arbitrary function instead, like so: - - def _description(): - return "%s version %s" % (this.name, this.version) - - description = _description() - -However, using early binding results in a package definition that is cleaner and more explicit - it -is clear that an attribute is intended to be evaluated at build time, and you avoid the need to -define an arbitrary function earlier in the python source. You can always use a combination of the -two as well - an early binding function can call an arbitrary function defined at the bottom of -your definition file. - -##### Available Objects - -Following is the list of objects that are available during early evaluation. - -* *building* - see [building](Package-Commands#building); -* *build_variant_index* - the index of the variant currently being built. This is only relevant if - `building` is True. -* *build_variant_requires* - the subset of package requirements specific to the variant - currently being built. This is a list of `PackageRequest` objects. This is only relevant if - `building` is True. -* *this* - the current package, as described previously. - -Be aware that early-bound functions are actually evaluated multiple times during a build - once -pre-build, and once per variant, during its build. This is necessary in order for early-bound -functions to change their return value based on variables like `build_variant_index`. Note that the -*pre-build* evaluated value is the one set into the installed package, and in this case, `building` -is False. - -An example of where you'd need to be aware of this is if you wanted the `requires` field to include -a certain package at runtime only (ie, not present during the package build). In this case, `requires` -might look like so: - - @early() - def requires(): - if building: - return ["python-2"] - else: - return ["runtimeonly-1.2", "python-2"] - -> [[media/icons/warning.png]] You **must** ensure that your early-bound function returns the value -> you want to see in the installed package, when `building` is False. - -#### Late Binding Functions - -Late binding functions stay as functions in the installed package definition, and are only evaluated -lazily, when the attribute is accessed for the first time (the return value is then cached). - -Not any attribute can be implemented as a late binding function. The allowed attributes are: - -* requires -* build_requires -* private_build_requires -* tools -* help -* any arbitrary attribute - -Here is an example of a late binding *tools* attribute: - - @late() - def tools(): - import os - - # get everything in bin dir - binpath = os.path.join(this.root, "bin") - result = os.listdir(binpath) - - # we don't want artists to see the admin tools - if os.getenv("_USER_ROLE") != "superuser": - result = set(result) - set(["delete-all", "mod-things"]) - - return list(result) - -> [[media/icons/warning.png]] Late binding function attributes *must* perform any necessary imports -> *within* the function, not at the top of the *package.py* file. - -Note that, if this function just returned the binaries found in the bin dir, it would have made -more sense to implement this as an *early binding* function - no code evaluation has to happen at -runtime then, so it's cheaper. However here a modification is made based on the value of the -*_USER_ROLE* environment variable, which isn't known at build time. - -If some information for an attribute could be calculated once at build time, you can reduce the -runtime cost by storing that part into an early binding arbitrary attribute. For example, we could -reimplement the above example like so: - - @late() - def tools(): - import os - result = this._tools - - # we don't want artists to see the admin tools - if os.getenv("_USER_ROLE") != "superuser": - result = set(result) - set(["delete-all", "mod-things"]) - - return list(result) - - @early() - def _tools(): - import os - return os.listdir("./bin") - -Note how in the *_tools* function we're referring to a relative path. Remember that early binding -functions are evaluated at build time - the package hasn't actually been built or installed yet, -so attributes such as *this.root* don't exist. - -##### The *in_context* Function - -When late binding functions are evaluated, a boolean function *in_context* is present, which -returns True if the package is part of a resolved context, or False otherwise. For example, -if you just use the rez API to iterate over packages (as the *rez-search* tool does), these -packages do not belong to a context; however if you create a *ResolvedContext* object (as -the *rez-env* tool does) and iterate over its resolved packages, these belong to a context. - -The in-context or not-in-context distinction is important, because often the package attribute -will need information from the context to give desired behavior. For example, consider the -late binding *tool* attribute below: - - @late() - def tools(): - result = ["edit"] - - if in_context() and "maya" in request: - result.append("maya-edit") - - return result - -Here the *request* object is being checked to see if the *maya* package was requested in the -current env; if it was, a maya-specific tool *maya-edit* is added to the tool list. - -> [[media/icons/warning.png]] Always ensure your late binding function returns a sensible -> value regardless of whether *in_context* is True or False. Otherwise, simply trying to -> query the package attributes (using *rez-search* for example) may cause errors. - -##### Available Objects - -Following is the list of objects that are available during late evaluation, if *in_context* -is *True*: - -* *context* - the *ResolvedContext* instance this package belongs to; -* *system* - see [system](Package-Commands#system); -* *building* - see [building](Package-Commands#building); -* *request* - see [request](Package-Commands#request); -* *implicits* - see [implicits](Package-Commands#implicits). - -The following objects are available in *all* cases: - -* *this* - the current package/variant (see note below); -* *in_context* - the *in_context* function itself. - -> [[media/icons/warning.png]] The *this* object may be either a package or a variant, -> depending on the situation. For example, if *in_context* is True, then *this* is a -> variant, because variants are the objects present in a resolved context. On the other -> hand, if a package is accessed via API (for example, by using the *rez-search* tool), -> then *this* may be a package. The difference matters, because variants have some -> attributes that packages don't - notably, *root* and *index*. Use the properties -> `this.is_package` and `this.is_variant` to distinguish the case if needed. - -##### Example - Late Bound build_requires - -Here is an example of a package.py with a late-bound `build_requires` field: - - name = "maya_thing" - - version = "1.0.0" - - variants = [ - ["maya-2017"], - ["maya-2018"] - ] - - @late() - def build_requires(): - if this.is_package: - return [] - elif this.index == 0: - return ["maya_2017_build_utils"] - else: - return ["maya_2018_build_utils"] - -Note the check for `this.is_package`. This is necessary, otherwise the evaluation would -fail in some circumstances. Specifically, if someone ran the following command, the `this` -field would actually be a `Package` instance, which doesn't have an `index`: - - ]$ rez-search maya_thing --type package --format '{build_requires}' - -In this case, `build_requires` is somewhat nonsensical (there is no common build requirement -for both variants here), but something needs to be returned nonetheless. - -## Sharing Code Across Package Definition Files - -It is possible to share common code across package definition function attributes, but the -mechanism that is used is different depending on whether a function is early binding or late -binding. This is to avoid installed packages being dependent on external code that may change -at any time; builds being dependent on external code is not problematic however. - -### Sharing Code During A Build - -Functions in a *package.py* file which are evaluated at build time include: - -* The *preprocess* function; -* Any package attribute implemented as a function using the *@early* decorator. - -You expose common code to these functions by using the -[package_definition_build_python_paths](Configuring-Rez#package_definition_build_python_paths) -config setting. - -### Sharing Code Across Installed Packages - -Functions that are evaluated in installed packages' definition files include: - -* The various *commands* functions; -* Any package attribute implemented as a function using the *@late* decorator. - -You expose common code to these functions by using the *@include* decorator, which relies on the -[package_definition_python_path](Configuring-Rez#package_definition_python_path) config setting. -The module source files are actually copied into each package's install payload, so the package -stays self-contained, and will not break or change behavior if the original modules' source -files are changed. The downside though, is that these modules are not imported, and they themselves -cannot import other modules managed in the same way. - -Here is an example of a package's *commands* using a shared module: - - # in package.py - @include("utils") - def commands(): - utils.set_common_env_vars(this, env) - -## Requirements Expansion - -Often a package may be compatible with a broader range of its dependencies at build time than it is -at runtime. For example, a C++ package may build against any version of *boost-1*, but may -then need to link to the specific minor version that it was built against, say *boost-1.55*. - -You can describe this in your package's *requires* attribute (or any of the related attributes, -such as *build_requires*) by using wildcards as shown here: - - requires = [ - "boost-1.*" - ] - -If you check the *package.py* of the built package, you will see that the boost reference in the -requires list will be expanded to the latest found within the given range ("boost-1.55" for example). - -There is also a special wilcard available - *"**"*. This expands to the full package version. For -example, the requirement *boost-1.*** might expand to *boost-1.55.1*. - -You can also achieve requirements expansion by implementing *requires* as an early binding -function (and you may want to use some variation of this to generate *variants* for example), and -using the rez *expand_requires* function: - - @early() - def requires(): - from rez.package_py_utils import expand_requires - return expand_requires(["boost-1.*"]) - -## Package Preprocessing - -You can define a *preprocessing* function either globally or in a *package.py*. This can be used to -validate a package, or even change some of its attributes, before it is built. To set a global -preprocessing function, see the -[package_preprocess_function](Configuring-Rez#package_preprocess_function) config setting. - -Consider the following preprocessing function, defined in a *package.py*: - - def preprocess(this, data): - from rez.package_py_utils import InvalidPackageError - import re - - if not re.match("[a-z]+$", this.name): - raise InvalidPackageError("Invalid name, only lowercase letters allowed") - - if not this.authors: - from preprocess_utils import get_git_committers - data["authors"] = get_git_committers() - -This preprocessor checks the package name against a regex; and sets the package authors list to its -git committers, if not already supplied in the *package.py*. To update package attributes, you have -to update the given *data* dict, *not* the *package* instance. - -To halt a build because a package is not valid, you must raise an *InvalidPackageError* as shown -above. - -> [[media/icons/info.png]] To see the preprocessed contents of a package.py, run the command -> *"rez-build --view-pre"* in the source root directory. This will just print the preprocessed -> package to standard out, then exit. - -### Overriding Config Settings In Preprocessing - -It is not uncommon to override config settings such as the release path in a package, like so: - - # in package.py - with scope("config") as c: - c.release_packages_path = "/software/packages/external" - -Let's say we have a scenario where we want to install third party packages to a specific install -path, and that we set the arbitrary attribute *external* to True for these packages. We could do -this with a global preprocessing function like this: - - def preprocess(this, data): - if not data.get("external"): - return - - try: - _ = data["config"]["release_packages_path"] - return # already explicitly specified by package - except KeyError: - pass - - data["config"] = data.get("config", {}) - data["config"]["release_packages_path"] = "/software/packages/external" - -The *"with scope(...)"* statement is just a fancy way of defining a dict, so you can do the same -thing in the preprocess function simply by updating the *config* dict within *data*. - -## Example Package - -Here is an example package definition, demonstrating several features. This is an example of a -python package which, instead of actually installing python, detects the existing system python -installation instead, and binds that into a rez package. - - name = "python" - - @early() - def version(): - return this.__version + "-detected" - - authors = [ - "Guido van Rossum" - ] - - description = \ - """ - The Python programming language. - """ - - @early() - def variants(): - from rez.package_py_utils import expand_requires - requires = ["platform-**", "arch-**", "os-**"] - return [expand_requires(*requires)] - - @early() - def tools(): - version_parts = this.__version.split('.') - - return [ - "2to3", - "pydoc", - "python", - "python%s" % (version_parts[0]), - "python%s.%s" % (version_parts[0], version_parts[1]) - ] - - uuid = "recipes.python" - - def commands(): - env.PATH.append("{this._bin_path}") - - if building: - env.CMAKE_MODULE_PATH.append("{root}/cmake") - - # --- internals - - def _exec_python(attr, src): - import subprocess - - p = subprocess.Popen( - ["python", "-c", src], - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - out, err = p.communicate() - - if p.returncode: - from rez.exceptions import InvalidPackageError - raise InvalidPackageError( - "Error determining package attribute '%s':\n%s" % (attr, err)) - - return out.strip() - - @early() - def _bin_path(): - return this._exec_python( - "_bin_path", - "import sys, os.path; print(os.path.dirname(sys.executable))") - - def _version(): - return _exec_python( - "version", - "import sys; print(sys.version.split()[0])") - - __version = _version() - -Note the following: - -* *variants* is implemented as an early bound attribute, and uses *requirements expansion* to - dynamically define the variant requirements. Even though only the *requires* and related attributes - natively expand wildcards, you can still use the *expand_requirement* function yourself, as - illustrated here. -* A *_version* function has been defined, and its return value stored into the *__version* variable. - This is done because two other early binding attributes - *version* and *tools* - use this value, - and we avoid calling the function twice. Both *_version* and *__version* are later stripped from - the package, because one is a normal function, and the other has double leading underscores. -* An arbitrary attribute *_bin_path* has been defined, and implemented as an early bound attribute. - The *commands* function then uses this value. In this example, it was far better to take this - approach than the alternative - running the python subprocess in the *commands* function. Doing that - would have been very costly, since commands are executed every time a new environment is created - (and launching a subprocess is slow). Instead, here we take this cost at build time, and cache the - result into the package attribute. -* Common code was provided in the normal function *_exec_python*, which will be stripped from the - installed package. - -## Standard Package Attributes - -Following is a list, in alphabetical order, of every standard attribute that a user can define in a -package definition file (you can also define your own arbitrary attributes). Each entry specifies -the data type, and includes a code snippet. - -### authors -*List of string* - - authors = ["jchrist", "sclaus"] - -Package authors. Should be in order, starting with the major contributor. - -### build_requires -*List of string* - - build_requires = [ - "cmake-2.8", - "doxygen" - ] - -This is the same as *requires*, except that these dependencies are only included during a build -(typically invoked using the *rez-build* tool). - -### cachable -*Boolean* - - cachable = True - -Determines whether a package can be cached when [package caching](Package-Caching#Overview) is -enabled. If not provided, this is determined from the global config setting [default_cachable](Configuring-Rez#default_cachable) and related `default_cachable_*` settings. - -### commands -*Function* - - def commands(): - env.PYTHONPATH.append("{root}/python") - env.PATH.append("{root}/bin") - -This is a block of python code which tells rez how to update an environment so that this package -can be used. It is executed when the package is brought into a rez environment, either by explicit -request or by another package's requirements. There is a python API provided (see -[here](Package-Commands) for more details) that lets you do things such as: - -* set, unset, prepend and append environment variables; -* create aliases; -* source scripts; -* print messages. - -In this example, the 'foo' package is appending a path to *PYTHONPATH*, and appending a path to -*PATH*. The special string "{root}" will expand out to the install location of the package. This is -a fairly typical example. - -### config - - with scope("config"): - release_packages_path = "/software/packages/apps" - -Packages are able to override rez configuration settings. This is useful in some cases; for example, -we may want a package to release to a different directory than the default (as this example shows). -See [here](Configuring-Rez) for more details. - -### description -*String* - - description = "Library for communicating with the dead." - -This is a general description of the package. It should not mention details about a particular -version of the package, just about the package in general. - -### has_plugins -*Boolean* - - has_plugins = True - -Indicates that the package is an application that may have plugins. These plugins are often made -available as rez packages also. - -### hashed_variants -*Boolean* - - hashed_variants = True - -Instructs the package to install variants into a subdirectory based on a hash of the variant's -contents (its requirements in other words). This is useful for variants with a high number of -requirements, or with requirements that do not translate well to directories on the filesystem -(such as conflict requirements). - -### help -*String or List of String* - - help = "https://github.com/__GITHUB_REPO__/wiki" - -URL for package webpage, or, if a string containing spaces, a command to run. You can show the help -for a package using the *rez-help* command line tool. If this value is a list, then this represents -multiple help entries, and you can specify the entry you want to see using the `SECTION` argument. - -### name -*String (mandatory)* - - name = "maya_utils" - -This is the name of the package. Alphanumerics and underscores are allowed. Name is case sensitive. - -### plugin_for -*String* - - plugin_for = "maya" - -Provided if this package is a plugin of another package. For example, this might be a maya plugin. - -### post_commands -*Function* - - def post_commands(): - env.FOO_PLUGIN_PATH.append("@") - -Similar to *pre_commands*, but runs in a final phase rather than the first. See that attribute for -further details. - -### pre_commands -*Function* - - def pre_commands(): - import os.path - env.FOO_PLUGIN_PATH = os.path.join(this.root, "plugins") - -This is the same as *commands*, except that all packages' *pre_commands* are executed in a first -pass; then, all *commands* are run in a second; and lastly, *post_commands* are all run in a third -phase. It is sometimes useful to ensure that some of a package's commands are run before, or after -all others, and using pre/post_commands is a way of doing that. - -### pre_test_commands -*Function* - - def pre_test_commands(): - if test.name == "unit": - env.IS_UNIT_TEST = 1 - -This is similar to *commands*, except that it is run prior to each test defined in -[tests](#tests) -See [here](Package-Commands#pre-test-commands) for more details. - -### relocatable -*Boolean* - - relocatable = True - -Determines whether a package can be copied to another package repository (using the `rez-cp` tool for -example). If not provided, this is determined from the global config setting [default_relocatable](Configuring-Rez#default_relocatable) and related `default_relocatable_*` settings. - -### requires -*List of string* - - requires = [ - "python-2", - "maya-2016", - "maya_utils-3.4+<4" - ] - -This is a list of other packages that this package depends on. A rez package should list all the -packages it needs - someone should be able to use your package without needing to know about how it -works internally - and this includes needing to know its dependencies. - -Rez has a syntax for these package requests. For example, "python-2.6" is a package request which -covers the range of all python packages starting with 2.6 - for example, "python-2.6.0", -"python-2.6.4" (it is not simply a prefix - "python-2.65" is not within the request). When you -request a package, you are asking rez for any version within this request, although rez will aim to -give you the latest possible version. - -For more details on request syntax, see [here](Basic-Concepts#package-requests). - -### tests -*Dict* - - tests = { - "unit": "python -m unittest discover -s {root}/python/tests", - "lint": { - "command": "pylint mymodule", - "requires": ["pylint"], - "run_on": ["default", "pre_release"] - }, - "maya_CI": { - "command": "python {root}/ci_tests/maya.py", - "on_variants": { - "type": "requires", - "value": ["maya"] - }, - "run_on": "explicit" - } - } - -This is a dict of tests that can be run on the package using the *rez-test* tool. For example, to -run a linter on the package *maya_utils* with the *test* attribute above, you would simply run: - - ]$ rez-test maya_utils lint - -If a test entry is a string or list of strings, this is interpreted as the command to run. Command -strings will expand any references to package attributes, such as *{root}*. - -If you provide a nested dict, you can specify extra fields per test, as follows: - -* **requires**: Extra package requirements to include in the test's runtime env. -* **run_on**: When to run this test. Valid values are: - * `default` (the default): Run when `rez-test` is run with no `TEST` args specified. - * `pre_install`: Run before an install (ie `rez-build -i`), and abort the install on fail. - * `pre_release`: Run before a release, and abort the release on fail. - * `explicit`: Only run if specified as `TEST` when `rez-test` is run. -* **on_variants**: Which variants the test should be run on. Valid values are: - * True: Run the test on all variants. - * False (the default): Run the test only on one variant (ie the variant you get by - default when the test env is resolved). This is useful for tests like linting, - where variants may be irrelevant. - * A dict. This is a variant selection mechanism. In the example above, the "maya_CI" test will - run only on those variants that directly require `maya` (or a package within this range, eg - `maya-2019`). Note that "requires" is the only filter type currently available. - -### tools -*List of string* - - tools = [ - "houdini", - "hescape", - "hython" - ] - -This is a list of tools that the package provides. This entry is important later on when we talk -about [suites](Suites#suite-tools). - -### uuid -*String* - - uuid = "489ad32867494baab7e5be3e462473c6" - -This string should uniquely identify this *package family* - in other words, all the versions of a -particular package, such as 'maya'. It is used to detect the case where two unrelated packages that -happen to have the same name are attempted to be released. If rez detects a uuid mismatch, it will -abort the release. - -You should set the uuid on a new package once, and not change it from then on. The format of the -string doesn't actually matter, but you'd typically use a true UUID, and you can generate one -like so: - - ]$ python -c 'import uuid; print(uuid.uuid4().hex)' - -### variants -*List of list of string* - - variants = [ - ["maya-2015.3"], - ["maya-2016.1"], - ["maya-2016.7"] - ] - -A package can contain *variants* - think of them as different flavors of the same package version, -but with differing dependencies. See the [variants chapter](Variants) for further details. - -### version -*String* - - version = "1.0.0" - -This is the version of the package. See [here](Basic-Concepts#versions) for further details on valid -package versions. - - -## Build Time Package Attributes - -The following package attributes only appear in packages to be built; they are stripped from the -package once installed because they are only used at build time. - -### build_command -*String or False* - - build_command = "bash {root}/build.sh {install}" - -Package build command. If present, this is used as the build command when *rez-build* is run, -rather than detecting the build system from present build scripts (such as *CMakeLists.txt*). If -*False*, this indicates that no build step is necessary (the package definition will still be -installed, and this is enough to define the package). - -The *{root}* string expands to the root directory of the package (where the package.py is -contained). Note that, like all builds, the working directory is set to the *build path*, which -is typically somewhere under a *build* subdirectory, and is where build outputs should go. - -The *{install}* string expands to "*install*" if an installation is occurring, or the empty string -otherwise. This is useful for passing the install target directly to the command (for example, when -using *make*) rather than relying on a build script checking the *REZ_BUILD_INSTALL* environment -variable. - -The full set of variables that can be referenced in the build command are: - -* *root*: (see above); -* *install*: (see above) -* *build_path*: The build path (this will also be the current working directory); -* *install_path*: Full path to install destination; -* *name*: Name of the package getting built; -* *variant_index*: Index of the current variant getting built, or an empty - string ('') if no variants are present. -* *version*: Package version currently getting built. - -### build_system -*String* - - build_system = "cmake" - -Specify the build system used to build this package. If not set, it is detected automatically when -a build occurs (or the user specifies it using the `--build-system` option). - -### pre_build_commands -*Function* - - def pre_build_commands(): - env.FOO_BUILT_BY_REZ = 1 - -This is similar to *commands*, except that it is run _prior to the current package being built_. -See [here](Package-Commands#pre-build-commands) for more details. - -### preprocess -*Function* - -See [Package Preprocessing](#package-preprocessing) - -### private_build_requires -*List of string* - - private_build_requires = [ - "cmake-2.8", - "doxygen" - ] - -This is the same as *build_requires*, except that these dependencies are only included if this -package is being built. Contrast this with *build_requires*, whose dependencies are included if a -build is occurring - regardless of whether this package specifically is being built, or whether -this package is a dependency of the package being built. - -### requires_rez_version -*String* - - requires_rez_version = "2.10" - -This defines the minimum version of rez needed to build this package. New package features have -been added over time, so older rez versions cannot necessarily build newer packages. - -## Release Time Package Attributes - -The following package attributes are created for you by Rez when your package is released via the -*rez-release* tool. If you look at the released *package.py* file you will notice that some or all -of these attributes have been added. - -### changelog -*String* - - changelog = \ - """ - commit 22abe31541ceebced8d4e209e3f6c44d8d0bea1c - Author: allan johns <> - Date: Sun May 15 15:39:10 2016 -0700 - - first commit - """ - -Change log containing all commits since the last released package. If the previous release was from -a different branch, the changelog given will go back to the last common commit ancestor. The syntax -of this changelog depends on the version control system; the example here is from a *git*-based -package. - -### previous_revision -*Type varies* - -Revision information of the previously released package, if any (see *revision* for code example - -the code for this attribute is the same). - -### previous_version -*String* - - previous_version = "1.0.1" - -The version of the package previously released, if any. - -### release_message -*String* - - release_message = "Fixed the flickering thingo" - -The package release message. This is supplied either via the *rez-release* tool's *--message* -option, or was entered in a text editor on release if rez is configured to do this (see the config -setting 'TODO_ADD_THIS'). A package may not have a release message. - -### revision -*Type varies* - - revision = \ - {'branch': 'main', - 'commit': '22abe31541ceebced8d4e209e3f6c44d8d0bea1c', - 'fetch_url': 'git@github.com:foo/dummy.git', - 'push_url': 'git@github.com:foo/dummy.git', - 'tracking_branch': 'origin/main'} - -Information about the source control revision containing the source code that was released. The -data type is determined by the version control system plugin that was used. The example code shown -here is the revision dict from a *git*-based package. - -### timestamp -*Integer* - - timestamp = 1463350552 - -Epoch time at which the package was released. - -### vcs -*String* - - vcs = "git" - -Name of the version control system this package was released from. diff --git a/wiki/pages/Pip.md b/wiki/pages/Pip.md deleted file mode 100644 index 060fb928d..000000000 --- a/wiki/pages/Pip.md +++ /dev/null @@ -1,132 +0,0 @@ -## Overview - -Rez is language agnostic. -But since python is so much used in the VFX industry (and outside), -it knows how to convert/translate it into a rez package. -To do so, it provides a `rez-pip` command. - -> [[media/icons/warning.png]] It doesn't know how to translate its own packages -into pip packages. - -### Usage - -``` -usage: rez pip [-h] [--python-version VERSION] [--pip-version VERSION] [-i] - [-s] [-r] [-v] - PACKAGE - -Install a pip-compatible python package, and its dependencies, as rez -packages. - -positional arguments: - PACKAGE package to install or archive/url to install from - -optional arguments: - -h, --help show this help message and exit - --python-version VERSION - python version (rez package) to use, default is - latest. Note that the pip package(s) will be installed - with a dependency on python-MAJOR.MINOR. - --pip-version VERSION - pip version (rez package) to use, default is latest. - This option is deprecated and will be removed in the - future. - -i, --install install the package - -s, --search search for the package on PyPi - -r, --release install as released package; if not set, package is - installed locally only - -p PATH, --prefix PATH - install to a custom package repository path. - -v, --verbose verbose mode, repeat for more verbosity -``` - -The `rez-pip` command allows you to do two main things. - -1. Install or release a pip package as a rez package. -2. Search for a package on PyPI - -### Which pip will be used? - -`rez-pip` uses a fallback mechanism to find which pip will be used to run the commands. -The logic is as follow: - -1. Search for pip in the rezified `python` package specified with `--python-version`, or -the latest version if not specified; -2. If found, use it; -3. If not found, search for pip in the rezified `pip` package specified with `--pip-version`, -or latest version if not specified. - - **Note that this is deprecated and will be removed in the future**; - -4. If rezified `pip` is found, use it; -5. If not found, fall back to pip installed in rez own virtualenv. - -> [[media/icons/info.png]] In all of these options, we also check if the version of pip is greater -or equal than 19.0. This is a hard requirement of `rez-pip`. - -Note that rez-pip output should tell you what it tries and which pip it will use. - -It is extremely important to know which pip is used and understand why it is used. Pip packages -define which python version they are compatible with. -When you install a pip package, pip checks which python version it is -currently running under to determine if a package can be downloaded and installed. -Not only this but it also checks which python implementation is used (CPython, PyPy, -IronPython, etc), the architecture python was built with, and other variables. So the thing you -really need to know first is which python you want to use and from there you should know -which pip is used. Knowing the pip version comes in second place. - -At some point, we supported the `--pip-version` argument, but considering what was just said -above, we decided to deprecate it (but not yet removed) just for backward compatibility reasons. -Pip is too much (read tightly) coupled to the python version/interpreter it is installed with -for us to support having pip as a rez package. We just can't garantee that pip can be -install once in a central way and work with multiple different python version, and potentially -different implementations. - -### How should I install pip? - -Following the [Which pip will be used?](#which-pip-will-be-used) section, we recommend to install -pip inside your python packages. For Python 2, this can be done when you compile it with the -`--with-ensurepip` flag of the `configure` script. This will install a version older than 19.0 -though, so you will need to upgrade it. For Python 3, it is already installed by default. -Though your milleage may vary for the version installed, depending on which Python version you -installed. So check the pip version and update it if necessary. We also encourage you -to install `wheel` and possibly update `setuptools`. `pip`, `setuptools` and `wheel` -are perfectly fine when installed in the interpreter directly as they are pretty core -packages and all have no dependencies (and that's what `virtualenv` does by default too). - -Tip: When installing something in an interpreter, make sure you really install in this interpreter. -That means using somehting like: - -``` -/path/to/python -E -s -m pip install -``` - -`-E` will render any `PYTHON*` environment variable to not be used and `-s` will -remove your [user site](https://docs.python.org/3.7/library/site.html) from the equation. - -#### Install/release - -You have two options when you want to convert a pip package to a rez package. You can -install it, or release it. Install means that it will install in your -[local_packages_path](Configuring-Rez#local_packages_path), while -release means it will be installed in your [release_packages_path](Configuring-Rez#release_packages_path). -You can also specify a custom installation location using `--prefix` (or `-p`). - - -You can (and we recommend) use the `--python-version` to choose for which python -version you want to install a given package. This will make `rez-pip` to resolve -the given version of the `python` rez package and use it to run the `pip install`. -See [Which pip will be used?](#which-pip-will-be-used) for more details. -If the pip package is not pure (so contains `.so` for example), you will need to -call `rez-pip` for each python version you want to install the pip package for. - -> [[media/icons/warning.png]] `--pip-version` is deprecated and will be removed in the future. -> See [How should I install pip?](#how-should-i-install-pip) on how we recommend -> to install pip from now on. - - -#### Search - -With `rez-pip --search `, you can search for a package in PyPI. The main -advantage of using it over using `pip search ` is that `rez-pip --search` -uses the same logic as `rez-pip --install` and `--release` to find which pip to use. diff --git a/wiki/pages/Suites.md b/wiki/pages/Suites.md deleted file mode 100644 index fee142441..000000000 --- a/wiki/pages/Suites.md +++ /dev/null @@ -1,150 +0,0 @@ -## Overview - -Let us say that you wish to provide a number of different tools to your users, -even though these tools may require being run in different environments. For -example, you may want artists at your studio to be able to run *maya* and *nuke* -from the command line, without needing to know that they execute within different -environments. - -Let's say that in order to do this, you create two *contexts* - *maya.rxt* and -*nuke.rxt* (see [here](Contexts#baking-resolves) for more information). In -order to run maya, you would do this: - - ]$ rez-env --input maya.rxt -- maya - -You may then decide to wrap this command within a wrapper script which is also -called *maya*, and that might look something like this: - - #!/bin/bash - rez-env --input maya.rxt -- maya $* - -Now, if you put this somewhere on *$PATH*, and do the same for *nuke*, then -voila, your artists can run these applications from the command line, without -needing to know what's happening under the hood. - -This, in a nutshell, is what a *suite* does. A suite is simply a directory -containing a set of contexts, and wrapper scripts which run tools within those -contexts. - -## The rez-suite Tool - -Let's go through the same example, this time using the *rez-suite* tool. - -First, we create the suite. This creates a directory called *mysuite* in the -current working directory: - - ]$ rez-suite --create mysuite - -Now we need to add contexts to our suite. First we create the contexts: - - ]$ rez-env maya-2016.2 --output maya.rxt - ]$ rez-env nuke --output nuke.rxt - -Then, we add these contexts to the suite (note that the *--context* arg just -gives each context a label - you would typically have this match the context -filename as shown). - - ]$ rez-suite --add maya.rxt --context maya mysuite - ]$ rez-suite --add nuke.rxt --context nuke mysuite - -The suite is created! Now all we need to do is to activate it, and that's as -simple as adding its *bin* path to *$PATH*: - - ]$ export PATH=$(pwd)/mysuite/bin:$PATH - -You should now see your tools coming from the suite: - - ]$ which maya - ./mysuite/bin/maya - - ]$ ls ./mysuite/bin - maya - nuke - -## Suite Tools - -The tools in a context which are exposed by the suite is determined by the -[tools](Package-Definition-Guide#tools) package attribute. For example, the -*maya* package might have a *tools* definition like so: - - # in maya package.py - tools = [ - "maya", - "mayapy", - "fcheck" - ] - -All these tools would be made available in the suite (although you can explicitly -hide tools - see the *rez-suite* *--hide* option). - -> [[media/icons/warning.png]] Only packages listed in the context *requests*, -> that are not weak or conflict requests, have their tools exposed - packages -> pulled in as dependencies do not. If you need to control the version of a package -> not in the request, without adding its command line tools, just add it as a weak -> reference to the request list. - -### Tool Aliasing - -Tools can be aliased to different names, either explicitly (on a per-tool basis), -or by applying a common prefix or suffix to all tools in a context. - -Prefixing/suffixing is particularly useful when you want to expose the same -package's tools, but in two or more contexts. For example, you may want to run a -stable version of maya, but also a newer beta version. These would run in -different contexts, and the beta context might prefix all tools with *_beta*, -hence making available tools such as *maya_beta*. - -For example, here we create a context with a newer version of maya, add it to -the suite, then add a suffix to all its tools: - - ]$ rez-env maya-2017 --output maya2017.rxt - ]$ rez-suite --add maya2017.rxt --context maya2017 mysuite - ]$ rez-suite --suffix _beta --context maya2017 mysuite - -### Control Arguments - -When using suite tools, any arguments passed to the wrappers are passed through -to the underlying tool, as expected. However, there is an exception to the case - -rez provides a set of *control* arguments, which are prefixed with `+`/`++` -rather than the typical `-`/`--`. These are suite-aware arguments that pass -directly to rez. You can get a listing of them using `+h`/`++help`, like so: - -``` -]$ maya ++help -usage: maya [+h] [+a] [+i] [+p [PKG [PKG ...]]] [++versions] - [++command COMMAND [ARG ...]] [++stdin] [++strict] [++nl] - [++peek] [++verbose] [++quiet] [++no-rez-args] - -optional arguments: - +h, ++help show this help message and exit - +a, ++about print information about the tool - +i, ++interactive launch an interactive shell within the tool's - configured environment - +p [PKG [PKG ...]], ++patch [PKG [PKG ...]] - run the tool in a patched environment - ++versions list versions of package providing this tool - ++command COMMAND [ARG ...] - read commands from string, rather than executing the - tool - ++stdin read commands from standard input, rather than - executing the tool - ++strict strict patching. Ignored if ++patch is not present - ++nl, ++no-local don't load local packages when patching - ++peek diff against the tool's context and a re-resolved copy - - this shows how 'stale' the context is - ++verbose verbose mode, repeat for more verbosity - ++quiet hide welcome message when entering interactive mode - ++no-rez-args pass all args to the tool, even if they start with '+' -``` - -For example, to see information about the suite wrapper: - - ]$ maya ++about - Tool: maya - Path: ./mysuite/bin/maya - Suite: ./mysuite - Context: ./mysuite/contexts/maya2016.rxt ('maya2016') - -> [[media/icons/info.png]] If the target tool also uses `+` for some of its -> own arguments, you can change the prefix character that rez uses for its -> control arguments. See the *rez-suite* *--prefix-char* option. diff --git a/wiki/pages/Variants.md b/wiki/pages/Variants.md deleted file mode 100644 index 7c2708708..000000000 --- a/wiki/pages/Variants.md +++ /dev/null @@ -1,226 +0,0 @@ -## Overview - -Packages in rez can contain different *variants*. Think of these as different -flavors of the same package version. Each variant has one or more package -dependencies that differ to the other variants in the same package. - -Use of variants is best illustrated with an example. Consider a maya plugin, -*my_maya_plugin*. Let us assume that there are two active versions of maya -currently in use at your studio - *2016.sp2* and *2017*. If your plugin is compiled, -you may need to build it separately for each maya version, even though the source -is no different. - -You would use *variants* to create a version of your plugin that will work with -both maya versions. Consider the following package definition for our plugin: - - name = "my_maya_plugin" - - version = "1.0.0" - - requires = [ - "openexr-2.2" - ] - - variants = [ - ["maya-2016.sp2"], - ["maya-2017"] - ] - -When you build and install this package, two separate builds will occur - one -using *maya-2016.sp2*, and the other *maya-2017*. When an environment is resolved -that includes *my_maya_plugin*, the correct variant will be selected depending on -the version of maya present. Only one variant of a package is ever used in a -given configured environment. - -Each variant entry is a list of dependencies, no different to the packages listed -in the *requires* field. These dependencies are appended to the *requires* list -for each variant. Thus the first variant requires *openexr-2.2* and *maya-2016.sp1*, -and the second variant requires *openexr-2.2* and *maya-2017*. - -## Disk Structure - -Package variants are stored within the package, under subdirectories that match -the variant requirements. For example, continuing on with our *my_maya_plugin* -package, the installation of that package would look like so: - - /rez/packages/my_maya_plugin/1.0.0/maya-2016.sp2/ - /maya-2017/ - -The anatomy of a package with variants is illustrated in the following diagram: - -

- - -

- -The *root* of a package is the root directory of its current variant (the one -the current environment is configured to use); the *base* of a package is the -directory containing its variants. In a package that does not have variants, -*base* and *root* are the same. - -## Hashed Variants - -There are two problems with the variant subpath as illustrated above: -* The variant install path can become long if there are many requirements; -* If some variant requirements contain characters such as `!` and `<`, they - can cause escaping problems that affect build systems; and, depending on the - platform, may not be a valid filesystem path. - -You can avoid these issues by using _hashed variants_. This sets the variant -subpath to a hash of its requirements, rather than the requirements themselves. -The resulting subdirectly is somewhat unwieldy (example: -`83e0c415db1b602f9d59cee028da6ac785e9bacc`). However, another feature - -_variant shortlinks_ - deals with this. A shortlink is a symlink to each variant, -created in a separate subdirectory (default `_v`). - -Here is an example hashed variant path: - - /rez/packages/my_maya_plugin/1.0.0/83e0c415db1b602f9d59cee028da6ac785e9bacc - -Here is the matching _shortlink_, which is what will be used in a resolved -environment: - - /rez/packages/my_maya_plugin/1.0.0/_v/a - -Hashed variants must be enabled explicitly for a package. To do this, simply set -this in your package definition: - - hashed_variants = True - - -## Platform As Variant - -It is not uncommon to see the platform, architecture and/or operating system -packages in package variants (recall that rez represents these as packages). For -example, you might see variants like this: - - # in package.py - variants = [ - ["platform-linux", "arch-x86_64", "os-Ubuntu-12.04"], - ["platform-linux", "arch-x86_64", "os-Ubuntu-16.04"] - ] - -This indicates that the package has been built for multiple platforms. The correct -variant will be selected for you, because you probably have the relevant -[implicit packages](Basic-Concepts#implicit-packages) set to limit packages to -the current platform. - -## Single Variants - -You may often see packages with just one variant. There are two reasons for this: - -* *Future proofing*. Let's say you have a compiled package that links against python. - It may currently support *python-2.7*, however it's conceivable that support for - newer python versions may be added later. It is not possible to add new variants - to a package that does not have any; so by adding the *python-2.7* variant now, - you can add variants later without needing to move to a newer version. -* *Installation path*. People often expect to see platform, architecture and/or - operating system information in the installation path of a piece of software (and - may also expect the same of python version, or other core packages). By putting - these dependencies into a variant, we ensure that they appear in the installation - path of the package. - -## Variant Selection - -As mentioned, rez will automatically select the correct variant of a package -depending on the environment being resolved. For example, consider: - - ]$ rez-env my_maya_plugin maya-2017 -- echo '$REZ_MY_MAYA_PLUGIN_ROOT' - /rez/packages/my_maya_plugin/1.0.0/maya-2017 - ]$ rez-env my_maya_plugin maya-2016 -- echo '$REZ_MY_MAYA_PLUGIN_ROOT' - /rez/packages/my_maya_plugin/1.0.0/maya-2016.sp2 - -You can see how the correct variant (ie the one that does not conflict with other -packages in the request) has been selected. But what if both variants are valid -for the given request? Consider: - - ]$ rez-env my_maya_plugin -- echo '$REZ_MY_MAYA_PLUGIN_ROOT' - /rez/packages/my_maya_plugin/1.0.0/maya-2017 - -Here *maya* was not in the request. Either variant of *my_maya_plugin* would have -satisfied the request, since we have not specified which version of maya we actually -want. - -By default, rez will prefer the variant with the higher-versioned packages, which -is why the *maya-2017* variant was selected in this example. If there are -multiple packages in the variant, priority is given to those that were in the request -list, if any; following that, priority is given to packages listed earlier in the -variant. For example, consider: - - name = "foo" - - variants = [ - ["python-2.6", "maya-2017"], - ["python-2.7", "maya-2016"] - ] - -If I run "rez-env foo", which variant will I get? The answer is not clear. In this -case it will be the second variant, since *python* is given priority (it is the -first listed package in the variant), and the second variant has the higher version -of python. However, if I ran "rez-env foo maya", I would get the *first* variant - -because priority is now given to *maya*, because it's listed in my request, and the -first variant has the higher version of maya. - -The rez setting -[variant_select_mode](Configuring-Rez#variant_select_mode) affects this selection behavior. -The default mode just described is *version_priority*, but there is another mode - -*intersection_priority*. In this mode, variants are preferred that have *the most -number of packages present in the request*; version priority is secondary. - -### Mutual Exclusivity - -In all the examples we've seen so far, a package's variants have been mutually -exclusive. For example, you cannot have both *python-2.6* and *python-2.7* in the -same environment, so when we request "foo python-2.6" we can be sure of which -variant we will get. - -Variants, however, do not need to be mutually exclusive. In fact, you may use -variants in order to provide support for different DCCs for your package. Consider -a package with the following variants: - - name = "geocache" - - variants = [ - ["maya-2016"], - ["houdini-14"] - ] - -Which variant will I get if I run "rez-env geocache"? Behavior in this case is -undefined - rez gives no guarantees as to which variant will be selected. We cannot -meaningfully compare version numbers across packages, so maya will not have preference -simply because 2016 > 14. However, *version_priority* mode does give priority to -packages listed in the request. So if we ran "rez-env geocache maya", we will get -the first variant... probably. - -#### Probably? - -The operative word here is *preference*. Because the variants are not mutually -exclusive, we can't make guarantees. A resolve is still deterministic - you aren't -going to get differing results when requesting the same environment - but predicting -which variant you'll get can be tricky. - -Consider the following request: - - ]$ rez-env geocache maya animtools-1.4 - -We would expect to get the *maya-2016* variant of *geocache*. However, what if -*animtools* requires *maya-2017*? This makes the first *geocache* variant impossible -to select, since a conflict would occur, and so the *houdini* variant of *geocache* -will be selected - quite possibly not what you expected. - -> [[media/icons/under_construction.png]] I plan on adding a new package request -> syntax, that is able to explicitly select package variants. This will avoid the -> ambiguity in cases like the one described here. - -## Why Use Variants? - -Variants are a powerful mechanism in rez. They avoid the need to maintain separate -branches of a package in order to support varying dependencies. You may have had -problems in the past where a common library depends on, say, boost, and is used in -various DCCs (maya, nuke etc), and depended on by many other packages in your -pipeline. When a DCC moves to a new version of boost (or python, or OIIO, etc) -you now have to branch this library, which potentially affects many other packages. -The problem gets worse if you have multiple dependencies with varying versions. -Variants solve that problem - you simply add another boost variant to your library, -and other dependent packages are not affected. Rez will correctly select the -package variant that does not conflict with the resolved environment. diff --git a/wiki/pages/_Footer.md b/wiki/pages/_Footer.md deleted file mode 100644 index 94ede22fb..000000000 --- a/wiki/pages/_Footer.md +++ /dev/null @@ -1,4 +0,0 @@ -[![Google Group](https://img.shields.io/badge/rez--config-Google%20Group-blue?style=flat-square&logo=google)](https://groups.google.com/forum/#!forum/rez-config) -[![Contributing Guidelines](https://img.shields.io/badge/rez-Contributing%20Guidelines-0b610e?style=flat-square&logo=github)](https://github.com/__GITHUB_REPO__/blob/main/CONTRIBUTING.md) -[![Report A Bug](https://img.shields.io/badge/rez-Report%20A%20Bug-critical?style=flat-square&logo=github)](https://github.com/__GITHUB_REPO__/issues/new) -[![Slack](https://img.shields.io/badge/slack-rez--talk-7a6800?style=flat-square&logo=slack)](https://rez-talk.slack.com/) diff --git a/wiki/pages/_Sidebar.md b/wiki/pages/_Sidebar.md deleted file mode 100644 index 1c9499085..000000000 --- a/wiki/pages/_Sidebar.md +++ /dev/null @@ -1,47 +0,0 @@ -- [[Home]] - -:beginner: Introduction: - -- [[Installation]] and [[Pip]] -- [[Getting Started]] -- [[Basic Concepts]] -- [[Configuring Rez]] - -:memo: `package.py`: - -- [[Package Commands]] -- [[Package Definition Guide]] -- [[Variants]] -- [[Ephemeral Packages]] - -:rocket: rez: - -- [[Contexts]] -- [[Bundles]] -- [[Suites]] -- [[Building Packages]] -- [[Managing Packages]] -- [[Environment Variables]] -- [[Command Line Tools]] - -:information_source: Others: - -- [API Reference](https://__GITHUB_USER__.github.io/__REPO_NAME__/) -- [[Glossary]] -- [[FAQ]] -- [[Notes]] -- [[Credits]] - -
:construction: Unwritten pages: - -- [[Advanced Topics]] -- [[Caching]] -- [[Releasing Packages]] -- [[Package Filters]] -- [[Testing Packages]] -- [[Rez GUI]] -- [[The Resolve Graph]] -- [[Timestamping]] -- [[Troubleshooting]] - -