From 3d618588341f67692a29fd35aa8be5a2783dd011 Mon Sep 17 00:00:00 2001 From: Yury-Fridlyand Date: Thu, 19 Jan 2023 10:44:13 -0800 Subject: [PATCH 1/7] Update python versions used in CI. Signed-off-by: Yury-Fridlyand --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2f78015..7721d19 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -61,7 +61,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9] + python-version: [2.7, 3.7, 3.8, 3.9, 3.10, 3.11] steps: - name: Checkout Repository From 5ab3a83240273f02e46320af371ec6b729fb798c Mon Sep 17 00:00:00 2001 From: Yury-Fridlyand Date: Thu, 19 Jan 2023 11:32:31 -0800 Subject: [PATCH 2/7] Add quotes to avoid 3.10 be parsed as 3.1. Signed-off-by: Yury-Fridlyand --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7721d19..9760242 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -61,7 +61,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [2.7, 3.7, 3.8, 3.9, 3.10, 3.11] + python-version: ['2.7', '3.7', '3.8', '3.9', '3.10', '3.11'] steps: - name: Checkout Repository From f2370e02b9f1b459b17edbcbace95bd875837f5c Mon Sep 17 00:00:00 2001 From: Yury-Fridlyand Date: Thu, 19 Jan 2023 11:40:38 -0800 Subject: [PATCH 3/7] Update changelog. Signed-off-by: Yury-Fridlyand --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d4992e4..f28f4cc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) ### Removed ### Fixed +- Update CI workflow python versions ([#92](https://github.com/opensearch-project/opensearch-dsl-py/pull/92)) ### Security From 7d3c035d9ece1f573b047d21bfa5860673f52fc3 Mon Sep 17 00:00:00 2001 From: Yury-Fridlyand Date: Tue, 31 Jan 2023 20:17:59 -0800 Subject: [PATCH 4/7] Update CI workflows to cover more python and opensearch versions. Signed-off-by: Yury-Fridlyand --- .github/workflows/ci.yml | 29 ++-- .github/workflows/integration-unreleased.yml | 164 ++++++++++++++++--- .github/workflows/integration.yml | 86 +++++++--- 3 files changed, 230 insertions(+), 49 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9760242..9ddbbf8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,9 +8,9 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout Repository - uses: actions/checkout@v1 + uses: actions/checkout@v3 - name: Set up Python 3.8 - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: 3.8 - name: Install dependencies @@ -28,9 +28,9 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout Repository - uses: actions/checkout@v1 + uses: actions/checkout@v3 - name: Set up Python 3.8 - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: 3.8 - name: Install dependencies @@ -43,9 +43,9 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout Repository - uses: actions/checkout@v1 + uses: actions/checkout@v3 - name: Set up Python 3.7 - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: 3.7 - name: Install build tools @@ -56,23 +56,32 @@ jobs: python3.7 -m build test-linux: - runs-on: ubuntu-latest + runs-on: ${{ matrix.runner }} strategy: fail-fast: false matrix: python-version: ['2.7', '3.7', '3.8', '3.9', '3.10', '3.11'] + runner: ['ubuntu-latest'] + include: + - runner: 'ubuntu-20.04' + python-version: '3.5' + - runner: 'ubuntu-20.04' + python-version: '3.6' steps: - name: Checkout Repository - uses: actions/checkout@v1 + uses: actions/checkout@v3 - name: Setup Python - ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} + env: + PIP_NO_PYTHON_VERSION_WARNING: 1 + PIP_DISABLE_PIP_VERSION_CHECK: 1 - name: Set up Python 3.8 for Nox if: matrix.python-version != '3.8' - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: 3.8 - name: Install nox diff --git a/.github/workflows/integration-unreleased.yml b/.github/workflows/integration-unreleased.yml index c52989e..c26b54c 100644 --- a/.github/workflows/integration-unreleased.yml +++ b/.github/workflows/integration-unreleased.yml @@ -2,49 +2,173 @@ name: Integration with Unreleased OpenSearch on: push: - branches: - - "main" pull_request: branches: - "main" jobs: - test: + test-ubuntu-22: runs-on: ubuntu-latest strategy: fail-fast: false matrix: - entry: - - { opensearch_ref: '1.x' } - - { opensearch_ref: '2.x' } - - { opensearch_ref: '2.0' } - - { opensearch_ref: 'main' } + opensearch_ref: [ '1.x', '2.x', '2.0', 'main' ] + python-version: [ '2.7', '3.7', '3.8', '3.9', '3.10', '3.11' ] + steps: - name: Checkout OpenSearch - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: repository: opensearch-project/opensearch - ref: ${{ matrix.entry.opensearch_ref }} + ref: ${{ matrix.opensearch_ref }} path: opensearch + - name: Get OpenSearch branch top + id: get-key + working-directory: opensearch + run: echo key=`git log -1 --format='%H'` >> $GITHUB_OUTPUT + + - name: Restore cached build + id: cache-restore + uses: actions/cache/restore@v3 + with: + path: opensearch/distribution/archives/linux-tar/build/distributions + key: ${{ steps.get-key.outputs.key }} + - name: Assemble OpenSearch + if: steps.cache-restore.outputs.cache-hit != 'true' + working-directory: opensearch + run: ./gradlew :distribution:archives:linux-tar:assemble + + - name: Save cached build + if: steps.cache-restore.outputs.cache-hit != 'true' + uses: actions/cache/save@v3 + with: + path: opensearch/distribution/archives/linux-tar/build/distributions + key: ${{ steps.get-key.outputs.key }} + + - name: Run OpenSearch + working-directory: opensearch/distribution/archives/linux-tar/build/distributions + run: | + pwd + tree + tar xf opensearch-min-* + ./opensearch-*/bin/opensearch & + for attempt in {1..20}; do sleep 5; if curl -s localhost:9200; then echo '=====> ready'; break; fi; echo '=====> waiting...'; done + + - name: Checkout High Level Python Client + uses: actions/checkout@v3 + with: + path: dsl-py + + - name: Setup Python - ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + env: + PIP_NO_PYTHON_VERSION_WARNING: 1 + + - name: Set up Python 3.8 for Nox + if: matrix.python-version != '3.8' + uses: actions/setup-python@v4 + with: + python-version: 3.8 + - name: Install nox run: | - cd opensearch - ./gradlew assemble + python3.8 -m pip install nox - # This step runs the docker image generated during gradle assemble in OpenSearch. It is tagged as opensearch:test. - # Reference: https://github.com/opensearch-project/OpenSearch/blob/2.0/distribution/docker/build.gradle#L190 - - name: Run Docker Image + - name: Run integration tests + working-directory: dsl-py run: | - docker run -p 9200:9200 -p 9600:9600 -d -e "discovery.type=single-node" -e "bootstrap.memory_lock=true" opensearch:test - sleep 90 + nox --no-error-on-missing-interpreter -rs lint test + + - name: Save server logs + if: failed() + uses: actions/upload-artifact@v3 + with: + name: opensearch-logs-${{ matrix.opensearch_ref }}-python-${{ matrix.python-version }} + path: | + opensearch/distribution/archives/linux-tar/build/distributions/**/logs/* + + test-ubuntu-20: + runs-on: ubuntu-20.04 + strategy: + fail-fast: false + matrix: + opensearch_ref: [ '1.x', '2.x', '2.0', 'main' ] + python-version: [ '3.5', '3.6' ] + + steps: + - name: Checkout OpenSearch + uses: actions/checkout@v3 + with: + repository: opensearch-project/opensearch + ref: ${{ matrix.opensearch_ref }} + path: opensearch + + - name: Get OpenSearch branch top + id: get-key + working-directory: opensearch + run: echo key=`git log -1 --format='%H'` >> $GITHUB_OUTPUT + + - name: Restore cached build + id: cache-restore + uses: actions/cache/restore@v3 + with: + path: opensearch/distribution/archives/linux-tar/build/distributions + key: ${{ steps.get-key.outputs.key }} + + - name: Assemble OpenSearch + if: steps.cache-restore.outputs.cache-hit != 'true' + working-directory: opensearch + run: ./gradlew :distribution:archives:linux-tar:assemble + + - name: Save cached build + if: steps.cache-restore.outputs.cache-hit != 'true' + uses: actions/cache/save@v3 + with: + path: opensearch/distribution/archives/linux-tar/build/distributions + key: ${{ steps.get-key.outputs.key }} + + - name: Run OpenSearch + working-directory: opensearch/distribution/archives/linux-tar/build/distributions + run: | + pwd + tree + tar xf opensearch-min-* + ./opensearch-*/bin/opensearch & + for attempt in {1..20}; do sleep 5; if curl -s localhost:9200; then echo '=====> ready'; break; fi; echo '=====> waiting...'; done - name: Checkout High Level Python Client - uses: actions/checkout@v2 + uses: actions/checkout@v3 + with: + path: dsl-py - - name: Install Nox - run: pip install nox + - name: Setup Python - ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + env: + PIP_NO_PYTHON_VERSION_WARNING: 1 + + - name: Set up Python 3.8 for Nox + if: matrix.python-version != '3.8' + uses: actions/setup-python@v4 + with: + python-version: 3.8 + - name: Install nox + run: | + python3.8 -m pip install nox - name: Run integration tests + working-directory: dsl-py run: | nox --no-error-on-missing-interpreter -rs lint test + + - name: Save server logs + if: failed() + uses: actions/upload-artifact@v3 + with: + name: opensearch-logs-${{ matrix.opensearch_ref }}-python-${{ matrix.python-version }} + path: | + opensearch/distribution/archives/linux-tar/build/distributions/**/logs/* diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 728333b..0c758ee 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -9,26 +9,13 @@ jobs: strategy: fail-fast: false matrix: - entry: - - { version: 1.0.0 } - - { version: 1.0.1 } - - { version: 1.1.0 } - - { version: 1.2.0 } - - { version: 1.2.1 } - - { version: 1.2.2 } - - { version: 1.2.3 } - - { version: 1.2.4 } - - { version: 1.3.0 } - - { version: 1.3.1 } - - { version: 1.3.2 } - - { version: 1.3.3 } - - { version: 2.0.0 } - - { version: 2.0.1 } - secured: ["true", "false"] + version: [ '1.0.1', '1.1.0', '1.2.4', '1.3.7', '2.0.1', '2.1.0', '2.2.1', '2.3.0', '2.4.0', '2.5.0' ] + secured: [ "true", "false" ] + python-version: [ '2.7', '3.7', '3.8', '3.9', '3.10', '3.11' ] steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Clean docker containers run: | @@ -43,8 +30,69 @@ jobs: docker-compose --project-directory .ci/opensearch build docker-compose --project-directory .ci/opensearch up -d - - name: Install Nox - run: pip install nox + - name: Setup Python - ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + env: + PIP_DISABLE_PIP_VERSION_CHECK: 1 + PIP_NO_PYTHON_VERSION_WARNING: 1 + + - name: Set up Python 3.8 for Nox + if: matrix.python-version != '3.8' + uses: actions/setup-python@v4 + with: + python-version: 3.8 + - name: Install nox + run: | + python3.8 -m pip install --upgrade pip nox + + - name: Integ OpenSearch secured=${{ matrix.secured }} + run: | + export SECURE_INTEGRATION=${{ matrix.secured }} + nox --no-error-on-missing-interpreter -rs lint test + + integration-ubuntu-20: + name: Integ + runs-on: ubuntu-20.04 + strategy: + fail-fast: false + matrix: + version: [ '1.0.1', '1.1.0', '1.2.4', '1.3.7', '2.0.1', '2.1.0', '2.2.1', '2.3.0', '2.4.0', '2.5.0' ] + secured: [ "true", "false" ] + python-version: [ '3.5', '3.6' ] + + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Clean docker containers + run: | + docker volume prune --force + docker network prune --force + docker system prune --volumes --force + + - name: Launch OpenSearch cluster + run: | + export OPENSEARCH_VERSION=${{ matrix.entry.version }} + export SECURE_INTEGRATION=${{ matrix.secured }} + docker-compose --project-directory .ci/opensearch build + docker-compose --project-directory .ci/opensearch up -d + + - name: Setup Python - ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + env: + PIP_DISABLE_PIP_VERSION_CHECK: 1 + + - name: Set up Python 3.8 for Nox + uses: actions/setup-python@v4 + with: + python-version: 3.8 + - name: Install nox + run: | + python3.8 -m pip install --upgrade pip nox - name: Integ OpenSearch secured=${{ matrix.secured }} run: | From 46d993485fd3612dd7075644e7e0d7dbb7742142 Mon Sep 17 00:00:00 2001 From: Yury-Fridlyand Date: Tue, 31 Jan 2023 20:35:24 -0800 Subject: [PATCH 5/7] Fix files according to `lint`/`black` suggestion. Signed-off-by: Yury-Fridlyand --- opensearch_dsl/index.py | 1 - opensearch_dsl/update_by_query.py | 1 - 2 files changed, 2 deletions(-) diff --git a/opensearch_dsl/index.py b/opensearch_dsl/index.py index 610696e..7bcefd2 100644 --- a/opensearch_dsl/index.py +++ b/opensearch_dsl/index.py @@ -59,7 +59,6 @@ def to_dict(self): return d def save(self, using=None): - opensearch = get_connection(using or self._index._using) return opensearch.indices.put_template( name=self._template_name, body=self.to_dict() diff --git a/opensearch_dsl/update_by_query.py b/opensearch_dsl/update_by_query.py index 3d1005d..797bcef 100644 --- a/opensearch_dsl/update_by_query.py +++ b/opensearch_dsl/update_by_query.py @@ -32,7 +32,6 @@ class UpdateByQuery(Request): - query = ProxyDescriptor("query") def __init__(self, **kwargs): From 2d250f1eb4fd3377ba6c43c0fcd85922031eb4ee Mon Sep 17 00:00:00 2001 From: Yury-Fridlyand Date: Tue, 31 Jan 2023 20:35:57 -0800 Subject: [PATCH 6/7] Typo fix. Signed-off-by: Yury-Fridlyand --- .github/workflows/integration-unreleased.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration-unreleased.yml b/.github/workflows/integration-unreleased.yml index c26b54c..60a2e35 100644 --- a/.github/workflows/integration-unreleased.yml +++ b/.github/workflows/integration-unreleased.yml @@ -83,7 +83,7 @@ jobs: nox --no-error-on-missing-interpreter -rs lint test - name: Save server logs - if: failed() + if: failure() uses: actions/upload-artifact@v3 with: name: opensearch-logs-${{ matrix.opensearch_ref }}-python-${{ matrix.python-version }} @@ -166,7 +166,7 @@ jobs: nox --no-error-on-missing-interpreter -rs lint test - name: Save server logs - if: failed() + if: failure() uses: actions/upload-artifact@v3 with: name: opensearch-logs-${{ matrix.opensearch_ref }}-python-${{ matrix.python-version }} From bf484a945914117b515fd30168e4d4658da31aa4 Mon Sep 17 00:00:00 2001 From: Yury-Fridlyand Date: Mon, 13 Feb 2023 13:19:59 -0800 Subject: [PATCH 7/7] Remove multiple python versions from integration tests. Signed-off-by: Yury-Fridlyand --- .github/workflows/integration-unreleased.yml | 96 +------------------- .github/workflows/integration.yml | 56 +----------- 2 files changed, 5 insertions(+), 147 deletions(-) diff --git a/.github/workflows/integration-unreleased.yml b/.github/workflows/integration-unreleased.yml index 60a2e35..2af6610 100644 --- a/.github/workflows/integration-unreleased.yml +++ b/.github/workflows/integration-unreleased.yml @@ -7,13 +7,13 @@ on: - "main" jobs: - test-ubuntu-22: + integ-test: runs-on: ubuntu-latest strategy: fail-fast: false matrix: opensearch_ref: [ '1.x', '2.x', '2.0', 'main' ] - python-version: [ '2.7', '3.7', '3.8', '3.9', '3.10', '3.11' ] + python-version: [ '3.8' ] steps: - name: Checkout OpenSearch @@ -50,8 +50,6 @@ jobs: - name: Run OpenSearch working-directory: opensearch/distribution/archives/linux-tar/build/distributions run: | - pwd - tree tar xf opensearch-min-* ./opensearch-*/bin/opensearch & for attempt in {1..20}; do sleep 5; if curl -s localhost:9200; then echo '=====> ready'; break; fi; echo '=====> waiting...'; done @@ -68,97 +66,9 @@ jobs: env: PIP_NO_PYTHON_VERSION_WARNING: 1 - - name: Set up Python 3.8 for Nox - if: matrix.python-version != '3.8' - uses: actions/setup-python@v4 - with: - python-version: 3.8 - - name: Install nox - run: | - python3.8 -m pip install nox - - - name: Run integration tests - working-directory: dsl-py - run: | - nox --no-error-on-missing-interpreter -rs lint test - - - name: Save server logs - if: failure() - uses: actions/upload-artifact@v3 - with: - name: opensearch-logs-${{ matrix.opensearch_ref }}-python-${{ matrix.python-version }} - path: | - opensearch/distribution/archives/linux-tar/build/distributions/**/logs/* - - test-ubuntu-20: - runs-on: ubuntu-20.04 - strategy: - fail-fast: false - matrix: - opensearch_ref: [ '1.x', '2.x', '2.0', 'main' ] - python-version: [ '3.5', '3.6' ] - - steps: - - name: Checkout OpenSearch - uses: actions/checkout@v3 - with: - repository: opensearch-project/opensearch - ref: ${{ matrix.opensearch_ref }} - path: opensearch - - - name: Get OpenSearch branch top - id: get-key - working-directory: opensearch - run: echo key=`git log -1 --format='%H'` >> $GITHUB_OUTPUT - - - name: Restore cached build - id: cache-restore - uses: actions/cache/restore@v3 - with: - path: opensearch/distribution/archives/linux-tar/build/distributions - key: ${{ steps.get-key.outputs.key }} - - - name: Assemble OpenSearch - if: steps.cache-restore.outputs.cache-hit != 'true' - working-directory: opensearch - run: ./gradlew :distribution:archives:linux-tar:assemble - - - name: Save cached build - if: steps.cache-restore.outputs.cache-hit != 'true' - uses: actions/cache/save@v3 - with: - path: opensearch/distribution/archives/linux-tar/build/distributions - key: ${{ steps.get-key.outputs.key }} - - - name: Run OpenSearch - working-directory: opensearch/distribution/archives/linux-tar/build/distributions - run: | - pwd - tree - tar xf opensearch-min-* - ./opensearch-*/bin/opensearch & - for attempt in {1..20}; do sleep 5; if curl -s localhost:9200; then echo '=====> ready'; break; fi; echo '=====> waiting...'; done - - - name: Checkout High Level Python Client - uses: actions/checkout@v3 - with: - path: dsl-py - - - name: Setup Python - ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - env: - PIP_NO_PYTHON_VERSION_WARNING: 1 - - - name: Set up Python 3.8 for Nox - if: matrix.python-version != '3.8' - uses: actions/setup-python@v4 - with: - python-version: 3.8 - name: Install nox run: | - python3.8 -m pip install nox + python -m pip install nox - name: Run integration tests working-directory: dsl-py diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 0c758ee..e9a076e 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -11,7 +11,7 @@ jobs: matrix: version: [ '1.0.1', '1.1.0', '1.2.4', '1.3.7', '2.0.1', '2.1.0', '2.2.1', '2.3.0', '2.4.0', '2.5.0' ] secured: [ "true", "false" ] - python-version: [ '2.7', '3.7', '3.8', '3.9', '3.10', '3.11' ] + python-version: [ '3.8' ] steps: - name: Checkout @@ -38,61 +38,9 @@ jobs: PIP_DISABLE_PIP_VERSION_CHECK: 1 PIP_NO_PYTHON_VERSION_WARNING: 1 - - name: Set up Python 3.8 for Nox - if: matrix.python-version != '3.8' - uses: actions/setup-python@v4 - with: - python-version: 3.8 - - name: Install nox - run: | - python3.8 -m pip install --upgrade pip nox - - - name: Integ OpenSearch secured=${{ matrix.secured }} - run: | - export SECURE_INTEGRATION=${{ matrix.secured }} - nox --no-error-on-missing-interpreter -rs lint test - - integration-ubuntu-20: - name: Integ - runs-on: ubuntu-20.04 - strategy: - fail-fast: false - matrix: - version: [ '1.0.1', '1.1.0', '1.2.4', '1.3.7', '2.0.1', '2.1.0', '2.2.1', '2.3.0', '2.4.0', '2.5.0' ] - secured: [ "true", "false" ] - python-version: [ '3.5', '3.6' ] - - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Clean docker containers - run: | - docker volume prune --force - docker network prune --force - docker system prune --volumes --force - - - name: Launch OpenSearch cluster - run: | - export OPENSEARCH_VERSION=${{ matrix.entry.version }} - export SECURE_INTEGRATION=${{ matrix.secured }} - docker-compose --project-directory .ci/opensearch build - docker-compose --project-directory .ci/opensearch up -d - - - name: Setup Python - ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - env: - PIP_DISABLE_PIP_VERSION_CHECK: 1 - - - name: Set up Python 3.8 for Nox - uses: actions/setup-python@v4 - with: - python-version: 3.8 - name: Install nox run: | - python3.8 -m pip install --upgrade pip nox + python -m pip install --upgrade pip nox - name: Integ OpenSearch secured=${{ matrix.secured }} run: |