diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index d88ded34e..fea4754ad 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -169,6 +169,7 @@ jobs: run: | make generate_latest_sbom make fetch_grid_scaler_resources + make update_browser_versions_matrix - name: Tag browser images if: github.event.inputs.skip-build-push-image != 'true' uses: nick-invision/retry@master diff --git a/.github/workflows/release-chrome-versions.yml b/.github/workflows/release-chrome-versions.yml index 0585af44d..0b6cbe95f 100644 --- a/.github/workflows/release-chrome-versions.yml +++ b/.github/workflows/release-chrome-versions.yml @@ -31,7 +31,7 @@ on: browser-versions: description: 'List browser version to build. E.g: [130, 131]' required: true - default: '[95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138]' + default: '[95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139]' push-image: description: 'Push image after testing successfully' required: true @@ -61,6 +61,7 @@ jobs: permissions: write-all strategy: fail-fast: false + max-parallel: 10 matrix: browser-version: ${{ fromJSON(github.event.inputs.browser-versions)}} outputs: @@ -106,12 +107,6 @@ jobs: run: | echo ${GRID_VERSION} echo "GRID_VERSION=${GRID_VERSION}" >> "$GITHUB_OUTPUT" - - name: Fetch latest version - run: | - python3 -m pip install -r tests/requirements.txt - python3 tests/build-backward-compatible/fetch_firefox_version.py - python3 tests/build-backward-compatible/fetch_version.py - make update_selenium_version_matrix - name: Login Docker Hub run: docker login -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD" env: @@ -124,6 +119,7 @@ jobs: max_attempts: 3 retry_wait_seconds: 60 command: | + make update_browser_versions_matrix ./tests/build-backward-compatible/bootstrap.sh ${GRID_VERSION} ${BROWSER_VERSION} ${BROWSER_NAME} ${REUSE_BASE} cat .env | xargs -I {} echo {} >> $GITHUB_ENV - name: Build Hub image for testing @@ -177,10 +173,7 @@ jobs: - name: Create CHANGELOG directory run: mkdir -p ./CHANGELOG/${{ env.GRID_VERSION }} - name: Fetch latest version - run: | - python3 -m pip install -r tests/requirements.txt - python3 tests/build-backward-compatible/fetch_version.py - make update_selenium_version_matrix + run: make update_browser_versions_matrix - name: Download results uses: actions/download-artifact@v5 with: diff --git a/.github/workflows/release-edge-versions.yml b/.github/workflows/release-edge-versions.yml index a1748c715..96bfd3919 100644 --- a/.github/workflows/release-edge-versions.yml +++ b/.github/workflows/release-edge-versions.yml @@ -61,6 +61,7 @@ jobs: permissions: write-all strategy: fail-fast: false + max-parallel: 10 matrix: browser-version: ${{ fromJSON(github.event.inputs.browser-versions)}} outputs: @@ -106,12 +107,6 @@ jobs: run: | echo ${GRID_VERSION} echo "GRID_VERSION=${GRID_VERSION}" >> "$GITHUB_OUTPUT" - - name: Fetch latest version - run: | - python3 -m pip install -r tests/requirements.txt - python3 tests/build-backward-compatible/fetch_firefox_version.py - python3 tests/build-backward-compatible/fetch_version.py - make update_selenium_version_matrix - name: Login Docker Hub run: docker login -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD" env: @@ -124,6 +119,7 @@ jobs: max_attempts: 3 retry_wait_seconds: 60 command: | + make update_browser_versions_matrix ./tests/build-backward-compatible/bootstrap.sh ${GRID_VERSION} ${BROWSER_VERSION} ${BROWSER_NAME} ${REUSE_BASE} cat .env | xargs -I {} echo {} >> $GITHUB_ENV - name: Build Hub image for testing @@ -177,10 +173,7 @@ jobs: - name: Create CHANGELOG directory run: mkdir -p ./CHANGELOG/${{ env.GRID_VERSION }} - name: Fetch latest version - run: | - python3 -m pip install -r tests/requirements.txt - python3 tests/build-backward-compatible/fetch_version.py - make update_selenium_version_matrix + run: make update_browser_versions_matrix - name: Download results uses: actions/download-artifact@v5 with: diff --git a/.github/workflows/release-firefox-versions.yml b/.github/workflows/release-firefox-versions.yml index b7cd9df14..8015f3252 100644 --- a/.github/workflows/release-firefox-versions.yml +++ b/.github/workflows/release-firefox-versions.yml @@ -31,7 +31,7 @@ on: browser-versions: description: 'List browser version to build. E.g: [130, 131]' required: true - default: '[98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140]' + default: '[98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141]' push-image: description: 'Push image after testing successfully' required: true @@ -61,6 +61,7 @@ jobs: permissions: write-all strategy: fail-fast: false + max-parallel: 10 matrix: browser-version: ${{ fromJSON(github.event.inputs.browser-versions)}} outputs: @@ -106,12 +107,6 @@ jobs: run: | echo ${GRID_VERSION} echo "GRID_VERSION=${GRID_VERSION}" >> "$GITHUB_OUTPUT" - - name: Fetch latest version - run: | - python3 -m pip install -r tests/requirements.txt - python3 tests/build-backward-compatible/fetch_firefox_version.py - python3 tests/build-backward-compatible/fetch_version.py - make update_selenium_version_matrix - name: Login Docker Hub run: docker login -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD" env: @@ -124,6 +119,7 @@ jobs: max_attempts: 3 retry_wait_seconds: 60 command: | + make update_browser_versions_matrix ./tests/build-backward-compatible/bootstrap.sh ${GRID_VERSION} ${BROWSER_VERSION} ${BROWSER_NAME} ${REUSE_BASE} cat .env | xargs -I {} echo {} >> $GITHUB_ENV - name: Build Hub image for testing @@ -177,11 +173,7 @@ jobs: - name: Create CHANGELOG directory run: mkdir -p ./CHANGELOG/${{ env.GRID_VERSION }} - name: Fetch latest version - run: | - python3 -m pip install -r tests/requirements.txt - python3 tests/build-backward-compatible/fetch_firefox_version.py - python3 tests/build-backward-compatible/fetch_version.py - make update_selenium_version_matrix + run: make update_browser_versions_matrix - name: Download results uses: actions/download-artifact@v5 with: diff --git a/Makefile b/Makefile index 197b48eb1..9447d8a24 100644 --- a/Makefile +++ b/Makefile @@ -87,6 +87,13 @@ format_shell_scripts: fi ; \ exit $$EXIT_CODE +install_python_deps: + python3 -m pip install -r tests/requirements.txt --break-system-packages + +format_python_scripts: install_python_deps + python3 -m isort tests/ ; \ + python3 -m black --line-length=120 --skip-string-normalization tests/ + generate_readme_charts: if [ ! -f $$HOME/go/bin/helm-docs ] ; then \ echo "helm-docs is not installed. Please install it or run 'make setup_dev_env' once." ; \ @@ -94,13 +101,17 @@ generate_readme_charts: $$HOME/go/bin/helm-docs --chart-search-root charts/selenium-grid --output-file CONFIGURATION.md --sort-values-order file ; \ fi -update_list_env_vars: - python3 -m pip install -r tests/requirements.txt ; \ +update_list_env_vars: install_python_deps python3 scripts/generate_list_env_vars/extract_env.py -update_selenium_version_matrix: +update_selenium_version_matrix: install_python_deps python3 tests/build-backward-compatible/add_selenium_version.py $(BASE_VERSION) +update_browser_versions_matrix: update_selenium_version_matrix + python3 tests/build-backward-compatible/fetch_firefox_version.py ; \ + python3 tests/build-backward-compatible/fetch_version.py ; \ + python3 tests/build-backward-compatible/update_workflow_versions.py + lint_readme_charts: generate_readme_charts git diff --stat --exit-code ; \ EXIT_CODE=$$? ; \ diff --git a/README.md b/README.md index 49dbd5a72..1ddde2ace 100644 --- a/README.md +++ b/README.md @@ -1507,6 +1507,12 @@ In generic, the script takes the following arguments: - `$3` (optional): browser name. If not provided, it will iterate over all the browsers (`chrome`, `edge`, `firefox`) - `$4` (optional): Push image to registry. By default, it is `false`. If you want to push the image to the registry, set it to `true` (required Docker login to your namespace done before running the script). +To update the browser versions matrix, you can run the following command: + +```bash +make update_browser_versions_matrix +``` + To set your namespace for the images, you can set the environment variable `NAME` before running the script. For example: ```bash diff --git a/charts/selenium-grid/multiple-nodes-platform-version.yaml b/charts/selenium-grid/multiple-nodes-platform-version.yaml index e96148cae..d3a938eb6 100644 --- a/charts/selenium-grid/multiple-nodes-platform-version.yaml +++ b/charts/selenium-grid/multiple-nodes-platform-version.yaml @@ -13,6 +13,11 @@ crossBrowsers: hpa: platformName: 'Linux' browserVersion: '' + - nameOverride: '{{ $.Release.Name }}-node-chrome-139' + imageTag: '139.0-20250808' + hpa: + platformName: 'Linux' + browserVersion: '139.0' - nameOverride: '{{ $.Release.Name }}-node-chrome-138' imageTag: '138.0-20250808' hpa: @@ -455,6 +460,11 @@ crossBrowsers: hpa: platformName: 'Linux' browserVersion: '' + - nameOverride: '{{ $.Release.Name }}-node-edge-139' + imageTag: '139.0-20250808' + hpa: + platformName: 'Linux' + browserVersion: '139.0' - nameOverride: '{{ $.Release.Name }}-node-edge-138' imageTag: '138.0-20250808' hpa: diff --git a/tests/build-backward-compatible/browser-matrix.yml b/tests/build-backward-compatible/browser-matrix.yml index f7d06cd44..946470dee 100644 --- a/tests/build-backward-compatible/browser-matrix.yml +++ b/tests/build-backward-compatible/browser-matrix.yml @@ -1,5 +1,8 @@ matrix: browser: + '142': + FIREFOX_VERSION: 142.0.1 + FIREFOX_PLATFORMS: linux/amd64,linux/arm64 '141': EDGE_VERSION: null CHROME_VERSION: null @@ -7,12 +10,12 @@ matrix: FIREFOX_PLATFORMS: linux/amd64,linux/arm64 '140': EDGE_VERSION: null - CHROME_VERSION: null + CHROME_VERSION: google-chrome-stable=140.0.7339.80-1 FIREFOX_VERSION: 140.0.4 FIREFOX_PLATFORMS: linux/amd64,linux/arm64 '139': - EDGE_VERSION: microsoft-edge-stable=139.0.3405.86-1 - CHROME_VERSION: google-chrome-stable=139.0.7258.127-1 + EDGE_VERSION: microsoft-edge-stable=139.0.3405.125-1 + CHROME_VERSION: google-chrome-stable=139.0.7258.154-1 FIREFOX_VERSION: 139.0.4 FIREFOX_PLATFORMS: linux/amd64,linux/arm64 '138': diff --git a/tests/build-backward-compatible/fetch_version.py b/tests/build-backward-compatible/fetch_version.py index 4b0703432..351c0076e 100755 --- a/tests/build-backward-compatible/fetch_version.py +++ b/tests/build-backward-compatible/fetch_version.py @@ -1,5 +1,3 @@ -from collections import OrderedDict - import requests import yaml @@ -32,6 +30,8 @@ def merge_dicts(dict1, dict2): def update_local_yaml(local_data, source_data): updated = False + local_versions = local_data['matrix']['browser'].keys() + local_min_version = str(min(int(v) for v in local_versions)) if local_versions else "0" for version, details in source_data['matrix']['browser'].items(): if version in local_data['matrix']['browser']: original_details = local_data['matrix']['browser'][version] @@ -39,10 +39,29 @@ def update_local_yaml(local_data, source_data): if key in original_details and '_PACKAGE_' not in key: original_details[key] = details[key] if details[key] is not None else "" updated = True + elif '_PACKAGE_' not in key: + original_details[key] = details[key] if details[key] is not None else "" + updated = True merge_dicts(original_details, details) + else: + if int(version) > int(local_min_version): + local_data['matrix']['browser'][version] = details + local_data['matrix']['browser'][version]['FIREFOX_PLATFORMS'] = 'linux/amd64,linux/arm64' + updated = True return updated +def sort_keys(local_data): + # Sort local_data by key in matrix.browser + list_versions = list(local_data['matrix']['browser'].keys()) + list_versions.sort(key=lambda x: int(x), reverse=True) + sorted_browser_dict = {} + for version in list_versions: + sorted_browser_dict[version] = local_data['matrix']['browser'][version] + local_data['matrix']['browser'] = sorted_browser_dict + return local_data + + def main(): # Fetch source YAML data chrome_data = fetch_yaml(chrome_url) @@ -61,6 +80,7 @@ def main(): # Save updated local YAML data if updated: with open(local_file, 'w') as file: + sort_keys(local_data) yaml.dump(local_data, file, default_flow_style=False, sort_keys=False) print("Local YAML file updated.") else: diff --git a/tests/build-backward-compatible/firefox-matrix.yml b/tests/build-backward-compatible/firefox-matrix.yml index 0d5a45b49..7749bae03 100644 --- a/tests/build-backward-compatible/firefox-matrix.yml +++ b/tests/build-backward-compatible/firefox-matrix.yml @@ -1,5 +1,7 @@ matrix: browser: + '142': + FIREFOX_VERSION: 142.0.1 '141': FIREFOX_VERSION: 141.0.3 '140': diff --git a/tests/build-backward-compatible/update_workflow_versions.py b/tests/build-backward-compatible/update_workflow_versions.py new file mode 100644 index 000000000..70852dfd8 --- /dev/null +++ b/tests/build-backward-compatible/update_workflow_versions.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 +import yaml +import re +from pathlib import Path + +# Minimum browser versions to include in workflows +MIN_CHROME_VERSION = 95 +MIN_FIREFOX_VERSION = 98 +MIN_EDGE_VERSION = 95 + + +def read_browser_matrix(file_path, min_chrome_version, min_firefox_version, min_edge_version): + """Read the browser matrix YAML file and extract browser versions. + + Args: + file_path: Path to the browser matrix YAML file + min_chrome_version: Minimum Chrome version to include + min_firefox_version: Minimum Firefox version to include + min_edge_version: Minimum Edge version to include + """ + with open(file_path, 'r') as f: + data = yaml.safe_load(f) + + chrome_versions = [] + firefox_versions = [] + edge_versions = [] + + browsers = data.get('matrix', {}).get('browser', {}) + + for version, details in browsers.items(): + version_int = int(version) + + # Check for Chrome versions (not null or empty) and above minimum + chrome_version = details.get('CHROME_VERSION') + if (chrome_version and chrome_version != 'null' and str(chrome_version).strip() + and version_int >= min_chrome_version): + chrome_versions.append(version_int) + + # Check for Firefox versions (not null or empty) and above minimum + firefox_version = details.get('FIREFOX_VERSION') + if (firefox_version and firefox_version != 'null' and str(firefox_version).strip() + and version_int >= min_firefox_version): + firefox_versions.append(version_int) + + # Check for Edge versions (not null or empty) and above minimum + edge_version = details.get('EDGE_VERSION') + if (edge_version and edge_version != 'null' and str(edge_version).strip() + and version_int >= min_edge_version): + edge_versions.append(version_int) + + # Sort versions in ascending order + chrome_versions.sort() + firefox_versions.sort() + edge_versions.sort() + + # Exclude the last (newest) version from each list + if chrome_versions: + chrome_versions = chrome_versions[:-1] + if firefox_versions: + firefox_versions = firefox_versions[:-1] + if edge_versions: + edge_versions = edge_versions[:-1] + + return chrome_versions, firefox_versions, edge_versions + + +def format_version_list(versions): + """Format version list as a string like '[95, 96, 97, ...]'""" + return str(versions) + + +def update_workflow_file(workflow_file, versions_list): + """Update the workflow file with new version list for browser-versions.default only.""" + with open(workflow_file, 'r') as f: + lines = f.readlines() + + updated_lines = [] + in_browser_versions = False + + for i, line in enumerate(lines): + # Check if we're in the browser-versions section + if re.match(r'^(\s*)browser-versions:\s*$', line): + in_browser_versions = True + updated_lines.append(line) + elif in_browser_versions and re.match(r'^(\s*)default:\s*', line): + # We found the default line within browser-versions section + indent_match = re.match(r'^(\s*)default:', line) + indent = indent_match.group(1) if indent_match else '' + # Replace the line with new version list + updated_lines.append(f"{indent}default: '{versions_list}'\n") + in_browser_versions = False # Reset flag after updating + elif in_browser_versions and re.match(r'^(\s+)(description|required|type):\s*', line): + # Still within browser-versions section, continue + updated_lines.append(line) + elif in_browser_versions and re.match(r'^(\s*)[a-zA-Z-]+:\s*', line) and not re.match(r'^(\s+)', line): + # We've moved to another top-level field, reset the flag + in_browser_versions = False + updated_lines.append(line) + else: + updated_lines.append(line) + + with open(workflow_file, 'w') as f: + f.writelines(updated_lines) + + +def main(): + # Paths + browser_matrix_file = Path('tests/build-backward-compatible/browser-matrix.yml') + chrome_workflow_file = Path('.github/workflows/release-chrome-versions.yml') + firefox_workflow_file = Path('.github/workflows/release-firefox-versions.yml') + edge_workflow_file = Path('.github/workflows/release-edge-versions.yml') + + # Read browser versions with minimum version filtering + chrome_versions, firefox_versions, edge_versions = read_browser_matrix( + browser_matrix_file, + min_chrome_version=MIN_CHROME_VERSION, + min_firefox_version=MIN_FIREFOX_VERSION, + min_edge_version=MIN_EDGE_VERSION + ) + + # Format version lists + chrome_list = format_version_list(chrome_versions) + firefox_list = format_version_list(firefox_versions) + edge_list = format_version_list(edge_versions) + + print(f"Chrome versions: {chrome_list}") + print(f"Firefox versions: {firefox_list}") + print(f"Edge versions: {edge_list}") + + # Update workflow files + if chrome_workflow_file.exists(): + update_workflow_file(chrome_workflow_file, chrome_list) + print(f"Updated {chrome_workflow_file}") + + if firefox_workflow_file.exists(): + update_workflow_file(firefox_workflow_file, firefox_list) + print(f"Updated {firefox_workflow_file}") + + if edge_workflow_file.exists(): + update_workflow_file(edge_workflow_file, edge_list) + print(f"Updated {edge_workflow_file}") + + +if __name__ == '__main__': + main() diff --git a/tests/charts/make/chart_setup_env.sh b/tests/charts/make/chart_setup_env.sh index b4f5971f9..34c95c4b5 100755 --- a/tests/charts/make/chart_setup_env.sh +++ b/tests/charts/make/chart_setup_env.sh @@ -1,5 +1,7 @@ #!/bin/bash +set -x + echo "Set ENV variables" CLUSTER=${CLUSTER:-"minikube"} DOCKER_VERSION=${DOCKER_VERSION:-""} @@ -38,8 +40,23 @@ if [ -n "${DOCKER_VERSION}" ]; then echo "Installing package docker-ce${DOCKER_VERSION}" ALLOW_DOWNGRADE="--allow-downgrades" fi -sudo apt-get install -yqf ${ALLOW_DOWNGRADE} docker-ce${DOCKER_VERSION} docker-ce-cli${DOCKER_VERSION} -sudo apt-get install -yqf ${ALLOW_DOWNGRADE} containerd.io docker-buildx-plugin docker-compose-plugin gcc-aarch64-linux-gnu qemu-user-static +echo "Installing Docker CE packages..." +timeout 5m sudo apt-get install -yqf ${ALLOW_DOWNGRADE} docker-ce${DOCKER_VERSION} docker-ce-cli${DOCKER_VERSION} || { + echo "Docker CE installation timed out or failed, retrying..." + sudo apt-get install -yf ${ALLOW_DOWNGRADE} docker-ce${DOCKER_VERSION} docker-ce-cli${DOCKER_VERSION} +} + +echo "Installing Docker plugins and container runtime..." +timeout 5m sudo apt-get install -yqf ${ALLOW_DOWNGRADE} containerd.io docker-buildx-plugin docker-compose-plugin || { + echo "Docker plugins installation timed out or failed, retrying..." + sudo apt-get install -yf ${ALLOW_DOWNGRADE} containerd.io docker-buildx-plugin docker-compose-plugin +} + +echo "Installing cross-compilation tools (may take a while)..." +timeout 5m sudo apt-get install -yqf gcc-aarch64-linux-gnu qemu-user-static || { + echo "Cross-compilation tools installation timed out or failed, retrying without quiet mode..." + sudo apt-get install -yf gcc-aarch64-linux-gnu qemu-user-static +} sudo chmod 666 /var/run/docker.sock if [ -n "${DOCKER_VERSION_EXPECT}" ]; then DOCKER_VERSION_ACTUAL="$(docker version --format '{{.Server.Version}}')" diff --git a/tests/requirements.txt b/tests/requirements.txt index 2e9119d06..da27a28e4 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -6,3 +6,5 @@ selenium pyyaml yamale yamllint +isort +black