diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 036e4a25526ac0..352049ffbd9211 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -88,6 +88,7 @@ /src/frontends/tensorflow_common/ @openvinotoolkit/openvino-tf-frontend-maintainers /src/frontends/tensorflow_lite/ @openvinotoolkit/openvino-tf-frontend-maintainers /src/frontends/pytorch/ @openvinotoolkit/openvino-pytorch-frontend-maintainers +/src/frontends/jax/ @openvinotoolkit/openvino-jax-frontend-maintainers # OpenVINO ONNX Frontend: /src/frontends/onnx/ @openvinotoolkit/openvino-onnx-frontend-maintainers @@ -99,7 +100,7 @@ /tests/layer_tests/ @openvinotoolkit/openvino-tests-maintainers @openvinotoolkit/openvino-mo-maintainers /tests/layer_tests/pytorch_tests/ @openvinotoolkit/openvino-pytorch-frontend-maintainers /tests/layer_tests/tensorflow_tests @openvinotoolkit/openvino-tf-frontend-maintainers -/tests/layer_tests/jax_tests @openvinotoolkit/openvino-tf-frontend-maintainers +/tests/layer_tests/jax_tests @openvinotoolkit/openvino-tf-frontend-maintainers @openvinotoolkit/openvino-jax-frontend-maintainers /tests/model_hub_tests @openvinotoolkit/openvino-tf-frontend-maintainers /tests/model_hub_tests/pytorch @openvinotoolkit/openvino-pytorch-frontend-maintainers diff --git a/.github/actions/smart-ci/smart_ci.py b/.github/actions/smart-ci/smart_ci.py index 80d285da0e8d0e..1c8558f4779108 100644 --- a/.github/actions/smart-ci/smart_ci.py +++ b/.github/actions/smart-ci/smart_ci.py @@ -225,7 +225,7 @@ def main(): skip_workflow = False if is_merge_queue or (args.pr and not run_full_scope): - if args.skip_when_only_listed_labels_set: + if args.skip_when_only_listed_labels_set and changed_component_names: excepted_labels = set(args.skip_when_only_listed_labels_set.split(',')) excepted_labels_only = changed_component_names - excepted_labels == set() skip_workflow = excepted_labels_only diff --git a/.github/components.yml b/.github/components.yml index 8ec4b759db39e6..9c5d7c0089c9ca 100644 --- a/.github/components.yml +++ b/.github/components.yml @@ -13,6 +13,9 @@ LP_transformations: - TFL_FE - ONNX_FE - PDPD_FE + - JAX_FE + build: + - TOKENIZERS # TF_FE tests depends on tokenizers build preprocessing: revalidate: @@ -32,12 +35,14 @@ CPU: - PyTorch_FE - TF_FE - ONNX_FE + - JAX_FE build: - AUTO - HETERO - AUTO_BATCH - TEMPLATE - IR_FE + - TOKENIZERS # TF_FE tests depends on tokenizers build GPU: build: @@ -124,7 +129,7 @@ TF_FE: build: - CPU - Python_API - - TOKENIZERS + - TOKENIZERS # TF_FE tests depends on tokenizers build TFL_FE: revalidate: @@ -139,7 +144,13 @@ PyTorch_FE: build: - CPU - Python_API - - TOKENIZERS + +JAX_FE: + revalidate: + - MO + build: + - CPU + - Python_API C_API: build: @@ -155,6 +166,7 @@ Python_API: - samples - MO - tools + - TF_FE build: - CPU - HETERO @@ -164,9 +176,10 @@ Python_API: - IR_FE - ONNX_FE - PDPD_FE - - TF_FE - TFL_FE - PyTorch_FE + - JAX_FE + - TOKENIZERS # TF_FE tests depends on tokenizers build JS_API: build: @@ -200,6 +213,7 @@ MO: - TF_FE build: - Python_API + - TOKENIZERS # TF_FE tests depends on tokenizers build tools: build: @@ -223,5 +237,6 @@ ONNX_RT: build: [] TOKENIZERS: - revalidate: [] + revalidate: + - TF_FE build: [] diff --git a/.github/dependency_review.yml b/.github/dependency_review.yml index 7a1b3a4386a9da..11639f4d2d4b22 100644 --- a/.github/dependency_review.yml +++ b/.github/dependency_review.yml @@ -3,8 +3,10 @@ allow-licenses: - 'BSD-2-Clause' - 'BSD-3-Clause' - 'BSD-2-Clause AND BSD-3-Clause' + - 'BSD-3-Clause AND BSD-3-Clause-Clear' - 'MIT' - 'Apache-2.0' + - 'Apache-2.0 AND BSD-3-Clause' - 'ISC' - 'Apache-2.0 AND MIT' - 'BlueOak-1.0.0' diff --git a/.github/dockerfiles/docker_tag b/.github/dockerfiles/docker_tag index 1143a81836cd5a..956897c9270ff9 100644 --- a/.github/dockerfiles/docker_tag +++ b/.github/dockerfiles/docker_tag @@ -1 +1 @@ -pr-24742 \ No newline at end of file +pr-25130 diff --git a/.github/dockerfiles/ov_build/ubuntu_22_04_android_arm64/Dockerfile b/.github/dockerfiles/ov_build/ubuntu_22_04_android_arm64/Dockerfile new file mode 100644 index 00000000000000..f3d8caa5220f28 --- /dev/null +++ b/.github/dockerfiles/ov_build/ubuntu_22_04_android_arm64/Dockerfile @@ -0,0 +1,55 @@ +FROM openvinogithubactions.azurecr.io/dockerhub/ubuntu:22.04 + +USER root + +# APT configuration +RUN echo 'Acquire::Retries "10";' > /etc/apt/apt.conf && \ + echo 'APT::Get::Assume-Yes "true";' >> /etc/apt/apt.conf && \ + echo 'APT::Get::Fix-Broken "true";' >> /etc/apt/apt.conf && \ + echo 'APT::Get::no-install-recommends "true";' >> /etc/apt/apt.conf + +ENV DEBIAN_FRONTEND="noninteractive" \ + TZ="Europe/London" + +RUN apt update && \ + apt install software-properties-common git ca-certificates && \ + add-apt-repository --yes --no-update ppa:git-core/ppa && \ + apt update && \ + apt install \ + scons \ + wget \ + ninja-build \ + build-essential \ + python3-pip && \ + # vcpkg requires cmake 3.19 or later + python3 -m pip install -U pip cmake~=3.28.0 && \ + # vcpkg's tool dependencies + apt install curl zip unzip tar && \ + # vcpkg 'python3' port dependencies + apt install autoconf libtool autoconf-archive && \ + # vcpkg tree of dependencies require extra packages + apt install pkgconf linux-libc-dev && \ + apt --no-install-recommends install default-jdk && \ + rm -rf /var/lib/apt/lists/* + +# Install sscache +ARG SCCACHE_VERSION="v0.7.5" +ENV SCCACHE_HOME="/opt/sccache" \ + SCCACHE_PATH="/opt/sccache/sccache" + +RUN mkdir ${SCCACHE_HOME} && cd ${SCCACHE_HOME} && \ + SCCACHE_ARCHIVE="sccache-${SCCACHE_VERSION}-x86_64-unknown-linux-musl.tar.gz" && \ + curl -SLO https://github.com/mozilla/sccache/releases/download/${SCCACHE_VERSION}/${SCCACHE_ARCHIVE} && \ + tar -xzf ${SCCACHE_ARCHIVE} --strip-components=1 && rm ${SCCACHE_ARCHIVE} + +ENV PATH="$SCCACHE_HOME:$PATH" + +# Install Android SDK, NDK and Tools +ENV ANDROID_TOOLS /deps/android_tools +ENV ANDROID_NDK_HOME /deps/android_tools/ndk-bundle +RUN mkdir -p ${ANDROID_NDK_HOME} +ENV ANDROID_SDK_VERSION 29 + +RUN wget https://dl.google.com/android/repository/commandlinetools-linux-7583922_latest.zip && \ + unzip commandlinetools-linux-7583922_latest.zip +RUN echo "yes" | ./cmdline-tools/bin/sdkmanager --sdk_root=${ANDROID_TOOLS} --install "ndk-bundle" "platform-tools" "platforms;android-${ANDROID_SDK_VERSION}" diff --git a/.github/dockerfiles/ov_build/ubuntu_22_04_riscv/Dockerfile b/.github/dockerfiles/ov_build/ubuntu_22_04_riscv/Dockerfile new file mode 100644 index 00000000000000..5eda853da56545 --- /dev/null +++ b/.github/dockerfiles/ov_build/ubuntu_22_04_riscv/Dockerfile @@ -0,0 +1,68 @@ +FROM openvinogithubactions.azurecr.io/dockerhub/ubuntu:22.04 + +USER root + +# APT configuration +RUN echo 'Acquire::Retries "10";' > /etc/apt/apt.conf && \ + echo 'APT::Get::Assume-Yes "true";' >> /etc/apt/apt.conf && \ + echo 'APT::Get::Fix-Broken "true";' >> /etc/apt/apt.conf && \ + echo 'APT::Get::no-install-recommends "true";' >> /etc/apt/apt.conf + +ENV DEBIAN_FRONTEND="noninteractive" \ + TZ="Europe/London" + +RUN apt-get update && \ + apt-get install software-properties-common && \ + add-apt-repository --yes --no-update ppa:git-core/ppa && \ + add-apt-repository --yes --no-update ppa:deadsnakes/ppa && \ + apt-get update && \ + # install compilers to build OpenVINO for RISC-V 64 + apt-get install gcc-riscv64-linux-gnu g++-riscv64-linux-gnu && \ + apt-get install \ + curl \ + git \ + cmake \ + ccache \ + ninja-build \ + fdupes \ + patchelf \ + ca-certificates \ + gpg-agent \ + tzdata \ + # Python \ + python3-dev \ + python3-pip \ + python3-venv \ + python3-distutils \ + # Compilers + gcc \ + g++ \ + && \ + rm -rf /var/lib/apt/lists/* + +# Install RISC-V native debian packages +RUN echo deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ jammy main restricted > riscv64-sources.list && \ + echo deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ jammy-updates main restricted >> riscv64-sources.list && \ + echo deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ jammy universe >> riscv64-sources.list && \ + echo deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ jammy-updates universe >> riscv64-sources.list && \ + echo deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ jammy multiverse >> riscv64-sources.list && \ + echo deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ jammy-updates multiverse >> riscv64-sources.list && \ + echo deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ jammy-backports main restricted universe multiverse >> riscv64-sources.list && \ + echo deb [arch=amd64] http://security.ubuntu.com/ubuntu/ jammy-security main restricted >> riscv64-sources.list && \ + echo deb [arch=amd64] http://security.ubuntu.com/ubuntu/ jammy-security universe >> riscv64-sources.list && \ + echo deb [arch=amd64] http://security.ubuntu.com/ubuntu/ jammy-security multiverse >> riscv64-sources.list && \ + echo deb [arch=riscv64] http://ports.ubuntu.com/ubuntu-ports/ jammy main >> riscv64-sources.list && \ + echo deb [arch=riscv64] http://ports.ubuntu.com/ubuntu-ports/ jammy universe >> riscv64-sources.list && \ + echo deb [arch=riscv64] http://ports.ubuntu.com/ubuntu-ports/ jammy-updates main >> riscv64-sources.list && \ + echo deb [arch=riscv64] http://ports.ubuntu.com/ubuntu-ports/ jammy-security main >> riscv64-sources.list && \ + mv riscv64-sources.list /etc/apt/sources.list.d/ + +RUN dpkg --add-architecture riscv64 && \ + apt-get update -o Dir::Etc::sourcelist=/etc/apt/sources.list.d/riscv64-sources.list && \ + apt-get install -y --no-install-recommends libpython3-dev:riscv64 + +# Setup pip +ENV PIP_VERSION="24.0" +RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ + python3 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ + rm -f get-pip.py diff --git a/.github/labeler.yml b/.github/labeler.yml index aeb8e8f2655819..eea70d31684e4c 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -161,6 +161,10 @@ - any: ['tests/model_hub_tests/**', '!tests/model_hub_tests/tensorflow/**/*'] +'category: JAX FE': +- 'src/frontends/jax/**/*' +- 'tests/layer_tests/jax_tests/**/*' + 'category: tools': - any: ['tools/**', '!tools/mo/**/*', diff --git a/.github/scripts/workflow_rerun/errors_to_look_for.json b/.github/scripts/workflow_rerun/errors_to_look_for.json index 8ae2a203ba7332..51e8106944ca9c 100644 --- a/.github/scripts/workflow_rerun/errors_to_look_for.json +++ b/.github/scripts/workflow_rerun/errors_to_look_for.json @@ -54,5 +54,9 @@ { "error_text": "Unable to fetch some archives", "ticket": 130965 + }, + { + "error_text": "status_string: \"Timeout was reached\"", + "ticket": 142653 } ] \ No newline at end of file diff --git a/.github/scripts/workflow_rerun/rerunner.py b/.github/scripts/workflow_rerun/rerunner.py index f92130734c3fe6..b37f51e282a1cd 100644 --- a/.github/scripts/workflow_rerun/rerunner.py +++ b/.github/scripts/workflow_rerun/rerunner.py @@ -6,12 +6,14 @@ import tempfile from pathlib import Path +import requests from github import Github, Auth from workflow_rerun.argument_parser import get_arguments from workflow_rerun.constants import GITHUB_TOKEN, LOGGER from workflow_rerun.log_analyzer import LogAnalyzer from workflow_rerun.log_collector import collect_logs_for_run + if __name__ == '__main__': args = get_arguments() @@ -53,7 +55,12 @@ LOGGER.info(f'RUNNING IN DRY RUN MODE, NOT RETRIGGERING, EXITING') sys.exit(0) - status = run.rerun() + # PyGitHub does not expose the "/repos/{owner}/{repo}/actions/runs/RUN_ID/rerun-failed-jobs" endpoint + # so we have to use requests + response = requests.post(url=f'https://api.github.com/repos/{repository_name}/actions/runs/{run_id}/rerun-failed-jobs', + headers={'Authorization': f'Bearer {GITHUB_TOKEN}'}) + status = response.status_code == 201 + if status: LOGGER.info(f'RUN RETRIGGERED SUCCESSFULLY: {run.html_url}') else: diff --git a/.github/workflows/android_arm64.yml b/.github/workflows/android_arm64.yml index a24987d3811fc6..de5b6c0011e34d 100644 --- a/.github/workflows/android_arm64.yml +++ b/.github/workflows/android_arm64.yml @@ -20,10 +20,11 @@ jobs: runs-on: ubuntu-latest outputs: affected_components: "${{ steps.smart_ci.outputs.affected_components }}" + changed_components: "${{ steps.smart_ci.outputs.changed_components }}" skip_workflow: "${{ steps.smart_ci.outputs.skip_workflow }}" steps: - name: checkout action - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: sparse-checkout: .github/actions/smart-ci @@ -40,15 +41,37 @@ jobs: skip_when_only_listed_labels_set: 'docs' skip_when_only_listed_files_changed: '*.md,*.rst,*.png,*.jpg,*.svg,*/layer_tests_summary/*,*/conformance/*' - Build: + Docker: needs: Smart_CI + runs-on: aks-linux-4-cores-16gb-docker-build + container: + image: openvinogithubactions.azurecr.io/docker_build:0.2 + volumes: + - /mount:/mount + outputs: + images: "${{ steps.handle_docker.outputs.images }}" + steps: + - name: Checkout + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + + - uses: ./.github/actions/handle_docker + id: handle_docker + with: + images: | + ov_build/ubuntu_22_04_android_arm64 + registry: 'openvinogithubactions.azurecr.io' + dockerfiles_root_dir: '.github/dockerfiles' + changed_components: ${{ needs.smart_ci.outputs.changed_components }} + + Build: + needs: [Smart_CI, Docker] timeout-minutes: 150 defaults: run: shell: bash runs-on: aks-linux-16-cores-32gb container: - image: openvinogithubactions.azurecr.io/dockerhub/ubuntu:20.04 + image: ${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_22_04_android_arm64 }} volumes: - /mount:/mount options: -e SCCACHE_AZURE_BLOB_CONTAINER -e SCCACHE_AZURE_CONNECTION_STRING @@ -64,8 +87,8 @@ jobs: OPENVINO_REPO: '/__w/openvino/openvino/openvino' VCPKG_ROOT: '/__w/openvino/openvino/vcpkg' BUILD_DIR: '/__w/openvino/openvino/build' - ANDROID_TOOLS: '/__w/openvino/openvino/android_tools' - ANDROID_NDK_HOME: '/__w/openvino/openvino/android_tools/ndk-bundle' + ANDROID_TOOLS: '/deps/android_tools' + ANDROID_NDK_HOME: '/deps/android_tools/ndk-bundle' ANDROID_SDK_VERSION: 29 ANDROID_ABI_CONFIG: arm64-v8a VCPKG_DEFAULT_BINARY_CACHE: '/mount/caches/ccache/android_arm64/vcpkg_cache' @@ -73,11 +96,8 @@ jobs: SCCACHE_AZURE_KEY_PREFIX: android_arm64 if: "!needs.smart_ci.outputs.skip_workflow" steps: - - name: Install git - run: apt-get update && apt-get install --assume-yes --no-install-recommends git ca-certificates - - name: Clone OpenVINO - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: path: 'openvino' @@ -92,7 +112,7 @@ jobs: popd - name: Clone vcpkg - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: repository: 'microsoft/vcpkg' # Keep in sync with /vcpkg.json @@ -107,35 +127,6 @@ jobs: - name: System info uses: ./openvino/.github/actions/system_info - # - # Dependencies - # - - - name: Install dependencies - run: | - # generic dependencies - apt --assume-yes install ccache scons ninja-build build-essential python3-pip - - # vcpkg requires cmake 3.19 or later - python3 -m pip install -U pip cmake~=3.28.0 - # vcpkg's tool dependencies - apt --assume-yes install curl zip unzip tar - # vcpkg 'python3' port dependencies - apt --assume-yes install autoconf libtool autoconf-archive - # vcpkg tree of dependencies require extra packages - apt --assume-yes install pkgconf linux-libc-dev - - # Install Android SDK, NDK and Tools - apt -y --no-install-recommends install unzip wget default-jdk - wget https://dl.google.com/android/repository/commandlinetools-linux-7583922_latest.zip - unzip commandlinetools-linux-7583922_latest.zip - echo "yes" | ./cmdline-tools/bin/sdkmanager --sdk_root=${ANDROID_TOOLS} --install "ndk-bundle" "platform-tools" "platforms;android-${{ env.ANDROID_SDK_VERSION }}" - - - name: Install sccache - uses: mozilla-actions/sccache-action@2e7f9ec7921547d4b46598398ca573513895d0bd # v0.0.4 - with: - version: "v0.7.5" - # # Build # diff --git a/.github/workflows/build_doc.yml b/.github/workflows/build_doc.yml index 320a0913440774..66d4c8067edea9 100644 --- a/.github/workflows/build_doc.yml +++ b/.github/workflows/build_doc.yml @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-20.04 steps: - name: Clone OpenVINO - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: submodules: 'true' lfs: 'true' diff --git a/.github/workflows/check_pr_commits.yml b/.github/workflows/check_pr_commits.yml index 571cca477d0aeb..75d18695c6e2f9 100644 --- a/.github/workflows/check_pr_commits.yml +++ b/.github/workflows/check_pr_commits.yml @@ -8,7 +8,7 @@ jobs: runs-on: ubuntu-22.04 steps: - name: Clone OpenVINO - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - name: Install dependencies run: python3 -m pip install -r ./.github/github_org_control/requirements.txt diff --git a/.github/workflows/cleanup_caches.yml b/.github/workflows/cleanup_caches.yml index c5cb8145d70d3f..6ba1a4164d9022 100644 --- a/.github/workflows/cleanup_caches.yml +++ b/.github/workflows/cleanup_caches.yml @@ -44,7 +44,7 @@ jobs: steps: - name: Checkout cach action - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: sparse-checkout: .github/actions/cache diff --git a/.github/workflows/code_snippets.yml b/.github/workflows/code_snippets.yml index 36a5489144c44b..af6758bafc11fc 100644 --- a/.github/workflows/code_snippets.yml +++ b/.github/workflows/code_snippets.yml @@ -27,7 +27,7 @@ jobs: runs-on: ${{ matrix.os }} steps: - name: Clone OpenVINO - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: submodules: 'true' diff --git a/.github/workflows/code_style.yml b/.github/workflows/code_style.yml index 0eb4176764d3a9..223a7418156e43 100644 --- a/.github/workflows/code_style.yml +++ b/.github/workflows/code_style.yml @@ -13,7 +13,7 @@ jobs: permissions: pull-requests: write steps: - - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: submodules: 'true' @@ -31,7 +31,7 @@ jobs: - name: suggester / clang-format if: startsWith(github.event_name, 'pull_request') - uses: reviewdog/action-suggester@185c9c06d0a28fbe43b50aca4b32777b649e7cbd # v1.12.0 + uses: reviewdog/action-suggester@a1d57ff096639094e0ba35ef3039e79316364796 # v1.15.0 with: github_token: ${{ secrets.GITHUB_TOKEN }} level: warning @@ -42,7 +42,7 @@ jobs: permissions: pull-requests: write steps: - - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: submodules: 'true' @@ -60,7 +60,7 @@ jobs: # always provide suggestions even for skipped scripts in ov_shellcheck tagret - name: ShellCheck action if: always() - uses: reviewdog/action-shellcheck@72365a51bf6476fe952a117c3ff703eb7775e40a # v1.20.0 + uses: reviewdog/action-shellcheck@52f34f737a16c65b8caa8c51ae1b23036afe5685 # v1.23.0 with: level: style reporter: github-pr-review @@ -73,7 +73,7 @@ jobs: NamingConventionCheck: runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: submodules: 'true' diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index a00d49814e1f91..436f335276d169 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -30,7 +30,7 @@ jobs: max-size: 50G - name: Clone OpenVINO - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: submodules: 'true' @@ -137,6 +137,6 @@ jobs: lcov --capture --directory ${{ github.workspace }}/. --output-file coverage.info genhtml coverage.info --output-directory coverage-report - name: Collect coverage - uses: codecov/codecov-action@125fc84a9a348dbcf27191600683ec096ec9021c # v4.4.1 + uses: codecov/codecov-action@e28ff129e5465c2c0dcc6f003fc735cb6ae0c673 # v4.5.0 with: verbose: true diff --git a/.github/workflows/coverity.yml b/.github/workflows/coverity.yml index 2b9ea632345efa..ef0fd80e7813cf 100644 --- a/.github/workflows/coverity.yml +++ b/.github/workflows/coverity.yml @@ -46,14 +46,14 @@ jobs: apt-get install --assume-yes --no-install-recommends git ca-certificates - name: Clone OpenVINO - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: path: ${{ env.OPENVINO_REPO }} submodules: 'true' ref: ${{ inputs.openvinoRef }} - name: Clone OpenVINO Contrib - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: repository: 'openvinotoolkit/openvino_contrib' path: ${{ env.OPENVINO_CONTRIB_REPO }} diff --git a/.github/workflows/dependency_review.yml b/.github/workflows/dependency_review.yml index 52db0bc671d731..fd2d4f02d57368 100644 --- a/.github/workflows/dependency_review.yml +++ b/.github/workflows/dependency_review.yml @@ -8,7 +8,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Clone OpenVINO - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - name: Dependency Review uses: actions/dependency-review-action@72eb03d02c7872a771aacd928f3123ac62ad6d3a # v4.3.3 diff --git a/.github/workflows/fedora.yml b/.github/workflows/fedora.yml index b9efd16b59151a..5835815e0d9e39 100644 --- a/.github/workflows/fedora.yml +++ b/.github/workflows/fedora.yml @@ -23,7 +23,7 @@ jobs: skip_workflow: "${{ steps.smart_ci.outputs.skip_workflow }}" steps: - name: checkout action - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: sparse-checkout: .github/actions/smart-ci @@ -73,7 +73,7 @@ jobs: run: yum update -y && yum install -y git - name: Clone OpenVINO - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: path: ${{ env.OPENVINO_REPO }} submodules: 'true' @@ -93,7 +93,7 @@ jobs: run: bash ${OPENVINO_REPO}/install_build_dependencies.sh - name: Install sccache - uses: mozilla-actions/sccache-action@2e7f9ec7921547d4b46598398ca573513895d0bd # v0.0.4 + uses: mozilla-actions/sccache-action@89e9040de88b577a072e3760aaf59f585da083af # v0.0.5 with: version: "v0.7.5" @@ -254,7 +254,7 @@ jobs: python3 -c 'from openvino import Core; Core().get_property("MULTI", "SUPPORTED_PROPERTIES")' python3 -c 'from openvino import Core; Core().get_property("HETERO", "SUPPORTED_PROPERTIES")' python3 -c 'from openvino import Core; Core().get_property("BATCH", "SUPPORTED_PROPERTIES")' - python3 -c 'from openvino.frontend import FrontEndManager; assert len(FrontEndManager().get_available_front_ends()) == 6' + python3 -c 'from openvino.frontend import FrontEndManager; assert len(FrontEndManager().get_available_front_ends()) == 7' benchmark_app --help ovc --help diff --git a/.github/workflows/files_size.yml b/.github/workflows/files_size.yml index 3828943d073a39..065fe71bc75893 100644 --- a/.github/workflows/files_size.yml +++ b/.github/workflows/files_size.yml @@ -11,7 +11,7 @@ jobs: Check_Files_Size: runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - name: git ls-tree run: git ls-tree -r -t -l --full-name HEAD | sort -n -r -k 4 diff --git a/.github/workflows/job_cpu_functional_tests.yml b/.github/workflows/job_cpu_functional_tests.yml index 0f94dd735bbb0f..04fa0c8860ab66 100644 --- a/.github/workflows/job_cpu_functional_tests.yml +++ b/.github/workflows/job_cpu_functional_tests.yml @@ -63,7 +63,7 @@ jobs: popd - name: Fetch setup_python action - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: sparse-checkout: | .github/actions/setup_python/action.yml diff --git a/.github/workflows/job_debian_packages.yml b/.github/workflows/job_debian_packages.yml index d4734805a2cb4d..a7547a2483dd16 100644 --- a/.github/workflows/job_debian_packages.yml +++ b/.github/workflows/job_debian_packages.yml @@ -87,7 +87,7 @@ jobs: python3 -c 'from openvino import Core; Core().get_property("MULTI", "SUPPORTED_PROPERTIES")' python3 -c 'from openvino import Core; Core().get_property("HETERO", "SUPPORTED_PROPERTIES")' python3 -c 'from openvino import Core; Core().get_property("BATCH", "SUPPORTED_PROPERTIES")' - python3 -c 'from openvino.frontend import FrontEndManager; assert len(FrontEndManager().get_available_front_ends()) == 6' + python3 -c 'from openvino.frontend import FrontEndManager; assert len(FrontEndManager().get_available_front_ends()) == 7' benchmark_app --help opt_in_out --help ovc --help diff --git a/.github/workflows/job_gpu_tests.yml b/.github/workflows/job_gpu_tests.yml index 7ba71afec09748..7a5af97cdcde49 100644 --- a/.github/workflows/job_gpu_tests.yml +++ b/.github/workflows/job_gpu_tests.yml @@ -21,6 +21,8 @@ on: required: false default: '{"image": null}' +permissions: read-all + jobs: GPU: timeout-minutes: 80 diff --git a/.github/workflows/job_onnx_models_tests.yml b/.github/workflows/job_onnx_models_tests.yml index 30c226488fbb5f..3fac0998d88ced 100644 --- a/.github/workflows/job_onnx_models_tests.yml +++ b/.github/workflows/job_onnx_models_tests.yml @@ -69,7 +69,7 @@ jobs: popd - name: Fetch model_zoo_preprocess script - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: sparse-checkout: | src/frontends/onnx/tests/tests_python/model_zoo_preprocess.sh diff --git a/.github/workflows/job_openvino_js.yml b/.github/workflows/job_openvino_js.yml index a90bafd968d60b..6ed0ddec08eb0b 100644 --- a/.github/workflows/job_openvino_js.yml +++ b/.github/workflows/job_openvino_js.yml @@ -31,7 +31,7 @@ jobs: NODE_VERSION: 20 steps: - name: Fetch OpenVINO JS sources - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: sparse-checkout: | src/bindings/js diff --git a/.github/workflows/job_python_unit_tests.yml b/.github/workflows/job_python_unit_tests.yml index 2348e64a78a01e..bab441e3e27453 100644 --- a/.github/workflows/job_python_unit_tests.yml +++ b/.github/workflows/job_python_unit_tests.yml @@ -70,7 +70,7 @@ jobs: popd - name: Fetch setup_python action - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: sparse-checkout: | .github/actions/setup_python/action.yml @@ -121,7 +121,7 @@ jobs: # - name: Python API Tests - # if: ${{ fromJSON(inputs.affected-components).Python_API.test && runner.arch != 'ARM64' }} # Ticket: 126380, 127101 + if: ${{ fromJSON(inputs.affected-components).Python_API.test }} run: | # for 'template' extension export LD_LIBRARY_PATH=${INSTALL_TEST_DIR}:$LD_LIBRARY_PATH @@ -246,6 +246,14 @@ jobs: env: TEST_DEVICE: CPU TEST_PRECISION: FP16 + + - name: JAX Layer Tests - JAX FE + if: ${{ fromJSON(inputs.affected-components).JAX_FE.test && runner.arch != 'ARM64' && runner.os != 'macOS' }} + run: python3 -m pytest ${LAYER_TESTS_INSTALL_DIR}/jax_tests/ -m precommit_jax_fe --junitxml=${INSTALL_TEST_DIR}/TEST-jax_fe.xml + env: + TEST_DEVICE: CPU + TEST_PRECISION: FP16 + JAX_TRACE_MODE: JAXPR - name: TensorFlow Lite Layer Tests - TFL FE if: fromJSON(inputs.affected-components).TFL_FE.test @@ -270,9 +278,9 @@ jobs: - name: Clone API snippets if: runner.os != 'macOS' - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: - sparse-checkout: docs/snippets + sparse-checkout: docs/articles_en/assets/snippets path: ${{ env.OPENVINO_REPO }} submodules: 'false' @@ -280,10 +288,10 @@ jobs: if: runner.os != 'macOS' run: | # to find 'snippets' module in docs - export PYTHONPATH=${OPENVINO_REPO}/docs + export PYTHONPATH=${OPENVINO_REPO}/docs/articles_en/assets # for 'template' extension export LD_LIBRARY_PATH=${INSTALL_TEST_DIR}:$LD_LIBRARY_PATH - python3 ${OPENVINO_REPO}/docs/snippets/main.py + python3 ${OPENVINO_REPO}/docs/articles_en/assets/snippets/main.py - name: Upload Test Results uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 diff --git a/.github/workflows/job_pytorch_models_tests.yml b/.github/workflows/job_pytorch_models_tests.yml index 6ebaaf183fef41..17828576336bfc 100644 --- a/.github/workflows/job_pytorch_models_tests.yml +++ b/.github/workflows/job_pytorch_models_tests.yml @@ -85,7 +85,7 @@ jobs: popd - name: Fetch setup_python action - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: sparse-checkout: | .github/actions/setup_python/action.yml diff --git a/.github/workflows/job_samples_tests.yml b/.github/workflows/job_samples_tests.yml index ac2274d84156ac..e453210d58b13b 100644 --- a/.github/workflows/job_samples_tests.yml +++ b/.github/workflows/job_samples_tests.yml @@ -7,11 +7,11 @@ on: description: 'Machine on which the tests would run' type: string required: true - image: - description: 'Docker image in which the tests would run' + container: + description: 'JSON to be converted to the value of the "container" configuration for the job' type: string required: false - default: null + default: '{"image": null}' affected-components: description: 'Components that are affected by changes in the commit defined by the Smart CI Action' type: string @@ -21,9 +21,9 @@ permissions: read-all jobs: Samples: + timeout-minutes: 30 runs-on: ${{ inputs.runner }} - container: - image: ${{ inputs.image }} + container: ${{ fromJSON(inputs.container) }} defaults: run: shell: bash @@ -66,7 +66,9 @@ jobs: run: brew install coreutils - name: Fetch setup_python action - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + # Python is already installed on Ubuntu within Dockerfile + if: runner.os != 'Linux' + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: sparse-checkout: | .github/actions/setup_python/action.yml @@ -74,6 +76,8 @@ jobs: path: 'openvino' - name: Setup Python 3.11 + # Python is already installed on Ubuntu within Dockerfile + if: runner.os != 'Linux' uses: ./openvino/.github/actions/setup_python with: version: '3.11' diff --git a/.github/workflows/job_tensorflow_layer_tests.yml b/.github/workflows/job_tensorflow_layer_tests.yml index e19e64d8aa058e..168d9bf61308d7 100644 --- a/.github/workflows/job_tensorflow_layer_tests.yml +++ b/.github/workflows/job_tensorflow_layer_tests.yml @@ -91,7 +91,7 @@ jobs: popd - name: Fetch setup_python action - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: sparse-checkout: | .github/actions/setup_python/action.yml diff --git a/.github/workflows/job_tensorflow_models_tests.yml b/.github/workflows/job_tensorflow_models_tests.yml index cf69d0c4f1daf7..1a452c94db0ace 100644 --- a/.github/workflows/job_tensorflow_models_tests.yml +++ b/.github/workflows/job_tensorflow_models_tests.yml @@ -77,7 +77,7 @@ jobs: popd - name: Fetch setup_python action - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: sparse-checkout: | .github/actions/setup_python/action.yml diff --git a/.github/workflows/job_tokenizers.yml b/.github/workflows/job_tokenizers.yml index bf3d650873f4ae..da9e096e7abd8b 100644 --- a/.github/workflows/job_tokenizers.yml +++ b/.github/workflows/job_tokenizers.yml @@ -50,7 +50,7 @@ jobs: echo "EXTENSION_BUILD_DIR=$GITHUB_WORKSPACE/build" >> "$GITHUB_ENV" - name: checkout action - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: sparse-checkout: | .github/actions/setup_python @@ -66,7 +66,7 @@ jobs: self-hosted-runner: ${{ runner.os == 'Linux' }} - name: Clone OpenVINO Tokenizers - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: repository: 'openvinotoolkit/openvino_tokenizers' path: ${{ env.OPENVINO_TOKENIZERS_REPO }} @@ -96,10 +96,21 @@ jobs: # Dependencies # - - name: Install python dependencies + - name: Install OpenVINO Python wheel (Linux and macOS) + if: runner.os != 'Windows' + run: | + # Find and install wheel + pushd ${INSTALL_DIR}/tools + wheel_name=$(find . -name 'openvino-*.whl') + python3 -m pip install $wheel_name + popd + + - name: Install OpenVINO Python wheel (Windows) + if: runner.os == 'Windows' run: | - # wheel packaging - python3 -m pip install -r ${{ env.OPENVINO_TOKENIZERS_REPO }}/requirements-build.txt + # Find and install wheel + $ovCoreWheelPath=Get-ChildItem -Path "${{ env.INSTALL_DIR }}\\tools" -Filter openvino-*.whl | % { $_.FullName } + python3 -m pip install "$ovCoreWheelPath" # # Build @@ -108,20 +119,18 @@ jobs: - name: Build tokenizers wheel (Linux and macOS) if: runner.os != 'Windows' run: | - source ${INSTALL_DIR}/setupvars.sh - python -m build --wheel --outdir ${EXTENSION_BUILD_DIR} ${OPENVINO_TOKENIZERS_REPO} + # use OpenVINO wheel package only to build the extension + export OpenVINO_DIR=$(python3 -c "from openvino.utils import get_cmake_path; print(get_cmake_path(), end='')") + python -m pip wheel -v --no-deps --wheel-dir ${EXTENSION_BUILD_DIR} ${OPENVINO_TOKENIZERS_REPO} env: - CMAKE_ARGS: '-DBUILD_FAST_TOKENIZERS=OFF' CMAKE_BUILD_PARALLEL_LEVEL: '4' - CMAKE_GENERATOR: 'Unix Makefiles' - name: Build tokenizers wheel (Windows) if: runner.os == 'Windows' run: | - . "${{ env.INSTALL_DIR }}/setupvars.ps1" - python3 -m build --wheel --outdir ${env:EXTENSION_BUILD_DIR} ${env:OPENVINO_TOKENIZERS_REPO} + $env:OpenVINO_DIR=$(python3 -c "from openvino.utils import get_cmake_path; print(get_cmake_path(), end='')") + python3 -m pip wheel -v --no-deps --wheel-dir ${env:EXTENSION_BUILD_DIR} ${env:OPENVINO_TOKENIZERS_REPO} env: - CMAKE_ARGS: '-DBUILD_FAST_TOKENIZERS=OFF' CMAKE_BUILD_PARALLEL_LEVEL: '4' # diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index c07df7494f7faa..f6464838555e50 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -24,7 +24,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout Labeller Script - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: sparse-checkout: '.github' diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 744e693b1cff51..4358af5707b077 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -31,7 +31,7 @@ jobs: skip_workflow: "${{ steps.smart_ci.outputs.skip_workflow }}" steps: - name: checkout action - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: sparse-checkout: .github/actions/smart-ci @@ -64,7 +64,7 @@ jobs: images: "${{ steps.handle_docker.outputs.images }}" steps: - name: Checkout - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - uses: ./.github/actions/handle_docker id: handle_docker @@ -113,7 +113,7 @@ jobs: steps: - name: Clone OpenVINO - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: path: ${{ env.OPENVINO_REPO }} submodules: 'true' @@ -128,7 +128,7 @@ jobs: git rev-parse HEAD - name: Clone OpenVINO Contrib - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: repository: 'openvinotoolkit/openvino_contrib' path: ${{ env.OPENVINO_CONTRIB_REPO }} @@ -224,7 +224,7 @@ jobs: run: | /usr/bin/python3.8 -m pip install -U pip /usr/bin/python3.8 -m pip install -r ${OPENVINO_REPO}/src/bindings/python/wheel/requirements-dev.txt - cmake -UPYTHON* \ + cmake -UPython* -U_Python* -UPYTHON* \ -UTBB* \ -DENABLE_SYSTEM_TBB=ON \ -DENABLE_PYTHON_PACKAGING=ON \ @@ -245,11 +245,9 @@ jobs: - name: CMake configure, build and install - OpenVINO JS API if: fromJSON(needs.smart_ci.outputs.affected_components).JS_API - run: - cmake -DCPACK_GENERATOR=NPM -DENABLE_SYSTEM_TBB=OFF -UTBB* -S ${OPENVINO_REPO} -B ${BUILD_DIR} - + run: | + cmake -UTBB* -DCPACK_GENERATOR=NPM -DENABLE_SYSTEM_TBB=OFF -S ${OPENVINO_REPO} -B ${BUILD_DIR} cmake --build ${BUILD_DIR} --parallel - cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR_JS} -P ${BUILD_DIR}/cmake_install.cmake # @@ -317,7 +315,7 @@ jobs: uses: ./.github/workflows/job_samples_tests.yml with: runner: 'aks-linux-4-cores-16gb' - image: ${{ fromJSON(needs.docker.outputs.images).ov_test.ubuntu_20_04_x64 }} + container: '{"image": "${{ fromJSON(needs.docker.outputs.images).ov_test.ubuntu_20_04_x64 }}", "volumes": ["/mount:/mount"]}' affected-components: ${{ needs.smart_ci.outputs.affected_components }} JS_API: @@ -383,7 +381,7 @@ jobs: popd - name: Fetch setup_python action - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: sparse-checkout: | .github/actions/setup_python/action.yml @@ -477,7 +475,7 @@ jobs: needs: [ Docker, Build, Smart_CI ] uses: ./.github/workflows/job_onnx_models_tests.yml with: - runner: 'aks-linux-16-cores-32gb' + runner: 'aks-linux-16-cores-64gb' container: '{"image": "${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_20_04_x64 }}", "volumes": ["/mount:/mount"]}' CXX_Unit_Tests: @@ -534,7 +532,7 @@ jobs: needs: [ Docker, Build, Smart_CI, Openvino_tokenizers ] uses: ./.github/workflows/job_tensorflow_models_tests.yml with: - runner: 'aks-linux-8-cores-32gb' + runner: 'aks-linux-8-cores-64gb' model_scope: 'nightly_tf_hub' container: '{"image": "${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_20_04_x64 }}", "volumes": ["/mount:/mount"]}' @@ -544,7 +542,7 @@ jobs: needs: [ Docker, Build, Smart_CI, Openvino_tokenizers ] uses: ./.github/workflows/job_tensorflow_models_tests.yml with: - runner: 'aks-linux-8-cores-32gb' + runner: 'aks-linux-8-cores-64gb' model_scope: 'nightly_hf' container: '{"image": "${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_20_04_x64 }}", "volumes": ["/mount:/mount"]}' @@ -618,7 +616,7 @@ jobs: popd - name: Clone OpenVINO Contrib - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: repository: 'openvinotoolkit/openvino_contrib' path: ${{ env.OPENVINO_CONTRIB_REPO }} diff --git a/.github/workflows/linux_arm64.yml b/.github/workflows/linux_arm64.yml index 95dab44352d159..9156fa711e752a 100644 --- a/.github/workflows/linux_arm64.yml +++ b/.github/workflows/linux_arm64.yml @@ -28,7 +28,7 @@ jobs: skip_workflow: "${{ steps.smart_ci.outputs.skip_workflow }}" steps: - name: checkout action - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: sparse-checkout: .github/actions/smart-ci @@ -61,7 +61,7 @@ jobs: images: "${{ steps.handle_docker.outputs.images }}" steps: - name: Checkout - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - uses: ./.github/actions/handle_docker id: handle_docker @@ -108,13 +108,13 @@ jobs: steps: - name: Clone OpenVINO - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: path: ${{ env.OPENVINO_REPO }} submodules: 'true' - name: Clone OpenVINO Contrib - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: repository: 'openvinotoolkit/openvino_contrib' path: ${{ env.OPENVINO_CONTRIB_REPO }} @@ -309,7 +309,7 @@ jobs: uses: ./.github/workflows/job_samples_tests.yml with: runner: 'aks-linux-16-cores-arm' - image: ${{ fromJSON(needs.docker.outputs.images).ov_test.ubuntu_20_04_arm64 }} + container: '{"image": "${{ fromJSON(needs.docker.outputs.images).ov_test.ubuntu_20_04_arm64 }}", "volumes": ["/mount:/mount"]}' affected-components: ${{ needs.smart_ci.outputs.affected_components }} JS_API: diff --git a/.github/workflows/linux_conditional_compilation.yml b/.github/workflows/linux_conditional_compilation.yml index 8cee38b67e0d33..cdae9c77af9b40 100644 --- a/.github/workflows/linux_conditional_compilation.yml +++ b/.github/workflows/linux_conditional_compilation.yml @@ -28,7 +28,7 @@ jobs: skip_workflow: "${{ steps.smart_ci.outputs.skip_workflow }}" steps: - name: checkout action - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: sparse-checkout: .github/actions/smart-ci @@ -61,7 +61,7 @@ jobs: images: "${{ steps.handle_docker.outputs.images }}" steps: - name: Checkout - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - uses: ./.github/actions/handle_docker id: handle_docker @@ -107,13 +107,13 @@ jobs: steps: - name: Clone OpenVINO - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: path: ${{ env.OPENVINO_REPO }} submodules: 'true' - name: Clone test models - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: repository: 'openvinotoolkit/testdata' path: ${{ env.MODELS_PATH }} @@ -279,13 +279,13 @@ jobs: steps: - name: Clone OpenVINO - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: path: ${{ env.OPENVINO_REPO }} submodules: 'true' - name: Clone test models - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: repository: 'openvinotoolkit/testdata' path: ${{ env.MODELS_PATH }} @@ -317,6 +317,7 @@ jobs: -DENABLE_OV_TF_LITE_FRONTEND=OFF \ -DENABLE_OV_PADDLE_FRONTEND=OFF \ -DENABLE_OV_PYTORCH_FRONTEND=OFF \ + -DENABLE_OV_JAX_FRONTEND=OFF \ -DENABLE_OV_ONNX_FRONTEND=OFF \ -DSELECTIVE_BUILD_STAT=${SELECTIVE_BUILD_STAT_DIR}/*.csv \ -DCMAKE_C_COMPILER_LAUNCHER=${{ env.CMAKE_C_COMPILER_LAUNCHER }} \ diff --git a/.github/workflows/linux_riscv.yml b/.github/workflows/linux_riscv.yml index bb5116df054578..6c26bb8b3859f0 100644 --- a/.github/workflows/linux_riscv.yml +++ b/.github/workflows/linux_riscv.yml @@ -1,8 +1,5 @@ name: Linux RISC-V with Conan (Ubuntu 22.04, Python 3.10) on: - schedule: - # at 00:00 on Wednesday and Saturday - - cron: '0 0 * * 3,6' workflow_dispatch: pull_request: merge_group: @@ -23,10 +20,11 @@ jobs: runs-on: ubuntu-latest outputs: affected_components: "${{ steps.smart_ci.outputs.affected_components }}" + changed_components: "${{ steps.smart_ci.outputs.changed_components }}" skip_workflow: "${{ steps.smart_ci.outputs.skip_workflow }}" steps: - name: checkout action - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: sparse-checkout: .github/actions/smart-ci @@ -43,15 +41,37 @@ jobs: skip_when_only_listed_labels_set: 'docs' skip_when_only_listed_files_changed: '*.md,*.rst,*.png,*.jpg,*.svg,*/layer_tests_summary/*,*/conformance/*' - Build: + Docker: needs: Smart_CI + runs-on: aks-linux-4-cores-16gb-docker-build + container: + image: openvinogithubactions.azurecr.io/docker_build:0.2 + volumes: + - /mount:/mount + outputs: + images: "${{ steps.handle_docker.outputs.images }}" + steps: + - name: Checkout + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + + - uses: ./.github/actions/handle_docker + id: handle_docker + with: + images: | + ov_build/ubuntu_22_04_riscv + registry: 'openvinogithubactions.azurecr.io' + dockerfiles_root_dir: '.github/dockerfiles' + changed_components: ${{ needs.smart_ci.outputs.changed_components }} + + Build: + needs: [Smart_CI, Docker] timeout-minutes: 150 defaults: run: shell: bash runs-on: aks-linux-16-cores-32gb container: - image: openvinogithubactions.azurecr.io/dockerhub/ubuntu:22.04 + image: ${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_22_04_riscv }} volumes: - /mount:/mount env: @@ -70,14 +90,8 @@ jobs: if: ${{ !needs.smart_ci.outputs.skip_workflow && github.event_name != 'merge_group' }} steps: - - name: Set apt retries - run: echo 'Acquire::Retries "10";' > /etc/apt/apt.conf.d/80-retries - - - name: Install git - run: apt-get update && apt-get install --assume-yes --no-install-recommends git ca-certificates - - name: Clone OpenVINO - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: path: 'openvino' @@ -119,43 +133,16 @@ jobs: # Dependencies # - - name: Install build dependencies + - name: Install Python dependencies run: | # create build directory mkdir -p ${OPENVINO_BUILD_DIR} - # install compilers to build OpenVINO for RISC-V 64 - apt --assume-yes install gcc-riscv64-linux-gnu g++-riscv64-linux-gnu - apt --assume-yes install -yf gcc g++ python3-pip python3-venv python3-dev - # generic dependencies - apt --assume-yes install cmake ccache ninja-build fdupes patchelf - python3 -m venv ${OPENVINO_BUILD_DIR}/env source ${OPENVINO_BUILD_DIR}/env/bin/activate python3 -m pip install -r ${OPENVINO_REPO}/src/bindings/python/wheel/requirements-dev.txt python3 -m pip install conan - - name: Install RISC-V native debian packages - run: | - echo deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ jammy main restricted > riscv64-sources.list - echo deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ jammy-updates main restricted >> riscv64-sources.list - echo deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ jammy universe >> riscv64-sources.list - echo deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ jammy-updates universe >> riscv64-sources.list - echo deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ jammy multiverse >> riscv64-sources.list - echo deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ jammy-updates multiverse >> riscv64-sources.list - echo deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ jammy-backports main restricted universe multiverse >> riscv64-sources.list - echo deb [arch=amd64] http://security.ubuntu.com/ubuntu/ jammy-security main restricted >> riscv64-sources.list - echo deb [arch=amd64] http://security.ubuntu.com/ubuntu/ jammy-security universe >> riscv64-sources.list - echo deb [arch=amd64] http://security.ubuntu.com/ubuntu/ jammy-security multiverse >> riscv64-sources.list - echo deb [arch=riscv64] http://ports.ubuntu.com/ubuntu-ports/ jammy main >> riscv64-sources.list - echo deb [arch=riscv64] http://ports.ubuntu.com/ubuntu-ports/ jammy universe >> riscv64-sources.list - echo deb [arch=riscv64] http://ports.ubuntu.com/ubuntu-ports/ jammy-updates main >> riscv64-sources.list - echo deb [arch=riscv64] http://ports.ubuntu.com/ubuntu-ports/ jammy-security main >> riscv64-sources.list - mv riscv64-sources.list /etc/apt/sources.list.d/ - dpkg --add-architecture riscv64 - apt-get update -o Dir::Etc::sourcelist=/etc/apt/sources.list.d/riscv64-sources.list - apt-get install -y --no-install-recommends libpython3-dev:riscv64 - - name: Create conan_toolchain.cmake file run: | source ${OPENVINO_BUILD_DIR}/env/bin/activate diff --git a/.github/workflows/linux_sanitizers.yml b/.github/workflows/linux_sanitizers.yml index 1daecd00a484e1..79d2e740261161 100644 --- a/.github/workflows/linux_sanitizers.yml +++ b/.github/workflows/linux_sanitizers.yml @@ -35,8 +35,8 @@ jobs: include: - SANITIZER: 'AddressAndLeak' SANITIZER_CMAKE_OPTION: '-DENABLE_SANITIZER=ON' - - SANITIZER: 'UndefinedBehavior' - SANITIZER_CMAKE_OPTION: '-DENABLE_UB_SANITIZER=ON' +# - SANITIZER: 'UndefinedBehavior' # Ticket: 144163 +# SANITIZER_CMAKE_OPTION: '-DENABLE_UB_SANITIZER=ON' # - SANITIZER: 'Thread' # Problems with protobuf # SANITIZER_CMAKE_OPTION: '-DENABLE_THREAD_SANITIZER=ON' env: @@ -64,13 +64,13 @@ jobs: apt-get install --assume-yes --no-install-recommends git ca-certificates - name: Clone OpenVINO - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: path: ${{ env.OPENVINO_REPO }} submodules: 'true' - name: Clone OpenVINO Contrib - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: repository: 'openvinotoolkit/openvino_contrib' path: ${{ env.OPENVINO_CONTRIB_REPO }} @@ -135,7 +135,6 @@ jobs: -DENABLE_NCC_STYLE=OFF \ -DENABLE_TESTS=ON \ -DENABLE_STRICT_DEPENDENCIES=OFF \ - -DENABLE_SYSTEM_TBB=ON \ -DENABLE_SYSTEM_OPENCL=ON \ -DCMAKE_VERBOSE_MAKEFILE=ON \ -DCPACK_GENERATOR=TGZ \ @@ -144,6 +143,7 @@ jobs: -DENABLE_OV_TF_LITE_FRONTEND=ON \ -DENABLE_OV_PADDLE_FRONTEND=ON \ -DENABLE_OV_PYTORCH_FRONTEND=ON \ + -DENABLE_OV_JAX_FRONTEND=ON \ -DENABLE_OV_ONNX_FRONTEND=ON \ -DENABLE_ONEDNN_FOR_GPU=OFF \ -DCMAKE_COMPILE_WARNING_AS_ERROR=OFF \ @@ -266,7 +266,7 @@ jobs: apt update && apt --assume-yes install clang lld - name: Fetch Sanitizer Suppression Lists - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: sparse-checkout: | tests/lsan/suppressions.txt @@ -360,7 +360,7 @@ jobs: --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-TensorFlowLiteFrontend.xml - name: Transformations func tests - if: always() + if: ${{ 'false' }} # Ticket: 143900 run: | source ${INSTALL_DIR}/setupvars.sh @@ -389,7 +389,7 @@ jobs: --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-CPUUnitTests.xml - name: ov_subgraphs_dumper_tests tests - if: ${{ 'false' }} # Ticket: 134419 + if: always() run: | source ${INSTALL_DIR}/setupvars.sh ${INSTALL_TEST_DIR}/ov_subgraphs_dumper_tests --gtest_print_time=1 \ diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml index 2c2f445fc5cc47..d044bcd0fad4c8 100644 --- a/.github/workflows/mac.yml +++ b/.github/workflows/mac.yml @@ -42,7 +42,7 @@ jobs: skip_workflow: "${{ steps.smart_ci.outputs.skip_workflow }}" steps: - name: checkout action - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: sparse-checkout: .github/actions/smart-ci @@ -80,13 +80,13 @@ jobs: BUILD_DIR: ${{ github.workspace }}/build steps: - name: Clone OpenVINO - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: path: 'openvino' submodules: 'true' - name: Clone OpenVINO Contrib - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: repository: 'openvinotoolkit/openvino_contrib' path: 'openvino_contrib' diff --git a/.github/workflows/mac_arm64.yml b/.github/workflows/mac_arm64.yml index 8d51e8ae34be8b..c9a9fceb2f8b41 100644 --- a/.github/workflows/mac_arm64.yml +++ b/.github/workflows/mac_arm64.yml @@ -42,7 +42,7 @@ jobs: skip_workflow: "${{ steps.smart_ci.outputs.skip_workflow }}" steps: - name: checkout action - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: sparse-checkout: .github/actions/smart-ci @@ -80,13 +80,13 @@ jobs: BUILD_DIR: ${{ github.workspace }}/build steps: - name: Clone OpenVINO - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: path: 'openvino' submodules: 'true' - name: Clone OpenVINO Contrib - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: repository: 'openvinotoolkit/openvino_contrib' path: 'openvino_contrib' diff --git a/.github/workflows/mo.yml b/.github/workflows/mo.yml index f8318f1a69190a..f810ce7ba5b805 100644 --- a/.github/workflows/mo.yml +++ b/.github/workflows/mo.yml @@ -23,7 +23,7 @@ jobs: runs-on: ubuntu-22.04 steps: - name: Clone OpenVINO - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - name: Setup Python uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 diff --git a/.github/workflows/ovc.yml b/.github/workflows/ovc.yml index 25c1f4c738bb93..36b0c00f59772e 100644 --- a/.github/workflows/ovc.yml +++ b/.github/workflows/ovc.yml @@ -18,7 +18,7 @@ jobs: runs-on: ubuntu-22.04 steps: - name: Clone OpenVINO - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - name: Setup Python uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 diff --git a/.github/workflows/py_checks.yml b/.github/workflows/py_checks.yml index 5766105f1649de..3a9b23ea2685db 100644 --- a/.github/workflows/py_checks.yml +++ b/.github/workflows/py_checks.yml @@ -27,7 +27,7 @@ jobs: runs-on: ubuntu-20.04 steps: - name: Clone OpenVINO - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - name: Setup Python uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 diff --git a/.github/workflows/send_workflows_to_opentelemetry.yml b/.github/workflows/send_workflows_to_opentelemetry.yml index c52b20bdcfd628..acf84074c41222 100644 --- a/.github/workflows/send_workflows_to_opentelemetry.yml +++ b/.github/workflows/send_workflows_to_opentelemetry.yml @@ -35,7 +35,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: sparse-checkout: '.github' diff --git a/.github/workflows/webassembly.yml b/.github/workflows/webassembly.yml index f14b7de568c336..469ccda02f6944 100644 --- a/.github/workflows/webassembly.yml +++ b/.github/workflows/webassembly.yml @@ -23,7 +23,7 @@ jobs: skip_workflow: "${{ steps.smart_ci.outputs.skip_workflow }}" steps: - name: checkout action - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: sparse-checkout: .github/actions/smart-ci @@ -66,13 +66,13 @@ jobs: run: apt-get update && apt-get install --assume-yes --no-install-recommends git ca-certificates - name: Clone OpenVINO - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: path: 'openvino' submodules: 'true' - name: Install sccache - uses: mozilla-actions/sccache-action@2e7f9ec7921547d4b46598398ca573513895d0bd # v0.0.4 + uses: mozilla-actions/sccache-action@89e9040de88b577a072e3760aaf59f585da083af # v0.0.5 with: version: "v0.7.5" diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 460bff4db18c35..15718ab2aa4326 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -26,7 +26,7 @@ jobs: skip_workflow: "${{ steps.smart_ci.outputs.skip_workflow }}" steps: - name: checkout action - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: sparse-checkout: .github/actions/smart-ci @@ -69,13 +69,13 @@ jobs: steps: - name: Clone OpenVINO - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: path: 'openvino' submodules: 'true' - name: Clone OpenVINO Contrib - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: repository: 'openvinotoolkit/openvino_contrib' path: 'openvino_contrib' @@ -278,7 +278,7 @@ jobs: popd - name: Fetch setup_python action - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: sparse-checkout: | .github/actions/setup_python/action.yml @@ -343,7 +343,7 @@ jobs: steps: - name: Fetch OpenVINO JS sources - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: sparse-checkout: | src/bindings/js @@ -430,7 +430,7 @@ jobs: popd - name: Fetch setup_python action - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: sparse-checkout: | .github/actions/setup_python/action.yml @@ -818,7 +818,7 @@ jobs: popd - name: Fetch setup_python action - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: sparse-checkout: | .github/actions/setup_python/action.yml diff --git a/.github/workflows/windows_conditional_compilation.yml b/.github/workflows/windows_conditional_compilation.yml index 182f5883bd1aea..977f9aee91bcce 100644 --- a/.github/workflows/windows_conditional_compilation.yml +++ b/.github/workflows/windows_conditional_compilation.yml @@ -29,7 +29,7 @@ jobs: skip_workflow: "${{ steps.smart_ci.outputs.skip_workflow }}" steps: - name: checkout action - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: sparse-checkout: .github/actions/smart-ci @@ -72,13 +72,13 @@ jobs: steps: - name: Clone OpenVINO - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: path: 'openvino' submodules: 'true' - name: Clone test models - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: repository: 'openvinotoolkit/testdata' path: 'testdata' @@ -278,13 +278,13 @@ jobs: steps: - name: Clone OpenVINO - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: path: 'openvino' submodules: 'true' - name: Clone test models - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: repository: 'openvinotoolkit/testdata' path: 'testdata' @@ -319,6 +319,7 @@ jobs: -DENABLE_OV_TF_LITE_FRONTEND=OFF ` -DENABLE_OV_PADDLE_FRONTEND=OFF ` -DENABLE_OV_PYTORCH_FRONTEND=OFF ` + -DENABLE_OV_JAX_FRONTEND=OFF ` -DENABLE_OV_ONNX_FRONTEND=OFF ` -DSELECTIVE_BUILD_STAT=${{ env.SELECTIVE_BUILD_STAT_DIR }}\*.csv ` -S ${{ env.OPENVINO_REPO }} ` @@ -363,7 +364,7 @@ jobs: run: Expand-Archive ${{ env.INSTALL_TEST_DIR }}/openvino_tests.zip -DestinationPath "${{ env.INSTALL_TEST_DIR }}" - name: Fetch setup_python action - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: sparse-checkout: | .github/actions/setup_python/action.yml diff --git a/.github/workflows/workflow_rerunner.yml b/.github/workflows/workflow_rerunner.yml index bb8ddfd6e64c0d..81af7aede704ea 100644 --- a/.github/workflows/workflow_rerunner.yml +++ b/.github/workflows/workflow_rerunner.yml @@ -30,7 +30,7 @@ jobs: checks: read steps: - name: Checkout - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: sparse-checkout: '.github/scripts/workflow_rerun' @@ -61,7 +61,7 @@ jobs: runs-on: aks-linux-2-cores-8gb steps: - name: Checkout - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: sparse-checkout: '.github/scripts/workflow_rerun' lfs: true diff --git a/CMakeLists.txt b/CMakeLists.txt index adfdc58a8de05b..494a3e4ce21d61 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -41,7 +41,10 @@ if(POLICY CMP0149) cmake_policy(SET CMP0149 NEW) endif() -project(OpenVINO DESCRIPTION "OpenVINO toolkit") +project(OpenVINO + DESCRIPTION "OpenVINO toolkit" + HOMEPAGE_URL "https://docs.openvino.ai/2024/home.html" + LANGUAGES C CXX) find_package(OpenVINODeveloperScripts REQUIRED PATHS "${OpenVINO_SOURCE_DIR}/cmake/developer_package" diff --git a/README.md b/README.md index b56a01911f6546..546a9e1a54ca57 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ Check out the [OpenVINO Cheat Sheet](https://docs.openvino.ai/2024/_static/downl [Get your preferred distribution of OpenVINO](https://docs.openvino.ai/2024/get-started/install-openvino.html) or use this command for quick installation: ```sh -pip install openvino +pip install -U openvino ``` Check [system requirements](https://docs.openvino.ai/2024/about-openvino/system-requirements.html) and [supported devices](https://docs.openvino.ai/2024/about-openvino/compatibility-and-support/supported-devices.html) for detailed information. diff --git a/cmake/coverage.cmake b/cmake/coverage.cmake index d58200400c904f..061d6c493cdf41 100644 --- a/cmake/coverage.cmake +++ b/cmake/coverage.cmake @@ -112,6 +112,13 @@ if(ENABLE_OV_IR_FRONTEND) PREFIX "${OV_COVERAGE_BASE_DIRECTORY}") endif() +if(ENABLE_OV_JAX_FRONTEND) + ov_coverage_extract(INPUT "openvino" OUTPUT "jax_frontend" + PATTERNS "${OV_COVERAGE_BASE_DIRECTORY}/src/frontends/jax/*") + ov_coverage_genhtml(INFO_FILE "jax_frontend" + PREFIX "${OV_COVERAGE_BASE_DIRECTORY}") +endif() + if(ENABLE_OV_ONNX_FRONTEND) ov_coverage_extract(INPUT "openvino" OUTPUT "onnx_frontend" PATTERNS "${OV_COVERAGE_BASE_DIRECTORY}/src/frontends/onnx/*") diff --git a/cmake/dependencies.cmake b/cmake/dependencies.cmake index 69dedb3e3bd527..219d464682b016 100644 --- a/cmake/dependencies.cmake +++ b/cmake/dependencies.cmake @@ -89,10 +89,10 @@ function(ov_download_tbb) # TODO: add target_path to be platform specific as well, to avoid following if # build oneTBB 2021.2.1 with Visual Studio 2019 (MSVC 14.21) RESOLVE_DEPENDENCY(TBB - ARCHIVE_WIN "oneapi-tbb-2021.2.2-win.zip" + ARCHIVE_WIN "oneapi-tbb-2021.2.5-win-trim.zip" TARGET_PATH "${TEMP}/tbb" ENVIRONMENT "TBBROOT" - SHA256 "103b19a8af288c6a7d83ed3f0d2239c4afd0dd189fc12aad1d34b3c9e78df94b" + SHA256 "a9384a25861946648db58f7f294c4aa2b1be7ae52748024fef3e13ca2762a1ba" USE_NEW_LOCATION TRUE) elseif(ANDROID AND X86_64) RESOLVE_DEPENDENCY(TBB @@ -104,10 +104,10 @@ function(ov_download_tbb) elseif(LINUX AND X86_64 AND OPENVINO_GNU_LIBC AND OV_LIBC_VERSION VERSION_GREATER_EQUAL 2.17) # build oneTBB 2021.2.1 with gcc 4.8 (glibc 2.17) RESOLVE_DEPENDENCY(TBB - ARCHIVE_LIN "oneapi-tbb-2021.2.4-lin.tgz" + ARCHIVE_LIN "oneapi-tbb-2021.2.5-lin-trim.tgz" TARGET_PATH "${TEMP}/tbb" ENVIRONMENT "TBBROOT" - SHA256 "6523661559a340e88131472ea9a595582c306af083e55293b7357d11b8015546" + SHA256 "9bea2c838df3085d292989d643523dc1cedce9b46d5a03eec90104151b49a180" USE_NEW_LOCATION TRUE) elseif(YOCTO_AARCH64) RESOLVE_DEPENDENCY(TBB @@ -119,34 +119,34 @@ function(ov_download_tbb) elseif(APPLE AND X86_64) # build oneTBB 2021.2.1 with OS version 11.4 RESOLVE_DEPENDENCY(TBB - ARCHIVE_MAC "oneapi-tbb-2021.2.1-mac.tgz" + ARCHIVE_MAC "oneapi-tbb-2021.2.5-mac-trim.tgz" TARGET_PATH "${TEMP}/tbb" ENVIRONMENT "TBBROOT" - SHA256 "c57ce4b97116cd3093c33e6dcc147fb1bbb9678d0ee6c61a506b2bfe773232cb" + SHA256 "5a6035fcf7b9d3bd8183ecc31b3e2b2026a749152b5b879f8e4d147e09479efc" USE_NEW_LOCATION TRUE) elseif(WIN32 AND AARCH64) # build oneTBB 2021.2.1 with Visual Studio 2022 (MSVC 14.35) RESOLVE_DEPENDENCY(TBB - ARCHIVE_WIN "oneapi-tbb-2021.2.1-win-arm64.zip" + ARCHIVE_WIN "oneapi-tbb-2021.2.5-win-arm64-trim.zip" TARGET_PATH "${TEMP}/tbb" ENVIRONMENT "TBBROOT" - SHA256 "09fe7f5e7be589aa34ccd20fdfd7cad9e0afa89d1e74ecdb008a75d0af71d6e1" + SHA256 "c26b7593e1808c2dd15a768cd4ea1ee14aa0aa2dacb210b86e326ab7960d2473" USE_NEW_LOCATION TRUE) elseif(LINUX AND AARCH64 AND OPENVINO_GNU_LIBC AND OV_LIBC_VERSION VERSION_GREATER_EQUAL 2.17) - # build oneTBB 2021.2.1 with gcc 4.8 (glibc 2.17) + # build oneTBB with glibc 2.17 RESOLVE_DEPENDENCY(TBB - ARCHIVE_LIN "oneapi-tbb-2021.2.5-lin-arm64.tgz" + ARCHIVE_LIN "oneapi-tbb-2021.13.0-rc1-lin-arm64-trim.tgz" TARGET_PATH "${TEMP}/tbb" ENVIRONMENT "TBBROOT" - SHA256 "86b24e4db254136bf88f0a8195ae478494e34db109e08569b1c059b174494865" + SHA256 "7fe49525217de9536980a820d90645784216ad4a61e11799b8c95129dcdeeecf" USE_NEW_LOCATION TRUE) elseif(APPLE AND AARCH64) - # build oneTBB 2021.2.1 with export MACOSX_DEPLOYMENT_TARGET=11.0 + # build oneTBB with export MACOSX_DEPLOYMENT_TARGET=11.0 RESOLVE_DEPENDENCY(TBB - ARCHIVE_MAC "oneapi-tbb-2021.2.5-mac-arm64.tgz" + ARCHIVE_MAC "oneapi-tbb-2021.13.0-rc1-mac-arm64-trim.tgz" TARGET_PATH "${TEMP}/tbb" ENVIRONMENT "TBBROOT" - SHA256 "bb1f84b1dcc50787f35c99b4b6ecea733d3068ca3467b5ebd2e369c4f7eccb53" + SHA256 "d3ce1c00e46a187baee459458e8d13d3421dc7242bff0c977b95d8d66d74441a" USE_NEW_LOCATION TRUE) else() message(WARNING "Prebuilt TBB is not available on current platform") diff --git a/cmake/developer_package/OpenVINODeveloperScriptsConfig.cmake b/cmake/developer_package/OpenVINODeveloperScriptsConfig.cmake index 904ad86e31d8f0..ee8111bd185d85 100644 --- a/cmake/developer_package/OpenVINODeveloperScriptsConfig.cmake +++ b/cmake/developer_package/OpenVINODeveloperScriptsConfig.cmake @@ -211,6 +211,8 @@ set(CMAKE_POLICY_DEFAULT_CMP0068 NEW) set(CMAKE_POLICY_DEFAULT_CMP0074 NEW) # CMake 3.13+: option() honors normal variables. set(CMAKE_POLICY_DEFAULT_CMP0077 NEW) +# CMake 3.15+: export(PACKAGE) does not populate package registry by default. +set(CMAKE_POLICY_DEFAULT_CMP0090 NEW) # CMake 3.15: Modules FindPython3, FindPython2 and FindPython use LOCATION for lookup strategy set(CMAKE_POLICY_DEFAULT_CMP0094 NEW) # CMake 3.19+: An imported target missing its location property fails during generation. @@ -222,6 +224,11 @@ set(CMAKE_POLICY_DEFAULT_CMP0135 NEW) # CMake 3.27+ :Visual Studio Generators select latest Windows SDK by default. set(CMAKE_POLICY_DEFAULT_CMP0149 NEW) +set(CMAKE_FIND_USE_PACKAGE_REGISTRY OFF CACHE BOOL "Disables search in user / system package registries") +set(CMAKE_FIND_PACKAGE_NO_PACKAGE_REGISTRY ON CACHE BOOL "Disables search in user package registries") +set(CMAKE_FIND_PACKAGE_NO_SYSTEM_PACKAGE_REGISTRY ON CACHE BOOL "Disables search in system package registries") +set(CMAKE_EXPORT_PACKAGE_REGISTRY OFF CACHE BOOL "Disables package registry. Required for 3rd party projects like rapidjson, gflags") +set(CMAKE_EXPORT_NO_PACKAGE_REGISTRY ON CACHE BOOL "Disables package registry. Required for 3rd party projects like rapidjson, gflags") set(CMAKE_WARN_DEPRECATED OFF CACHE BOOL "Don't warn about obsolete cmake versions in 3rdparty") set(CMAKE_WARN_ON_ABSOLUTE_INSTALL_DESTINATION ON CACHE BOOL "Warn about absolute paths in destination") diff --git a/cmake/developer_package/api_validator/api_validator.cmake b/cmake/developer_package/api_validator/api_validator.cmake index 0fd1f2c03a3343..ff6b0fbaa061e9 100644 --- a/cmake/developer_package/api_validator/api_validator.cmake +++ b/cmake/developer_package/api_validator/api_validator.cmake @@ -2,7 +2,11 @@ # SPDX-License-Identifier: Apache-2.0 # -if(WIN32) +macro(ov_search_api_validator) + if(NOT ENABLE_API_VALIDATOR) + return() + endif() + # CMAKE_VS_WINDOWS_TARGET_PLATFORM_VERSION is only set when # Visual Studio generators are used, but we need it # when we use Ninja as well @@ -12,7 +16,7 @@ if(WIN32) set(CMAKE_VS_WINDOWS_TARGET_PLATFORM_VERSION ${WINDOWS_SDK_VERSION}) message(STATUS "Use ${CMAKE_VS_WINDOWS_TARGET_PLATFORM_VERSION} Windows SDK version") else() - message(FATAL_ERROR "WindowsSDKVersion environment variable is not set,\ + message(FATAL_ERROR "WindowsSDKVersion environment variable is not set,\ can't find Windows SDK version. Try to use vcvarsall.bat script") endif() endif() @@ -41,6 +45,11 @@ can't find Windows SDK version. Try to use vcvarsall.bat script") message(STATUS "Found apivalidator: ${ONECORE_API_VALIDATOR}") endif() endif() +endmacro() + + +if(ENABLE_API_VALIDATOR) + ov_search_api_validator() endif() function(_ov_add_api_validator_post_build_step_recursive) diff --git a/cmake/developer_package/compile_flags/os_flags.cmake b/cmake/developer_package/compile_flags/os_flags.cmake index f846d8a0050afc..a49dce9bce7b50 100644 --- a/cmake/developer_package/compile_flags/os_flags.cmake +++ b/cmake/developer_package/compile_flags/os_flags.cmake @@ -75,11 +75,10 @@ macro(ov_dev_package_no_errors) endif() endif() - if (CMAKE_COMPILE_WARNING_AS_ERROR AND WIN32) + if(CMAKE_COMPILE_WARNING_AS_ERROR AND WIN32) if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") if(CMAKE_VERSION VERSION_LESS 3.24) - string(REPLACE "/WX" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") - string(REPLACE "/WX" "" CMAKE_C_FLAGS "${CMAKE_C_FLAGS}") + ov_add_compiler_flags(/WX-) endif() string(REPLACE "/WX" "" CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS}") endif() diff --git a/cmake/developer_package/compile_flags/sdl.cmake b/cmake/developer_package/compile_flags/sdl.cmake index ce47c9cd552f93..d399ec4f62fe0d 100644 --- a/cmake/developer_package/compile_flags/sdl.cmake +++ b/cmake/developer_package/compile_flags/sdl.cmake @@ -48,6 +48,7 @@ if(CMAKE_COMPILER_IS_GNUCXX OR OV_COMPILER_IS_CLANG OR OV_COMPILER_IS_INTEL_LLVM endif() elseif(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") set(OV_C_CXX_FLAGS "${OV_C_CXX_FLAGS} /sdl /guard:cf") + set(OV_LINKER_FLAGS "${OV_LINKER_FLAGS} /guard:cf") endif() if(ENABLE_QSPECTRE) diff --git a/cmake/developer_package/cross_compile/python_helpers.cmake b/cmake/developer_package/cross_compile/python_helpers.cmake index a6c7de739375f1..050c99db9a852f 100644 --- a/cmake/developer_package/cross_compile/python_helpers.cmake +++ b/cmake/developer_package/cross_compile/python_helpers.cmake @@ -49,6 +49,19 @@ endfunction() # Wrapper for find_package(Python3) to allow cross-compilation macro(ov_find_python3 find_package_mode) + # Settings for FindPython3.cmake + if(NOT DEFINED Python3_USE_STATIC_LIBS) + set(Python3_USE_STATIC_LIBS OFF) + endif() + + if(NOT DEFINED Python3_FIND_VIRTUALENV) + set(Python3_FIND_VIRTUALENV FIRST) + endif() + + if(NOT DEFINED Python3_FIND_IMPLEMENTATIONS) + set(Python3_FIND_IMPLEMENTATIONS CPython PyPy) + endif() + if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.18) set(python3_development_component Development.Module) else() diff --git a/cmake/developer_package/features.cmake b/cmake/developer_package/features.cmake index da2fbf6af4f5fe..d2fdcd4cf12049 100644 --- a/cmake/developer_package/features.cmake +++ b/cmake/developer_package/features.cmake @@ -5,8 +5,11 @@ include(options) include(target_flags) -set (CPACK_GENERATOR "TGZ" CACHE STRING "Cpack generator for OpenVINO") -list (APPEND OV_OPTIONS CPACK_GENERATOR) +if(WIN32) + set (CPACK_GENERATOR "ZIP" CACHE STRING "Cpack generator for OpenVINO") +else() + set (CPACK_GENERATOR "TGZ" CACHE STRING "Cpack generator for OpenVINO") +endif() ov_dependent_option (ENABLE_LTO "Enable Link Time Optimization" OFF "LINUX;NOT ARM;CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 4.9" OFF) diff --git a/cmake/developer_package/frontends/frontends.cmake b/cmake/developer_package/frontends/frontends.cmake index 0b959322950106..87efaea0ceb387 100644 --- a/cmake/developer_package/frontends/frontends.cmake +++ b/cmake/developer_package/frontends/frontends.cmake @@ -319,7 +319,7 @@ macro(ov_add_frontend) if(OV_FRONTEND_LINKABLE_FRONTEND) set(export_set EXPORT OpenVINOTargets) set(archive_dest ARCHIVE DESTINATION ${OV_CPACK_ARCHIVEDIR} COMPONENT ${dev_component} ${OV_CPACK_COMP_CORE_DEV_EXCLUDE_ALL}) - set(namelink NAMELINK_COMPONENT ${dev_component}) + set(namelink NAMELINK_COMPONENT ${OV_CPACK_COMP_LINKS} ${OV_CPACK_COMP_LINKS_EXCLUDE_ALL}) else() set(namelink NAMELINK_SKIP) endif() diff --git a/cmake/developer_package/ncc_naming_style/openvino.style b/cmake/developer_package/ncc_naming_style/openvino.style index 141f02b9ab5808..6c82ccb9b8c55a 100644 --- a/cmake/developer_package/ncc_naming_style/openvino.style +++ b/cmake/developer_package/ncc_naming_style/openvino.style @@ -1,6 +1,6 @@ # custom OpenVINO values CppMethod: '^(operator\W+|[a-z_\d]+|signaling_NaN|quiet_NaN|OPENVINO_OP)$' -ClassName: '^([A-Z][\w]+|b?float16|float8_e4m3|float8_e5m2|numeric_limits|ngraph_error|stopwatch|unsupported_op)$' +ClassName: '^([A-Z][\w]+|b?float16|float8_e4m3|float8_e5m2|float4_e2m1|float8_e8m0|numeric_limits|ngraph_error|stopwatch|unsupported_op)$' StructName: '^([A-Z][\w]+|element_type_traits|hash|oi_pair|stat)$' FunctionName: '^(operator\W+|[a-z_\d]+)|PrintTo$' Namespace: '^([a-z\d_]*|InferenceEngine)$' @@ -18,7 +18,7 @@ VariableReference: '^\w+$' EnumName: '^[A-Z][\w]+$' # excepts element_type -EnumConstantName: '^([A-Z\d_]+|undefined|dynamic|boolean|bf16|f16|f32|f64|i4|i8|i16|i32|i64|u1|u2|u3|u4|u6|u8|u16|u32|u64|nf4|f8e4m3|f8e5m2|string|asymmetric|align_corners|round_prefer_floor|round_prefer_ceil|floor|ceil|simple|nearest|linear|linear_onnx|cubic|area|scales|sizes|half_pixel|tf_half_pixel_for_nn|pytorch_half_pixel|asymetric)$' +EnumConstantName: '^([A-Z\d_]+|undefined|dynamic|boolean|bf16|f16|f32|f64|i4|i8|i16|i32|i64|u1|u2|u3|u4|u6|u8|u16|u32|u64|nf4|f8e4m3|f8e5m2|f4e2m1|f8e8m0|string|asymmetric|align_corners|round_prefer_floor|round_prefer_ceil|floor|ceil|simple|nearest|linear|linear_onnx|cubic|area|scales|sizes|half_pixel|tf_half_pixel_for_nn|pytorch_half_pixel|asymetric)$' # TODO: align UsingDeclaration: '^.*$' TypedefName: '^.*$' diff --git a/cmake/developer_package/options.cmake b/cmake/developer_package/options.cmake index a54a8bed1d4e08..3ad42f156c706c 100644 --- a/cmake/developer_package/options.cmake +++ b/cmake/developer_package/options.cmake @@ -11,11 +11,13 @@ endif() macro(ov_option variable description value) option(${variable} "${description}" ${value}) list(APPEND OV_OPTIONS ${variable}) + set(OV_OPTIONS "${OV_OPTIONS}" CACHE INTERNAL "A list of OpenVINO cmake options") endmacro() macro(ov_dependent_option variable description def_value condition fallback_value) cmake_dependent_option(${variable} "${description}" ${def_value} "${condition}" ${fallback_value}) list(APPEND OV_OPTIONS ${variable}) + set(OV_OPTIONS "${OV_OPTIONS}" CACHE INTERNAL "A list of OpenVINO cmake options") endmacro() macro(ov_option_enum variable description value) @@ -29,6 +31,7 @@ macro(ov_option_enum variable description value) endif() list(APPEND OV_OPTIONS ${variable}) + set(OV_OPTIONS "${OV_OPTIONS}" CACHE INTERNAL "A list of OpenVINO cmake options") set(${variable} ${value} CACHE STRING "${description}") set_property(CACHE ${variable} PROPERTY STRINGS ${OPTION_ENUM_ALLOWED_VALUES}) diff --git a/cmake/developer_package/packaging/archive.cmake b/cmake/developer_package/packaging/archive.cmake index 19c6ac2ae5fc11..d4e9c65096d245 100644 --- a/cmake/developer_package/packaging/archive.cmake +++ b/cmake/developer_package/packaging/archive.cmake @@ -90,6 +90,10 @@ macro(ov_define_component_include_rules) # scripts unset(OV_CPACK_COMP_INSTALL_DEPENDENCIES_EXCLUDE_ALL) unset(OV_CPACK_COMP_SETUPVARS_EXCLUDE_ALL) + # pkgconfig + set(OV_CPACK_COMP_PKG_CONFIG_EXCLUDE_ALL ${OV_CPACK_COMP_CORE_DEV_EXCLUDE_ALL}) + # symbolic links + set(OV_CPACK_COMP_LINKS_EXCLUDE_ALL ${OV_CPACK_COMP_CORE_DEV_EXCLUDE_ALL}) endmacro() ov_define_component_include_rules() diff --git a/cmake/developer_package/packaging/common-libraries.cmake b/cmake/developer_package/packaging/common-libraries.cmake index c47c123ab4b31d..247f107b83b6fc 100644 --- a/cmake/developer_package/packaging/common-libraries.cmake +++ b/cmake/developer_package/packaging/common-libraries.cmake @@ -57,6 +57,9 @@ macro(ov_override_component_names) # merge C++ and C runtimes set(OV_CPACK_COMP_CORE_C "${OV_CPACK_COMP_CORE}") set(OV_CPACK_COMP_CORE_C_DEV "${OV_CPACK_COMP_CORE_DEV}") + # merge links and pkgconfig with dev component + set(OV_CPACK_COMP_LINKS "${OV_CPACK_COMP_CORE_DEV}") + set(OV_CPACK_COMP_PKG_CONFIG "${OV_CPACK_COMP_CORE_DEV}") endmacro() ov_override_component_names() @@ -104,6 +107,10 @@ macro(ov_define_component_include_rules) # scripts set(OV_CPACK_COMP_INSTALL_DEPENDENCIES_EXCLUDE_ALL EXCLUDE_FROM_ALL) set(OV_CPACK_COMP_SETUPVARS_EXCLUDE_ALL EXCLUDE_FROM_ALL) + # pkgconfig + set(OV_CPACK_COMP_PKG_CONFIG_EXCLUDE_ALL ${OV_CPACK_COMP_CORE_DEV_EXCLUDE_ALL}) + # symbolic links + set(OV_CPACK_COMP_LINKS_EXCLUDE_ALL ${OV_CPACK_COMP_CORE_DEV_EXCLUDE_ALL}) endmacro() ov_define_component_include_rules() diff --git a/cmake/developer_package/packaging/debian/debian.cmake b/cmake/developer_package/packaging/debian/debian.cmake index bdb005ab6c3706..1b29dd7697d1c7 100644 --- a/cmake/developer_package/packaging/debian/debian.cmake +++ b/cmake/developer_package/packaging/debian/debian.cmake @@ -60,6 +60,9 @@ macro(ov_override_component_names) # merge all C / C++ samples as a single samples component set(OV_CPACK_COMP_CPP_SAMPLES "samples") set(OV_CPACK_COMP_C_SAMPLES "${OV_CPACK_COMP_CPP_SAMPLES}") + # merge links and pkgconfig with dev component + set(OV_CPACK_COMP_LINKS "${OV_CPACK_COMP_CORE_DEV}") + set(OV_CPACK_COMP_PKG_CONFIG "${OV_CPACK_COMP_CORE_DEV}") endmacro() ov_override_component_names() @@ -111,6 +114,10 @@ macro(ov_define_component_include_rules) # scripts set(OV_CPACK_COMP_INSTALL_DEPENDENCIES_EXCLUDE_ALL EXCLUDE_FROM_ALL) set(OV_CPACK_COMP_SETUPVARS_EXCLUDE_ALL EXCLUDE_FROM_ALL) + # pkgconfig + set(OV_CPACK_COMP_PKG_CONFIG_EXCLUDE_ALL ${OV_CPACK_COMP_CORE_DEV_EXCLUDE_ALL}) + # symbolic links + set(OV_CPACK_COMP_LINKS_EXCLUDE_ALL ${OV_CPACK_COMP_CORE_DEV_EXCLUDE_ALL}) endmacro() ov_define_component_include_rules() diff --git a/cmake/developer_package/packaging/npm.cmake b/cmake/developer_package/packaging/npm.cmake index 6d7ca6d04e680a..996b55e0ba032f 100644 --- a/cmake/developer_package/packaging/npm.cmake +++ b/cmake/developer_package/packaging/npm.cmake @@ -4,7 +4,7 @@ include(GNUInstallDirs) -# We have to specify RPATH, all runtime libs are in one dir +# We have to specify RPATH, all runtime libs are in one dir set(CMAKE_SKIP_INSTALL_RPATH OFF) # @@ -32,6 +32,20 @@ endmacro() ov_npm_cpack_set_dirs() +# +# Override CPack components name for NPM generator +# This is needed to change the granularity, i.e. merge several components +# into a single one +# + +macro(ov_override_component_names) + # merge links and pkgconfig with dev component + set(OV_CPACK_COMP_LINKS "${OV_CPACK_COMP_CORE_DEV}") + set(OV_CPACK_COMP_PKG_CONFIG "${OV_CPACK_COMP_CORE_DEV}") +endmacro() + +ov_override_component_names() + # # Override include / exclude rules for components # This is required to exclude some files from installation @@ -67,6 +81,10 @@ macro(ov_define_component_include_rules) # scripts set(OV_CPACK_COMP_INSTALL_DEPENDENCIES_EXCLUDE_ALL EXCLUDE_FROM_ALL) set(OV_CPACK_COMP_SETUPVARS_EXCLUDE_ALL EXCLUDE_FROM_ALL) + # pkgconfig + unset(OV_CPACK_COMP_PKG_CONFIG_EXCLUDE_ALL) + # symbolic links + unset(OV_CPACK_COMP_LINKS_EXCLUDE_ALL) endmacro() ov_define_component_include_rules() diff --git a/cmake/developer_package/packaging/nsis.cmake b/cmake/developer_package/packaging/nsis.cmake index 9dcda7315dab61..b84569e4f0b1d1 100644 --- a/cmake/developer_package/packaging/nsis.cmake +++ b/cmake/developer_package/packaging/nsis.cmake @@ -87,6 +87,20 @@ endmacro() ov_nsis_cpack_set_dirs() +# +# Override CPack components name for NSIS generator +# This is needed to change the granularity, i.e. merge several components +# into a single one +# + +macro(ov_override_component_names) + # merge links and pkgconfig with dev component + set(OV_CPACK_COMP_LINKS "${OV_CPACK_COMP_CORE_DEV}") + set(OV_CPACK_COMP_PKG_CONFIG "${OV_CPACK_COMP_CORE_DEV}") +endmacro() + +ov_override_component_names() + # # Override include / exclude rules for components # This is required to exclude some files from installation @@ -122,6 +136,10 @@ macro(ov_define_component_include_rules) # scripts unset(OV_CPACK_COMP_INSTALL_DEPENDENCIES_EXCLUDE_ALL) unset(OV_CPACK_COMP_SETUPVARS_EXCLUDE_ALL) + # pkgconfig + set(OV_CPACK_COMP_PKG_CONFIG_EXCLUDE_ALL ${OV_CPACK_COMP_CORE_DEV_EXCLUDE_ALL}) + # symbolic links + set(OV_CPACK_COMP_LINKS_EXCLUDE_ALL ${OV_CPACK_COMP_CORE_DEV_EXCLUDE_ALL}) endmacro() ov_define_component_include_rules() diff --git a/cmake/developer_package/packaging/packaging.cmake b/cmake/developer_package/packaging/packaging.cmake index 41c0308acafba6..d62a8ca7fe2084 100644 --- a/cmake/developer_package/packaging/packaging.cmake +++ b/cmake/developer_package/packaging/packaging.cmake @@ -178,6 +178,10 @@ macro(ov_define_component_names) # scripts set(OV_CPACK_COMP_INSTALL_DEPENDENCIES "install_dependencies") set(OV_CPACK_COMP_SETUPVARS "setupvars") + # symbolic_links + set(OV_CPACK_COMP_LINKS "core_dev_links") + # pkgconfig + set(OV_CPACK_COMP_PKG_CONFIG "core_dev_pkgconfig") endmacro() ov_define_component_names() diff --git a/cmake/developer_package/packaging/rpm/rpm.cmake b/cmake/developer_package/packaging/rpm/rpm.cmake index 99f11730983f04..56a0a12647079c 100644 --- a/cmake/developer_package/packaging/rpm/rpm.cmake +++ b/cmake/developer_package/packaging/rpm/rpm.cmake @@ -51,6 +51,9 @@ macro(ov_override_component_names) # merge all C / C++ samples as a single samples component set(OV_CPACK_COMP_CPP_SAMPLES "samples") set(OV_CPACK_COMP_C_SAMPLES "${OV_CPACK_COMP_CPP_SAMPLES}") + # merge links and pkgconfig with dev component + set(OV_CPACK_COMP_LINKS "${OV_CPACK_COMP_CORE_DEV}") + set(OV_CPACK_COMP_PKG_CONFIG "${OV_CPACK_COMP_CORE_DEV}") endmacro() ov_override_component_names() @@ -102,6 +105,10 @@ macro(ov_define_component_include_rules) # scripts set(OV_CPACK_COMP_INSTALL_DEPENDENCIES_EXCLUDE_ALL EXCLUDE_FROM_ALL) set(OV_CPACK_COMP_SETUPVARS_EXCLUDE_ALL EXCLUDE_FROM_ALL) + # pkgconfig + set(OV_CPACK_COMP_PKG_CONFIG_EXCLUDE_ALL ${OV_CPACK_COMP_CORE_DEV_EXCLUDE_ALL}) + # symbolic links + set(OV_CPACK_COMP_LINKS_EXCLUDE_ALL ${OV_CPACK_COMP_CORE_DEV_EXCLUDE_ALL}) endmacro() ov_define_component_include_rules() diff --git a/cmake/features.cmake b/cmake/features.cmake index bfa3a05481d77f..4063e2f8545ced 100644 --- a/cmake/features.cmake +++ b/cmake/features.cmake @@ -137,6 +137,7 @@ endif() ov_option(ENABLE_OV_PADDLE_FRONTEND "Enable PaddlePaddle FrontEnd" ON) ov_option(ENABLE_OV_IR_FRONTEND "Enable IR FrontEnd" ON) ov_option(ENABLE_OV_PYTORCH_FRONTEND "Enable PyTorch FrontEnd" ON) +ov_option(ENABLE_OV_JAX_FRONTEND "Enable JAX FrontEnd" ON) ov_option(ENABLE_OV_IR_FRONTEND "Enable IR FrontEnd" ON) ov_option(ENABLE_OV_TF_FRONTEND "Enable TensorFlow FrontEnd" ON) ov_option(ENABLE_OV_TF_LITE_FRONTEND "Enable TensorFlow Lite FrontEnd" ON) @@ -192,6 +193,8 @@ ov_dependent_option(ENABLE_JS "Enables JS API building" ON "NOT ANDROID;NOT EMSC ov_option(ENABLE_OPENVINO_DEBUG "Enable output for OPENVINO_DEBUG statements" OFF) +ov_dependent_option (ENABLE_API_VALIDATOR "Enables API Validator usage" ON "WIN32" OFF) + if(NOT BUILD_SHARED_LIBS AND ENABLE_OV_TF_FRONTEND) set(FORCE_FRONTENDS_USE_PROTOBUF ON) else() diff --git a/cmake/packaging/debian.cmake b/cmake/packaging/debian.cmake index 90b01a196cec84..b97f8eea67c8a6 100644 --- a/cmake/packaging/debian.cmake +++ b/cmake/packaging/debian.cmake @@ -234,6 +234,19 @@ macro(ov_cpack_settings) set(ir_copyright "generic") endif() + if(ENABLE_OV_JAX_FRONTEND) + set(CPACK_COMPONENT_JAX_DESCRIPTION "OpenVINO JAX Frontend") + set(CPACK_COMPONENT_JAX_DEPENDS "${OV_CPACK_COMP_CORE}") + set(CPACK_DEBIAN_JAX_PACKAGE_NAME "libopenvino-jax-frontend-${cpack_name_ver}") + # since we JAX FE is linkable target, we need to call ldconfig (i.e. `def_triggers`) + set(CPACK_DEBIAN_JAX_PACKAGE_CONTROL_EXTRA "${def_postinst};${def_postrm};${def_triggers}") + ov_debian_add_lintian_suppression(jax + # we have different package name strategy; it suggests libopenvino-jax-frontend202230 + "package-name-doesnt-match-sonames") + list(APPEND frontends jax) + set(jax_copyright "generic") + endif() + if(ENABLE_OV_ONNX_FRONTEND) set(CPACK_COMPONENT_ONNX_DESCRIPTION "OpenVINO ONNX Frontend") set(CPACK_COMPONENT_ONNX_DEPENDS "${OV_CPACK_COMP_CORE}") diff --git a/cmake/packaging/rpm.cmake b/cmake/packaging/rpm.cmake index 6bbeba02bdc11f..4d06ebff496421 100644 --- a/cmake/packaging/rpm.cmake +++ b/cmake/packaging/rpm.cmake @@ -208,6 +208,15 @@ macro(ov_cpack_settings) set(ir_copyright "generic") endif() + if(ENABLE_OV_JAX_FRONTEND) + set(CPACK_COMPONENT_JAX_DESCRIPTION "OpenVINO JAX Frontend") + set(CPACK_RPM_JAX_PACKAGE_NAME "libopenvino-jax-frontend-${cpack_name_ver}") + set(CPACK_RPM_JAX_POST_INSTALL_SCRIPT_FILE "${def_triggers}") + set(CPACK_RPM_JAX_POST_UNINSTALL_SCRIPT_FILE "${def_triggers}") + _ov_add_package(frontend_packages jax) + set(jax_copyright "generic") + endif() + if(ENABLE_OV_ONNX_FRONTEND) set(CPACK_COMPONENT_ONNX_DESCRIPTION "OpenVINO ONNX Frontend") set(CPACK_RPM_ONNX_PACKAGE_NAME "libopenvino-onnx-frontend-${cpack_name_ver}") diff --git a/cmake/templates/OpenVINOConfig.cmake.in b/cmake/templates/OpenVINOConfig.cmake.in index ac527e969068ef..af8dd1596c4a39 100644 --- a/cmake/templates/OpenVINOConfig.cmake.in +++ b/cmake/templates/OpenVINOConfig.cmake.in @@ -10,6 +10,7 @@ # The following components are supported: # # * `Runtime`: OpenVINO C++ and C Core & Inference Runtime, frontend common +# * `JAX`: OpenVINO JAX frontend # * `ONNX`: OpenVINO ONNX frontend # * `Paddle`: OpenVINO Paddle frontend # * `PyTorch`: OpenVINO PyTorch frontend @@ -37,6 +38,9 @@ # # Frontend specific targets: # +# `openvino::frontend::jax` +# JAX FrontEnd target (optional) +# # `openvino::frontend::onnx` # ONNX FrontEnd target (optional) # @@ -63,6 +67,9 @@ # `OpenVINO_Runtime_FOUND` # OpenVINO C++ Core & Inference Runtime is available # +# `OpenVINO_Frontend_JAX_FOUND` +# OpenVINO JAX frontend is available +# # `OpenVINO_Frontend_ONNX_FOUND` # OpenVINO ONNX frontend is available # @@ -510,7 +517,7 @@ unset(_OV_ENABLE_OPENVINO_BUILD_SHARED) set(_ov_imported_libs openvino::runtime openvino::runtime::c openvino::frontend::onnx openvino::frontend::paddle openvino::frontend::tensorflow - openvino::frontend::pytorch openvino::frontend::tensorflow_lite) + openvino::frontend::pytorch openvino::frontend::tensorflow_lite openvino::frontend::jax) if(_ov_as_external_package) foreach(target IN LISTS _ov_imported_libs) @@ -549,6 +556,7 @@ set(${CMAKE_FIND_PACKAGE_NAME}_TensorFlow_FOUND @ENABLE_OV_TF_FRONTEND@) set(${CMAKE_FIND_PACKAGE_NAME}_TensorFlowLite_FOUND @ENABLE_OV_TF_LITE_FRONTEND@) set(${CMAKE_FIND_PACKAGE_NAME}_IR_FOUND @ENABLE_OV_IR_FRONTEND@) set(${CMAKE_FIND_PACKAGE_NAME}_PyTorch_FOUND @ENABLE_OV_PYTORCH_FRONTEND@) +set(${CMAKE_FIND_PACKAGE_NAME}_JAX_FOUND @ENABLE_OV_JAX_FRONTEND@) set(${CMAKE_FIND_PACKAGE_NAME}_Frontend_ONNX_FOUND ${${CMAKE_FIND_PACKAGE_NAME}_ONNX_FOUND}) set(${CMAKE_FIND_PACKAGE_NAME}_Frontend_Paddle_FOUND ${${CMAKE_FIND_PACKAGE_NAME}_Paddle_FOUND}) @@ -556,6 +564,7 @@ set(${CMAKE_FIND_PACKAGE_NAME}_Frontend_TensorFlow_FOUND ${${CMAKE_FIND_PACKAGE_ set(${CMAKE_FIND_PACKAGE_NAME}_Frontend_TensorFlowLite_FOUND ${${CMAKE_FIND_PACKAGE_NAME}_TensorFlowLite_FOUND}) set(${CMAKE_FIND_PACKAGE_NAME}_Frontend_IR_FOUND ${${CMAKE_FIND_PACKAGE_NAME}_IR_FOUND}) set(${CMAKE_FIND_PACKAGE_NAME}_Frontend_PyTorch_FOUND ${${CMAKE_FIND_PACKAGE_NAME}_PyTorch_FOUND}) +set(${CMAKE_FIND_PACKAGE_NAME}_Frontend_JAX_FOUND ${${CMAKE_FIND_PACKAGE_NAME}_JAX_FOUND}) # if no components specified, only Runtime is provided if(NOT ${CMAKE_FIND_PACKAGE_NAME}_FIND_COMPONENTS) @@ -591,5 +600,6 @@ unset(${CMAKE_FIND_PACKAGE_NAME}_ONNX_FOUND) unset(${CMAKE_FIND_PACKAGE_NAME}_TensorFlow_FOUND) unset(${CMAKE_FIND_PACKAGE_NAME}_TensorFlowLite_FOUND) unset(${CMAKE_FIND_PACKAGE_NAME}_PyTorch_FOUND) +unset(${CMAKE_FIND_PACKAGE_NAME}_JAX_FOUND) cmake_policy(POP) diff --git a/docs/CMakeLists.txt b/docs/CMakeLists.txt index 8b57988ca14631..797c95ef7d91c5 100644 --- a/docs/CMakeLists.txt +++ b/docs/CMakeLists.txt @@ -33,6 +33,7 @@ function(build_docs) set(DOXYGEN_MAPPING_SCRIPT "${SCRIPTS_DIR}/create_mapping.py") set(DOCS_MAPPING_SCRIPT "${SCRIPTS_DIR}/create_doc_mapping.py") set(BREATHE_APIDOC_SCRIPT "${SCRIPTS_DIR}/apidoc.py") + set(OV_INSTALLATION_SCRIPT "${SCRIPTS_DIR}/install_appropriate_openvino_version.py") # Doxygen/Sphinx setup set(DOXYGEN_XML_OUTPUT "${DOCS_BUILD_DIR}/xml") @@ -62,7 +63,9 @@ function(build_docs) if(${ENABLE_PYTHON_API}) list(APPEND commands COMMAND ${CMAKE_COMMAND} -E cmake_echo_color --green "STARTED preprocessing OpenVINO Python API") - list(APPEND commands COMMAND ${Python3_EXECUTABLE} -m pip install openvino) + list(APPEND commands COMMAND ${Python3_EXECUTABLE} ${OV_INSTALLATION_SCRIPT} + --ov_dir=${SPHINX_SETUP_DIR} + --python=${Python3_EXECUTABLE}) list(APPEND commands COMMAND ${CMAKE_COMMAND} -E cmake_echo_color --green "FINISHED preprocessing OpenVINO Python API") endif() diff --git a/docs/articles_en/about-openvino/compatibility-and-support.rst b/docs/articles_en/about-openvino/compatibility-and-support.rst index f2c467be34676e..f9ea2b2b34ba90 100644 --- a/docs/articles_en/about-openvino/compatibility-and-support.rst +++ b/docs/articles_en/about-openvino/compatibility-and-support.rst @@ -1,5 +1,3 @@ -.. {#compatibility_and_support} - Compatibility and Support ========================= @@ -9,12 +7,15 @@ Compatibility and Support :hidden: compatibility-and-support/supported-devices + compatibility-and-support/supported-models compatibility-and-support/supported-operations-inference-devices compatibility-and-support/supported-operations-framework-frontend :doc:`Supported Devices ` - compatibility information for supported hardware accelerators. +:doc:`Supported Models ` - a list of selected models confirmed to work with given hardware. + :doc:`Supported Operations ` - a listing of framework layers supported by OpenVINO. :doc:`Supported Operations ` - a listing of layers supported by OpenVINO inference devices. diff --git a/docs/articles_en/about-openvino/compatibility-and-support/supported-devices.rst b/docs/articles_en/about-openvino/compatibility-and-support/supported-devices.rst index e8a6e96af27626..fbb3b948165dd2 100644 --- a/docs/articles_en/about-openvino/compatibility-and-support/supported-devices.rst +++ b/docs/articles_en/about-openvino/compatibility-and-support/supported-devices.rst @@ -77,14 +77,6 @@ Feature Support and API Coverage | || as of OpenVINO 2023.3, 08 Jan, 2024. | +-------------------------+-----------+------------------+-------------------+ - -Devices similar to the ones used for benchmarking can be accessed using -`Intel® DevCloud for the Edge `__, -a remote development environment with access to Intel® hardware and the latest versions -of the Intel® Distribution of OpenVINO™ Toolkit. -`Learn more `__ or -`Register here `__. - For setting up a relevant configuration, refer to the :doc:`Integrate with Customer Application <../../openvino-workflow/running-inference/integrate-openvino-with-your-application>` topic (step 3 "Configure input and output"). diff --git a/docs/articles_en/about-openvino/compatibility-and-support/supported-models.rst b/docs/articles_en/about-openvino/compatibility-and-support/supported-models.rst new file mode 100644 index 00000000000000..968c5b6571bca9 --- /dev/null +++ b/docs/articles_en/about-openvino/compatibility-and-support/supported-models.rst @@ -0,0 +1,45 @@ +AI Models that run with OpenVINO +===================================== + +This list of 500+ computer vision, LLM, and GenAI models includes pre-optimized OpenVINO +models/notebooks plus models from public model zoos (ONNX Model Zoo, Pytorch Model Zoo, and +HuggingFace). This list is not comprehensive and only includes models tested by Intel. + +**AI Models that run on Intel® Core Ultra ™ Processors with OpenVINO™ toolkit:** + +.. raw:: html + + +
+ + + + + +.. csv-table:: + :class: modeldata stripe + :name: supportedModelsTable + :header-rows: 1 + :file: ../../_static/download/supported_models.csv + + +| Note: +| The results as of June 17 2024, for OpenVINO version 2024.2. + +| The validation process involved using the OpenVINO toolkit (natively or as a backend) to load + each model onto the designated hardware and successfully execute inference without encountering + any errors or failures. These successfully run models are indicated with a check mark below and + blanks are not tested. In the precision column where it is listed as optimum-intel default, + that corresponds to FP32 for small models and INT8 for models greater than 1B parameters. diff --git a/docs/articles_en/about-openvino/performance-benchmarks.rst b/docs/articles_en/about-openvino/performance-benchmarks.rst index 39f22b5894d742..ced358db18c044 100644 --- a/docs/articles_en/about-openvino/performance-benchmarks.rst +++ b/docs/articles_en/about-openvino/performance-benchmarks.rst @@ -10,7 +10,7 @@ Performance Benchmarks :maxdepth: 1 :hidden: - performance-benchmarks/generativeAI-benchmarks + Efficient LLMs for AI PC performance-benchmarks/performance-benchmarks-faq OpenVINO Accuracy performance-benchmarks/getting-performance-numbers @@ -23,7 +23,7 @@ selection of public neural networks and Intel® devices. The results may help yo hardware to use in your applications or plan AI workload for the hardware you have already implemented in your solutions. Click the buttons below to see the chosen benchmark data. For more detailed view of performance numbers for generative AI models, check the -:doc:`Generative AI Benchmark Results <./performance-benchmarks/generativeAI-benchmarks>` +:doc:`Generative AI Benchmark Results <./performance-benchmarks/generative-ai-performance>` .. grid:: 1 1 2 2 :gutter: 4 @@ -127,21 +127,21 @@ For a listing of all platforms and configurations used for testing, refer to the .. grid-item:: - .. button-link:: ../_static/benchmarks_files/OV-2024.1-platform_list.pdf + .. button-link:: ../_static/benchmarks_files/OV-2024.2-platform_list.pdf :color: primary :outline: :expand: :material-regular:`download;1.5em` Click for Hardware Platforms [PDF] - .. button-link:: ../_static/benchmarks_files/OV-2024.1-system-info-detailed.xlsx + .. button-link:: ../_static/benchmarks_files/OV-2024.2-system-info-detailed.xlsx :color: primary :outline: :expand: :material-regular:`download;1.5em` Click for Configuration Details [XLSX] - .. button-link:: ../_static/benchmarks_files/OV-2024.1-Performance-Data.xlsx + .. button-link:: ../_static/benchmarks_files/OV-2024.2-Performance-Data.xlsx :color: primary :outline: :expand: @@ -199,25 +199,16 @@ connection is dedicated only to measuring performance. You can also test performance for your system yourself, following the guide on :doc:`getting performance numbers `. -Performance of a particular application can also be evaluated virtually using -`Intel® DevCloud for the Edge `__. -It is a remote development environment with access to Intel® hardware and the latest versions -of the Intel® Distribution of the OpenVINO™ Toolkit. To learn more about it, visit -`the website `__ -or -`create an account `__. - - .. raw:: html

Disclaimers

* Intel® Distribution of OpenVINO™ toolkit performance results are based on release - 2024.1, as of April 17, 2024. + 2024.2, as of June 17, 2024. * OpenVINO Model Server performance results are based on release - 2024.0, as of March 15, 2024. + 2024.1, as of April 26, 2024. The results may not reflect all publicly available updates. Intel technologies' features and benefits depend on system configuration and may require enabled hardware, software, or service diff --git a/docs/articles_en/about-openvino/performance-benchmarks/generative-ai-performance.rst b/docs/articles_en/about-openvino/performance-benchmarks/generative-ai-performance.rst new file mode 100644 index 00000000000000..03588eeb441f38 --- /dev/null +++ b/docs/articles_en/about-openvino/performance-benchmarks/generative-ai-performance.rst @@ -0,0 +1,64 @@ +Most Efficient Large Language Models for AI PC +============================================== + +The table below lists key performance indicators for a selection of Large Language Models +running on an Intel® Core™ Ultra 7-165H based system. + +.. raw:: html + + +

+ + + + + + + + + + +.. csv-table:: + :class: modeldata stripe + :name: supportedModelsTable + :header-rows: 1 + :file: ../../_static/llm_models.csv + + +This page is regularly updated to help you identify the best-performing LLMs on the +Intel® Core™ Ultra processor family and AI PCs. + +For complete information on the system config, see: +`Hardware Platforms [PDF] `__ \ No newline at end of file diff --git a/docs/articles_en/about-openvino/performance-benchmarks/generativeAI-benchmarks.rst b/docs/articles_en/about-openvino/performance-benchmarks/generativeAI-benchmarks.rst deleted file mode 100644 index 3c6365e56b25be..00000000000000 --- a/docs/articles_en/about-openvino/performance-benchmarks/generativeAI-benchmarks.rst +++ /dev/null @@ -1,8 +0,0 @@ -Generative AI Benchmark Results -=================================== - -This page will give you a detailed information on how OpenVINO performs running a selection of -Generative AI models. - - - diff --git a/docs/articles_en/about-openvino/performance-benchmarks/model-accuracy-int8-fp32.rst b/docs/articles_en/about-openvino/performance-benchmarks/model-accuracy-int8-fp32.rst index 4f080887742aeb..4c15d7ddc75499 100644 --- a/docs/articles_en/about-openvino/performance-benchmarks/model-accuracy-int8-fp32.rst +++ b/docs/articles_en/about-openvino/performance-benchmarks/model-accuracy-int8-fp32.rst @@ -29,73 +29,73 @@ the table for more information. * - bert-base-cased - SST-2_bert_cased_padded - spearman@cosine - - 2.93% - - 2.68% - - 2.76% - - 2.72% + - 3.33% + - 3.22% + - 3.69% + - 3.28% * - bert-large-uncased-whole-word-masking-squad-0001 - SQUAD_v1_1_bert_msl384_mql64_ds128_lowercase - F1 - 0.19% - - -0.03% + - 0.06% - 0.03% - 0.11% * - efficientdet-d0 - COCO2017_detection_91cl - coco_precision - - -0.84% - - -0.64% - - -0.62% + - -0.90% - -0.63% + - -0.61% + - -0.62% * - mask_rcnn_resnet50_atrous_coco - COCO2017_detection_91cl_bkgr - coco_orig_precision - -0.04% - - 0.02% - - 0.04% + - -0.03% - 0.04% + - 0.02% * - mobilenet-v2 - ImageNet2012 - accuracy @ top1 - - - -0.97% - - -0.97% + - -0.87% + - -0.89% - -0.95% * - resnet-50 - ImageNet2012 - accuracy @ top1 - - -0.09% - - -0.12% + - -0.20% + - -0.18% + - -0.18% - -0.13% - - -0.19% * - ssd-resnet34-1200 - COCO2017_detection_80cl_bkgr - map + - -0.03% - -0.02% - - -0.01% - - -0.02% - - 0.04% + - -0.03% + - -0.03% * - ssd-mobilenet-v1-coco - COCO2017_detection_80cl_bkgr - coco-precision - - -2.97% - - -0.29% - - -0.31% - - -0.26% + - -2.75% + - -0.11% + - -0.11% + - -0.08% * - unet-camvid-onnx-0001 - CamVid_12cl - mean_iou @ mean - -6.28% - - 6.41% + - 6.45% - 6.46% - 6.40% * - yolo_v3_tiny - COCO2017_detection_80cl - map - - -0.30% - - -0.43% - - -0.43% - - -0.87% + - -0.57% + - -0.58% + - -0.58% + - -0.70% * - yolo_v8n - COCO2017_detection_80cl - map @@ -104,18 +104,18 @@ the table for more information. - 0.04% - -0.08% * - chatGLM2-6b - - lambada openai + - Ceval-valid-high_school_history - ppl - - 0.75 - 0.75 - * - Llama-2-7b-chat - - Wiki, StackExch, Crawl + - Ceval-valid-high_school_history - ppl - - - 3.38 - - 3.27 + - 0.55 + - 0.25 - * - Stable-Diffusion-V2-1 - LIAON-5B @@ -125,18 +125,18 @@ the table for more information. - - * - Mistral-7b - - proprietary Mistral.ai + - Wikitext - ppl - - - 3.49 - - 3.19 + - 8.10 + - 8.10 - * - Falcon-7b-instruct - - Bai Ze (65%), GPT4All (25%), GPTeacher (5%), RefinedWeb-english (5%) + - Wikitext - ppl - - - - - + - 14.54 + - 14.55 - .. list-table:: Model Accuracy for BF16, FP32 and FP16 (FP16: Flex-170 only. BF16: Xeon(R) 8490H only) @@ -165,7 +165,7 @@ the table for more information. - 0.04% - 0.04% - 0.06% - - 0.05% + - 0.04% * - efficientdet-d0 - COCO2017_detection_91cl - coco_precision @@ -178,7 +178,7 @@ the table for more information. - COCO2017_detection_91cl_bkgr - coco_orig_precision - -0.01% - - -0.02% + - -0.01% - -0.01% - 0.09% - 0.00% @@ -205,7 +205,7 @@ the table for more information. - 0.00% - 0.00% - -0.02% - - 0.02% + - 0.00% * - ssd-mobilenet-v1-coco - COCO2017_detection_80cl_bkgr - coco-precision @@ -239,7 +239,7 @@ the table for more information. - 0.04% - -0.02% * - chatGLM2-6b - - lambada openai + - Ceval-valid-high_school_history - ppl - - 0.75 @@ -247,11 +247,11 @@ the table for more information. - - * - Llama-2-7b-chat - - Wiki, StackExch, Crawl + - Wikitext - ppl - - - 3.26 - - 3.26 + - 0.30 + - 0.55 - - * - Stable-Diffusion-V2-1 @@ -263,19 +263,19 @@ the table for more information. - - * - Mistral-7b - - proprietary Mistral.ai + - Wikitext - ppl - - - 3.18 - - 3.19 + - 8.09 + - 8.09 - - * - Falcon-7b-instruct - - Bai Ze (65%), GPT4All (25%), GPTeacher (5%), RefinedWeb-english (5%) + - Wikitext - ppl - - - - - + - 14.51 + - 14.51 - - diff --git a/docs/articles_en/about-openvino/performance-benchmarks/performance-benchmarks-faq.rst b/docs/articles_en/about-openvino/performance-benchmarks/performance-benchmarks-faq.rst index 2652626a5740df..6c731dc374010b 100644 --- a/docs/articles_en/about-openvino/performance-benchmarks/performance-benchmarks-faq.rst +++ b/docs/articles_en/about-openvino/performance-benchmarks/performance-benchmarks-faq.rst @@ -1,5 +1,3 @@ -.. {#openvino_docs_performance_benchmarks_faq} - Performance Information F.A.Q. ============================== @@ -19,7 +17,13 @@ Performance Information F.A.Q. .. dropdown:: Where can I find the models used in the performance benchmarks? - All models used are included in the GitHub repository of :doc:`Open Model Zoo <../../documentation/legacy-features/model-zoo>`. + All models used are included in the GitHub repository of + :doc:`Open Model Zoo <../../documentation/legacy-features/model-zoo>`. + + .. important:: + + Due to the deprecation of Open Model Zoo, models in the OpenVINO IR format are now + published on `Hugging Face `__. .. dropdown:: Will there be any new models added to the list used for benchmarking? @@ -119,8 +123,7 @@ Performance Information F.A.Q. Intel partners with vendors all over the world. For a list of Hardware Manufacturers, see the `Intel® AI: In Production Partners & Solutions Catalog `__. For more details, see the :doc:`Supported Devices <../compatibility-and-support/supported-devices>`. - documentation. Before purchasing any hardware, you can test and run - models remotely, using `Intel® DevCloud for the Edge `__. + documentation. .. dropdown:: How can I optimize my models for better performance or accuracy? diff --git a/docs/articles_en/about-openvino/release-notes-openvino.rst b/docs/articles_en/about-openvino/release-notes-openvino.rst index 2ad5a1c674ac7b..c8c32126e3e10c 100644 --- a/docs/articles_en/about-openvino/release-notes-openvino.rst +++ b/docs/articles_en/about-openvino/release-notes-openvino.rst @@ -14,44 +14,59 @@ OpenVINO Release Notes -2024.1 - 24 April 2024 +2024.2 - 17 June 2024 ############################# :doc:`System Requirements <./release-notes-openvino/system-requirements>` | :doc:`Release policy <./release-notes-openvino/release-policy>` | :doc:`Installation Guides <./../get-started/install-openvino>` + What's new +++++++++++++++++++++++++++++ -* More Gen AI coverage and framework integrations to minimize code changes. +* More :doc:`Gen AI <../learn-openvino/llm_inference_guide/genai-guide>` coverage and framework + integrations to minimize code changes. + + * Llama 3 optimizations for CPUs, built-in GPUs, and discrete GPUs for improved performance + and efficient memory usage. + * Support for Phi-3-mini, a family of AI models that leverages the power of small language + models for faster, more accurate and cost-effective text processing. + * Python Custom Operation is now enabled in OpenVINO making it easier for Python developers + to code their custom operations instead of using C++ custom operations (also supported). + Python Custom Operation empowers users to implement their own specialized operations into + any model. + * Notebooks expansion to ensure better coverage for new models. Noteworthy notebooks added: + DynamiCrafter, YOLOv10, Chatbot notebook with Phi-3, and QWEN2. - * Mixtral and URLNet models optimized for performance improvements on Intel® Xeon® processors. - * Stable Diffusion 1.5, ChatGLM3-6B, and Qwen-7B models optimized for improved inference speed - on Intel® Core™ Ultra processors with integrated GPU. - * Support for Falcon-7B-Instruct, a GenAI Large Language Model (LLM) ready-to-use chat/instruct - model with superior performance metrics. - * New Jupyter Notebooks added: YOLO V9, YOLO V8 Oriented Bounding Boxes Detection (OOB), Stable - Diffusion in Keras, MobileCLIP, RMBG-v1.4 Background Removal, Magika, TripoSR, AnimateAnyone, - LLaVA-Next, and RAG system with OpenVINO and LangChain. -* Broader LLM model support and more model compression techniques. +* Broader Large Language Model (LLM) support and more model compression techniques. - * LLM compilation time reduced through additional optimizations with compressed embedding. - Improved 1st token performance of LLMs on 4th and 5th generations of Intel® Xeon® processors - with Intel® Advanced Matrix Extensions (Intel® AMX). - * Better LLM compression and improved performance with oneDNN, INT4, and INT8 support for - Intel® Arc™ GPUs. - * Significant memory reduction for select smaller GenAI models on Intel® Core™ Ultra processors - with integrated GPU. + * GPTQ method for 4-bit weight compression added to NNCF for more efficient inference and + improved performance of compressed LLMs. + * Significant LLM performance improvements and reduced latency for both built-in GPUs and + discrete GPUs. + * Significant improvement in 2nd token latency and memory footprint of FP16 weight LLMs on + AVX2 (13th Gen Intel® Core™ processors) and AVX512 (3rd Gen Intel® Xeon® Scalable Processors) + based CPU platforms, particularly for small batch sizes. * More portability and performance to run AI at the edge, in the cloud, or locally. - * The preview NPU plugin for Intel® Core™ Ultra processors is now available in the OpenVINO - open-source GitHub repository, in addition to the main OpenVINO package on PyPI. - * The JavaScript API is now more easily accessible through the npm repository, enabling - JavaScript developers' seamless access to the OpenVINO API. - * FP16 inference on ARM processors now enabled for the Convolutional Neural Network (CNN) by - default. + * Model Serving Enhancements: + + * Preview: OpenVINO Model Server (OVMS) now supports OpenAI-compatible API along with Continuous + Batching and PagedAttention, enabling significantly higher throughput for parallel + inferencing, especially on Intel® Xeon® processors, when serving LLMs to many concurrent + users. + * OpenVINO backend for Triton Server now supports dynamic input shapes. + * Integration of TorchServe through torch.compile OpenVINO backend for easy model deployment, + provisioning to multiple instances, model versioning, and maintenance. + + * Preview: addition of the :doc:`Generate API <../learn-openvino/llm_inference_guide/genai-guide>`, + a simplified API for text generation using large language models with only a few lines of + code. The API is available through the newly launched OpenVINO GenAI package. + * Support for Intel Atom® Processor X Series. For more details, see :doc:`System Requirements <./release-notes-openvino/system-requirements>`. + * Preview: Support for Intel® Xeon® 6 processor. + OpenVINO™ Runtime @@ -60,218 +75,466 @@ OpenVINO™ Runtime Common ----------------------------- -* Unicode file paths for cached models are now supported on Windows. -* Pad pre-processing API to extend input tensor on edges with constants. -* A fix for inference failures of certain image generation models has been implemented - (fused I/O port names after transformation). -* Compiler's warnings-as-errors option is now on, improving the coding criteria and quality. - Build warnings will not be allowed for new OpenVINO code and the existing warnings have been - fixed. +* Operations and data types using UINT2, UINT3, and UINT6 are now supported, to allow for a more + efficient LLM weight compression. +* Common OV headers have been optimized, improving binary compilation time and reducing binary + size. + AUTO Inference Mode ----------------------------- -* Returning the ov::enable_profiling value from ov::CompiledModel is now supported. +* AUTO takes model caching into account when choosing the device for fast first-inference latency. + If model cache is already in place, AUTO will directly use the selected device instead of + temporarily leveraging CPU as first-inference device. +* Dynamic models are now loaded to the selected device, instead of loading to CPU without + considering device priority. +* Fixed the exceptions when use AUTO with stateful models having dynamic input or output. + CPU Device Plugin ----------------------------- -* 1st token performance of LLMs has been improved on the 4th and 5th generations of Intel® Xeon® - processors with Intel® Advanced Matrix Extensions (Intel® AMX). -* LLM compilation time and memory footprint have been improved through additional optimizations - with compressed embeddings. -* Performance of MoE (e.g. Mixtral), Gemma, and GPT-J has been improved further. -* Performance has been improved significantly for a wide set of models on ARM devices. -* FP16 inference precision is now the default for all types of models on ARM devices. -* CPU architecture-agnostic build has been implemented, to enable unified binary distribution - on different ARM devices. +* Performance when using latency mode in FP32 precision has been improved on Intel client + platforms, including Core Ultra (codename Meteor Lake) and 13th Gen Core processors + (codename Raptor Lake). +* 2nd token latency and memory footprint for FP16 LLMs have been improved significantly on AVX2 + and AVX512 based CPU platforms, particularly for small batch sizes. +* PagedAttention has been optimized on AVX2, AVX512 and AMX platforms together with INT8 KV cache + support to improve the performance when serving LLM workloads on Intel CPUs. +* LLMs with shared embeddings have been optimized to improve performance and memory consumption + on several models including Gemma. +* Performance on ARM-based servers is significantly improved with upgrade to TBB 2021.2.5. +* Improved FP32 and FP16 performance on ARM CPU. + GPU Device Plugin ----------------------------- +* Both first token and average token latency of LLMs is improved on all GPU platforms, most + significantly on discrete GPUs. Memory usage of LLMs has been reduced as well. +* Stable Diffusion FP16 performance improved on Core Ultra platforms, with significant pipeline + improvement for models with dynamic-shaped input. Memory usage of the pipeline has been reduced, + as well. +* Optimized permute_f_y kernel performance has been improved. -* LLM first token latency has been improved on both integrated and discrete GPU platforms. -* For the ChatGLM3-6B model, average token latency has been improved on integrated GPU platforms. -* For Stable Diffusion 1.5 FP16 precision, performance has been improved on Intel® Core™ Ultra - processors. NPU Device Plugin ----------------------------- -* NPU Plugin is now part of the OpenVINO GitHub repository. All the most recent plugin changes - will be immediately available in the repo. Note that NPU is part of Intel® Core™ Ultra - processors. -* New OpenVINO™ notebook “Hello, NPU!” introducing NPU usage with OpenVINO has been added. -* Version 22H2 or later is required for Microsoft Windows® 11 64-bit to run inference on NPU. +* A new set of configuration options is now available. +* Performance increase has been unlocked, with the new `2408 NPU driver `__. + OpenVINO Python API ----------------------------- -* GIL-free creation of RemoteTensors is now used - holding GIL means that the process is not suited - for multithreading and removing the GIL lock will increase performance which is critical for - the concept of Remote Tensors. -* Packed data type BF16 on the Python API level has been added, opening a new way of supporting - data types not handled by numpy. -* 'pad' operator support for ov::preprocess::PrePostProcessorItem has been added. -* ov.PartialShape.dynamic(int) definition has been provided. +* Writing custom Python operators is now supported for basic scenarios (alignment with OpenVINO + C++ API.) This empowers users to implement their own specialized operations into any model. + Full support with more advanced features is within the scope of upcoming releases. OpenVINO C API ----------------------------- +* More element types are now supported to algin with the OpenVINO C++ API. -* Two new pre-processing APIs for scale and mean have been added. OpenVINO Node.js API ----------------------------- -* New methods to align JavaScript API with CPP API have been added, such as - CompiledModel.exportModel(), core.import_model(), Core set/get property and Tensor.get_size(), - and Model.is_dynamic(). -* Documentation has been extended to help developers start integrating JavaScript applications - with OpenVINO™. +* OpenVINO node.js packages now support the electron.js framework. +* Extended and improved JS API documentation for more complete usage guidelines. +* Better JS API alignment with OpenVINO C++ API, delivering more advanced features to JS users. + TensorFlow Framework Support ----------------------------- -* `tf.keras.layers.TextVectorization tokenizer `__ - is now supported. -* Conversion of models with Variable and HashTable (dictionary) resources has been improved. -* 8 NEW operations have been added - (`see the list here, marked as NEW `__). -* 10 operations have received complex tensor support. -* Input tensor names for TF1 models have been adjusted to have a single name per input. -* Hugging Face model support coverage has increased significantly, due to: +* 3 new operations are now supported. See operations marked as `NEW here `__. +* LookupTableImport has received better support, required for 2 models from TF Hub: + + * mil-nce + * openimages-v4-ssd-mobilenet-v2 + - * extraction of input signature of a model in memory has been fixed, - * reading of variable values for a model in memory has been fixed. +TensorFlow Lite Framework Support +---------------------------------- + +* The GELU operation required for customer model is now supported. PyTorch Framework Support ----------------------------- -* ModuleExtension, a new type of extension for PyTorch models is now supported - (`PR #23536 `__). -* 22 NEW operations have been added. -* Experimental support for models produced by torch.export (FX graph) has been added - (`PR #23815 `__). +* 9 new operations are now supported. +* aten::set_item now supports negative indices. +* Issue with adaptive pool when shape is list has been fixed (PR `#24586 `__). + -ONNX Framework Support +ONNX Support ----------------------------- -* 8 new operations have been added. + +* The InputModel interface should be used from now on, instead of a number of deprecated APIs + and class symbols +* Translation for ReduceMin-18 and ReduceSumSquare-18 operators has been added, to address + customer model requests +* Behavior of the Gelu-20 operator has been fixed for the case when “none” is set as the + default value. OpenVINO Model Server +++++++++++++++++++++++++++++ -* OpenVINO™ Runtime backend used is now 2024.1 -* OpenVINO™ models with String data type on output are supported. Now, OpenVINO™ Model Server - can support models with input and output of the String type, so developers can take advantage - of the tokenization built into the model as the first layer. Developers can also rely on any - postprocessing embedded into the model which returns text only. Check the - `demo on string input data with the universal-sentence-encoder model `__ - and the - `String output model demo `__. -* MediaPipe Python calculators have been updated to support relative paths for all related - configuration and Python code files. Now, the complete graph configuration folder can be - deployed in an arbitrary path without any code changes. -* KServe REST API support has been extended to properly handle the string format in JSON body, - just like the binary format compatible with NVIDIA Triton™. -* `A demo showcasing a full RAG algorithm `__ - fully delegated to the model server has been added. +* OpenVINO Model server can be now used for text generation use cases using OpenAI compatible API. +* Added support for continuous batching and PagedAttention algorithms for text generation with + fast and efficient in high concurrency load especially on Intel Xeon processors. + `Learn more about it `__. Neural Network Compression Framework -++++++++++++++++++++++++++++++++++++++++++ ++++++++++++++++++++++++++++++++++++++ -* Model subgraphs can now be defined in the ignored scope for INT8 Post-training Quantization, - nncf.quantize(), which simplifies excluding accuracy-sensitive layers from quantization. -* A batch size of more than 1 is now partially supported for INT8 Post-training Quantization, - speeding up the process. Note that it is not recommended for transformer-based models as it - may impact accuracy. Here is an - `example demo `__. -* Now it is possible to apply fine-tuning on INT8 models after Post-training Quantization to - improve model accuracy and make it easier to move from post-training to training-aware - quantization. Here is an - `example demo `__. +* GPTQ method is now supported in nncf.compress_weights() for data-aware 4-bit weight + compression of LLMs. Enabled by `gptq=True`` in nncf.compress_weights(). +* Scale Estimation algorithm for more accurate 4-bit compressed LLMs. Enabled by + `scale_estimation=True`` in nncf.compress_weights(). +* Added support for models with bf16 weights in nncf.compress_weights(). +* nncf.quantize() method is now the recommended path for quantization initialization of + PyTorch models in Quantization-Aware Training. See example for more details. +* compressed_model.nncf.get_config() and nncf.torch.load_from_config() API have been added to + save and restore quantized PyTorch models. See example for more details. +* Automatic support for int8 quantization of PyTorch models with custom modules has been added. + Now it is not needed to register such modules before quantization. -OpenVINO Tokenizers -++++++++++++++++++++++++++++++++++++++++++ -* TensorFlow support has been extended - TextVectorization layer translation: +Other Changes and Known Issues ++++++++++++++++++++++++++++++++++++++ - * Aligned existing ops with TF ops and added a translator for them. - * Added new ragged tensor ops and string ops. +Jupyter Notebooks +----------------------------- -* A new tokenizer type, RWKV is now supported: +* Latest notebooks along with the GitHub validation status can be found in the + `OpenVINO notebook section `__ +* The following notebooks have been updated or newly added: - * Added Trie tokenizer and Fuse op for ragged tensors. - * A new way to get OV Tokenizers: build a vocab from file. + * `Image to Video Generation with Stable Video Diffusion `__ + * `Image generation with Stable Cascade `__ + * `One Step Sketch to Image translation with pix2pix-turbo and OpenVINO `__ + * `Animating Open-domain Images with DynamiCrafter and OpenVINO `__ + * `Text-to-Video retrieval with S3D MIL-NCE and OpenVINO `__ + * `Convert and Optimize YOLOv10 with OpenVINO `__ + * `Visual-language assistant with nanoLLaVA and OpenVINO `__ + * `Person Counting System using YOLOV8 and OpenVINO™ `__ + * `Quantization-Sparsity Aware Training with NNCF, using PyTorch framework `__ + * `Create an LLM-powered Chatbot using OpenVINO `__ -* Tokenizer caching has been redesigned to work with the OpenVINO™ model caching mechanism. +Known Issues +----------------------------- + +| **Component: TBB** +| ID: TBB-1400/ TBB-1401 +| Description: +| In 2024.2, oneTBB 2021.2.x is used for Intel Distribution of OpenVINO Ubuntu and Red Hat + archives, instead of system TBB/oneTBB. This improves performance on the new generation of + Xeon platforms but may increase latency of some models on the previous generation. You can + build OpenVINO with **-DSYSTEM_TBB=ON** to get better latency performance for these models. + +| **Component: python API** +| ID: CVS-141744 +| Description: +| During post commit tests we found problem related with custom operations. Fix is ready and + will be delivered with 2024.3 release. +| - Initial problem: test_custom_op hanged on destruction because it was waiting for a + thread which tried to acquire GIL. +| - The second problem is that pybind11 doesn't allow to work with GIL besides of current + scope and it's impossible to release GIL for destructors. Blocking destructors and the + GIL pybind/pybind11#1446 +| - Current solution allows to release GIL for InferRequest and all called by chain destructors. + + +| **Component: CPU runtime** +| *ID:* MFDNN-11428 +| *Description:* +| Due to adopting a new OneDNN library, improving performance for most use cases, + particularly for AVX2 BRGEMM kernels with the latency hint, the following regressions may + be noticed: +| a. latency regression on certain models, such as unet-camvid-onnx-0001 and mask_rcnn_resnet50_atrous_coco on MTL Windows latency mode +| b. performance regression on Intel client platforms if the throughput hint is used +| The issue is being investigated and planned to be resolved in the following releases. + + +| **Component: Hardware Configuration** +| *ID:* N/A +| *Description:* +| Reduced performance for LLMs may be observed on newer CPUs. To mitigate, modify the default settings in BIOS to change the system into 2 NUMA node system: +| 1. Enter the BIOS configuration menu. +| 2. Select EDKII Menu -> Socket Configuration -> Uncore Configuration -> Uncore General Configuration -> SNC. +| 3. The SNC setting is set to *AUTO* by default. Change the SNC setting to *disabled* to configure one NUMA node per processor socket upon boot. +| 4. After system reboot, confirm the NUMA node setting using: `numatcl -H`. Expect to see only nodes 0 and 1 on a 2-socket system with the following mapping: +| Node - 0 - 1 +| 0 - 10 - 21 +| 1 - 21 - 10 -Other Changes and Known Issues -++++++++++++++++++++++++++++++++++++++++++ -Jupyter Notebooks ------------------------------ -The default branch for the OpenVINO™ Notebooks repository has been changed from 'main' to -'latest'. The 'main' branch of the notebooks repository is now deprecated and will be maintained -until September 30, 2024. -The new branch, 'latest', offers a better user experience and simplifies maintenance due to -significant refactoring and an improved directory naming structure. -Use the local -`README.md `__ -file and OpenVINO™ Notebooks at -`GitHub Pages `__ -to navigate through the content. -The following notebooks have been updated or newly added: -* `Grounded Segment Anything `__ -* `Visual Content Search with MobileCLIP `__ -* `YOLO V8 Oriented Bounding Box Detection Optimization `__ -* `Magika: AI-powered fast and efficient file type identification `__ -* `Keras Stable Diffusion `__ -* `RMBG background removal `__ -* `AnimateAnyone: pose guided image to video generation `__ -* `LLaVA-Next visual-language assistant `__ -* `TripoSR: single image 3d reconstruction `__ -* `RAG system with OpenVINO and LangChain `__ -Known Issues ------------------------------ -| **Component - CPU Plugin** -| *ID* - N/A -| *Description:* -| Default CPU pinning policy on Windows has been changed to follow Windows' policy - instead of controlling the CPU pinning in the OpenVINO plugin. This brings certain dynamic or - performance variance on Windows. Developers can use ov::hint::enable_cpu_pinning to enable - or disable CPU pinning explicitly. -| **Component - Hardware Configuration** -| *ID* - N/A -| *Description:* -| Reduced performance for LLMs may be observed on newer CPUs. To mitigate, modify the default settings in BIOS to -| change the system into 2 NUMA node system: -| 1. Enter the BIOS configuration menu. -| 2. Select EDKII Menu -> Socket Configuration -> Uncore Configuration -> Uncore General Configuration -> SNC. -| 3. The SNC setting is set to *AUTO* by default. Change the SNC setting to *disabled* to configure one NUMA node per processor socket upon boot. -| 4. After system reboot, confirm the NUMA node setting using: `numatcl -H`. Expect to see only nodes 0 and 1 on a -| 2-socket system with the following mapping: -| Node - 0 - 1 -| 0 - 10 - 21 -| 1 - 21 - 10 Previous 2024 releases +++++++++++++++++++++++++++++ +.. dropdown:: 2024.1 - 24 April 2024 + :animate: fade-in-slide-down + :color: secondary + + **What's new** + + * More Gen AI coverage and framework integrations to minimize code changes. + + * Mixtral and URLNet models optimized for performance improvements on Intel® Xeon® processors. + * Stable Diffusion 1.5, ChatGLM3-6B, and Qwen-7B models optimized for improved inference speed + on Intel® Core™ Ultra processors with integrated GPU. + * Support for Falcon-7B-Instruct, a GenAI Large Language Model (LLM) ready-to-use chat/instruct + model with superior performance metrics. + * New Jupyter Notebooks added: YOLO V9, YOLO V8 Oriented Bounding Boxes Detection (OOB), Stable + Diffusion in Keras, MobileCLIP, RMBG-v1.4 Background Removal, Magika, TripoSR, AnimateAnyone, + LLaVA-Next, and RAG system with OpenVINO and LangChain. + + * Broader LLM model support and more model compression techniques. + + * LLM compilation time reduced through additional optimizations with compressed embedding. + Improved 1st token performance of LLMs on 4th and 5th generations of Intel® Xeon® processors + with Intel® Advanced Matrix Extensions (Intel® AMX). + * Better LLM compression and improved performance with oneDNN, INT4, and INT8 support for + Intel® Arc™ GPUs. + * Significant memory reduction for select smaller GenAI models on Intel® Core™ Ultra processors + with integrated GPU. + + * More portability and performance to run AI at the edge, in the cloud, or locally. + + * The preview NPU plugin for Intel® Core™ Ultra processors is now available in the OpenVINO + open-source GitHub repository, in addition to the main OpenVINO package on PyPI. + * The JavaScript API is now more easily accessible through the npm repository, enabling + JavaScript developers' seamless access to the OpenVINO API. + * FP16 inference on ARM processors now enabled for the Convolutional Neural Network (CNN) by + default. + + + **OpenVINO™ Runtime** + + *Common* + + + * Unicode file paths for cached models are now supported on Windows. + * Pad pre-processing API to extend input tensor on edges with constants. + * A fix for inference failures of certain image generation models has been implemented + (fused I/O port names after transformation). + * Compiler's warnings-as-errors option is now on, improving the coding criteria and quality. + Build warnings will not be allowed for new OpenVINO code and the existing warnings have been + fixed. + + *AUTO Inference Mode* + + * Returning the ov::enable_profiling value from ov::CompiledModel is now supported. + + *CPU Device Plugin* + + * 1st token performance of LLMs has been improved on the 4th and 5th generations of Intel® Xeon® + processors with Intel® Advanced Matrix Extensions (Intel® AMX). + * LLM compilation time and memory footprint have been improved through additional optimizations + with compressed embeddings. + * Performance of MoE (e.g. Mixtral), Gemma, and GPT-J has been improved further. + * Performance has been improved significantly for a wide set of models on ARM devices. + * FP16 inference precision is now the default for all types of models on ARM devices. + * CPU architecture-agnostic build has been implemented, to enable unified binary distribution + on different ARM devices. + + *GPU Device Plugin* + + * LLM first token latency has been improved on both integrated and discrete GPU platforms. + * For the ChatGLM3-6B model, average token latency has been improved on integrated GPU platforms. + * For Stable Diffusion 1.5 FP16 precision, performance has been improved on Intel® Core™ Ultra + processors. + + *NPU Device Plugin* + + * NPU Plugin is now part of the OpenVINO GitHub repository. All the most recent plugin changes + will be immediately available in the repo. Note that NPU is part of Intel® Core™ Ultra + processors. + * New OpenVINO™ notebook “Hello, NPU!” introducing NPU usage with OpenVINO has been added. + * Version 22H2 or later is required for Microsoft Windows® 11 64-bit to run inference on NPU. + + *OpenVINO Python API* + + * GIL-free creation of RemoteTensors is now used - holding GIL means that the process is not suited + for multithreading and removing the GIL lock will increase performance which is critical for + the concept of Remote Tensors. + * Packed data type BF16 on the Python API level has been added, opening a new way of supporting + data types not handled by numpy. + * 'pad' operator support for ov::preprocess::PrePostProcessorItem has been added. + * ov.PartialShape.dynamic(int) definition has been provided. + + + *OpenVINO C API* + + * Two new pre-processing APIs for scale and mean have been added. + + *OpenVINO Node.js API* + + * New methods to align JavaScript API with CPP API have been added, such as + CompiledModel.exportModel(), core.import_model(), Core set/get property and Tensor.get_size(), + and Model.is_dynamic(). + * Documentation has been extended to help developers start integrating JavaScript applications + with OpenVINO™. + + *TensorFlow Framework Support* + + * `tf.keras.layers.TextVectorization tokenizer `__ + is now supported. + * Conversion of models with Variable and HashTable (dictionary) resources has been improved. + * 8 NEW operations have been added + (`see the list here, marked as NEW `__). + * 10 operations have received complex tensor support. + * Input tensor names for TF1 models have been adjusted to have a single name per input. + * Hugging Face model support coverage has increased significantly, due to: + + * extraction of input signature of a model in memory has been fixed, + * reading of variable values for a model in memory has been fixed. + + + *PyTorch Framework Support* + + * ModuleExtension, a new type of extension for PyTorch models is now supported + (`PR #23536 `__). + * 22 NEW operations have been added. + * Experimental support for models produced by torch.export (FX graph) has been added + (`PR #23815 `__). + + *ONNX Framework Support* + + * 8 new operations have been added. + + + **OpenVINO Model Server** + + * OpenVINO™ Runtime backend used is now 2024.1 + * OpenVINO™ models with String data type on output are supported. Now, OpenVINO™ Model Server + can support models with input and output of the String type, so developers can take advantage + of the tokenization built into the model as the first layer. Developers can also rely on any + postprocessing embedded into the model which returns text only. Check the + `demo on string input data with the universal-sentence-encoder model `__ + and the + `String output model demo `__. + * MediaPipe Python calculators have been updated to support relative paths for all related + configuration and Python code files. Now, the complete graph configuration folder can be + deployed in an arbitrary path without any code changes. + * KServe REST API support has been extended to properly handle the string format in JSON body, + just like the binary format compatible with NVIDIA Triton™. + * `A demo showcasing a full RAG algorithm `__ + fully delegated to the model server has been added. + + + **Neural Network Compression Framework** + + * Model subgraphs can now be defined in the ignored scope for INT8 Post-training Quantization, + nncf.quantize(), which simplifies excluding accuracy-sensitive layers from quantization. + * A batch size of more than 1 is now partially supported for INT8 Post-training Quantization, + speeding up the process. Note that it is not recommended for transformer-based models as it + may impact accuracy. Here is an + `example demo `__. + * Now it is possible to apply fine-tuning on INT8 models after Post-training Quantization to + improve model accuracy and make it easier to move from post-training to training-aware + quantization. Here is an + `example demo `__. + + **OpenVINO Tokenizers** + + * TensorFlow support has been extended - TextVectorization layer translation: + + * Aligned existing ops with TF ops and added a translator for them. + * Added new ragged tensor ops and string ops. + + * A new tokenizer type, RWKV is now supported: + + * Added Trie tokenizer and Fuse op for ragged tensors. + * A new way to get OV Tokenizers: build a vocab from file. + + * Tokenizer caching has been redesigned to work with the OpenVINO™ model caching mechanism. + + + **Other Changes and Known Issues** + + *Jupyter Notebooks* + + The default branch for the OpenVINO™ Notebooks repository has been changed from 'main' to + 'latest'. The 'main' branch of the notebooks repository is now deprecated and will be maintained + until September 30, 2024. + + The new branch, 'latest', offers a better user experience and simplifies maintenance due to + significant refactoring and an improved directory naming structure. + + Use the local + `README.md `__ + file and OpenVINO™ Notebooks at + `GitHub Pages `__ + to navigate through the content. + + + The following notebooks have been updated or newly added: + + * `Grounded Segment Anything `__ + * `Visual Content Search with MobileCLIP `__ + * `YOLO V8 Oriented Bounding Box Detection Optimization `__ + * `Magika: AI-powered fast and efficient file type identification `__ + * `Keras Stable Diffusion `__ + * `RMBG background removal `__ + * `AnimateAnyone: pose guided image to video generation `__ + * `LLaVA-Next visual-language assistant `__ + * `TripoSR: single image 3d reconstruction `__ + * `RAG system with OpenVINO and LangChain `__ + + + *Known Issues* + + | **Component: CPU Plugin** + | *ID:* N/A + | *Description:* + | Default CPU pinning policy on Windows has been changed to follow Windows' policy + instead of controlling the CPU pinning in the OpenVINO plugin. This brings certain dynamic or + performance variance on Windows. Developers can use ov::hint::enable_cpu_pinning to enable + or disable CPU pinning explicitly. + + | **Component: Hardware Configuration** + | *ID:* N/A + | *Description:* + | Reduced performance for LLMs may be observed on newer CPUs. To mitigate, modify the default settings in BIOS to + | change the system into 2 NUMA node system: + | 1. Enter the BIOS configuration menu. + | 2. Select EDKII Menu -> Socket Configuration -> Uncore Configuration -> Uncore General Configuration -> SNC. + | 3. The SNC setting is set to *AUTO* by default. Change the SNC setting to *disabled* to configure one NUMA node per processor socket upon boot. + | 4. After system reboot, confirm the NUMA node setting using: `numatcl -H`. Expect to see only nodes 0 and 1 on a + | 2-socket system with the following mapping: + | Node - 0 - 1 + | 0 - 10 - 21 + | 1 - 21 - 10 + + + + + + + + + + .. dropdown:: 2024.0 - 06 March 2024 :animate: fade-in-slide-down :color: secondary @@ -491,21 +754,21 @@ Previous 2024 releases *Known issues* - | **Component - CPU Plugin** - | *ID* - N/A + | **Component: CPU Plugin** + | *ID:* N/A | *Description:* | Starting with 24.0, model inputs and outputs will no longer have tensor names, unless explicitly set to align with the PyTorch framework behavior. - | **Component - GPU runtime** - | *ID* - 132376 + | **Component: GPU runtime** + | *ID:* 132376 | *Description:* | First-inference latency slow down for LLMs on Intel® Core™ Ultra processors. Up to 10-20% drop may occur due to radical memory optimization for processing long sequences (about 1.5-2 GB reduced memory usage). - | **Component - CPU runtime** - | *ID* - N/A + | **Component: CPU runtime** + | *ID:* N/A | *Description:* | Performance results (first token latency) may vary from those offered by the previous OpenVINO version, for “latency” hint inference of LLMs with long prompts on Xeon platforms with 2 or more @@ -523,6 +786,7 @@ Previous 2024 releases + Deprecation And Support +++++++++++++++++++++++++++++ Using deprecated features and components is not advised. They are available to enable a smooth @@ -575,90 +839,93 @@ Deprecated and to be removed in the future * “auto shape” and “auto batch size” (reshaping a model in runtime) will be removed in the future. OpenVINO's dynamic shape models are recommended instead. -* The following notebooks have been deprecated and will be removed. For an up-to-date listing - of available notebooks, refer to - `OpenVINO™ Notebook index (openvinotoolkit.github.io) `__. +* A number of notebooks have been deprecated. For an up-to-date listing of available notebooks, + refer to the `OpenVINO™ Notebook index (openvinotoolkit.github.io) `__. + + .. dropdown:: See the deprecated notebook list + :animate: fade-in-slide-down + :color: muted - * `Handwritten OCR with OpenVINO™ `__ + * `Handwritten OCR with OpenVINO™ `__ - * See alternative: `Optical Character Recognition (OCR) with OpenVINO™ `__, - * See alternative: `PaddleOCR with OpenVINO™ `__, - * See alternative: `Handwritten Text Recognition Demo `__ + * See alternative: `Optical Character Recognition (OCR) with OpenVINO™ `__, + * See alternative: `PaddleOCR with OpenVINO™ `__, + * See alternative: `Handwritten Text Recognition Demo `__ - * `Image In-painting with OpenVINO™ `__ + * `Image In-painting with OpenVINO™ `__ - * See alternative: `Image Inpainting Python Demo `__ + * See alternative: `Image Inpainting Python Demo `__ - * `Interactive Machine Translation with OpenVINO `__ + * `Interactive Machine Translation with OpenVINO `__ - * See alternative: `Machine Translation Python* Demo `__ + * See alternative: `Machine Translation Python* Demo `__ - * `Open Model Zoo Tools Tutorial `__ + * `Open Model Zoo Tools Tutorial `__ - * No alternatives, demonstrates deprecated tools. + * No alternatives, demonstrates deprecated tools. - * `Super Resolution with OpenVINO™ `__ + * `Super Resolution with OpenVINO™ `__ - * See alternative: `Super Resolution with PaddleGAN and OpenVINO `__ - * See alternative: `Image Processing C++ Demo `__ + * See alternative: `Super Resolution with PaddleGAN and OpenVINO `__ + * See alternative: `Image Processing C++ Demo `__ - * `Image Colorization with OpenVINO Tutorial `__ - * `Interactive Question Answering with OpenVINO™ `__ + * `Image Colorization with OpenVINO Tutorial `__ + * `Interactive Question Answering with OpenVINO™ `__ - * See alternative: `BERT Question Answering Embedding Python* Demo `__ - * See alternative: `BERT Question Answering Python* Demo `__ + * See alternative: `BERT Question Answering Embedding Python* Demo `__ + * See alternative: `BERT Question Answering Python* Demo `__ - * `Vehicle Detection And Recognition with OpenVINO™ `__ + * `Vehicle Detection And Recognition with OpenVINO™ `__ - * See alternative: `Security Barrier Camera C++ Demo `__ + * See alternative: `Security Barrier Camera C++ Demo `__ - * `The attention center model with OpenVINO™ `_ - * `Image Generation with DeciDiffusion `_ - * `Image generation with DeepFloyd IF and OpenVINO™ `_ - * `Depth estimation using VI-depth with OpenVINO™ `_ - * `Instruction following using Databricks Dolly 2.0 and OpenVINO™ `_ + * `The attention center model with OpenVINO™ `_ + * `Image Generation with DeciDiffusion `_ + * `Image generation with DeepFloyd IF and OpenVINO™ `_ + * `Depth estimation using VI-depth with OpenVINO™ `_ + * `Instruction following using Databricks Dolly 2.0 and OpenVINO™ `_ - * See alternative: `LLM Instruction-following pipeline with OpenVINO `__ + * See alternative: `LLM Instruction-following pipeline with OpenVINO `__ - * `Image generation with FastComposer and OpenVINO™ `__ - * `Video Subtitle Generation with OpenAI Whisper `__ + * `Image generation with FastComposer and OpenVINO™ `__ + * `Video Subtitle Generation with OpenAI Whisper `__ - * See alternative: `Automatic speech recognition using Distil-Whisper and OpenVINO `__ + * See alternative: `Automatic speech recognition using Distil-Whisper and OpenVINO `__ - * `Introduction to Performance Tricks in OpenVINO™ `__ - * `Speaker Diarization with OpenVINO™ `__ - * `Subject-driven image generation and editing using BLIP Diffusion and OpenVINO `__ - * `Text Prediction with OpenVINO™ `__ - * `Training to Deployment with TensorFlow and OpenVINO™ `__ - * `Speech to Text with OpenVINO™ `__ - * `Convert and Optimize YOLOv7 with OpenVINO™ `__ - * `Quantize Data2Vec Speech Recognition Model using NNCF PTQ API `__ + * `Introduction to Performance Tricks in OpenVINO™ `__ + * `Speaker Diarization with OpenVINO™ `__ + * `Subject-driven image generation and editing using BLIP Diffusion and OpenVINO `__ + * `Text Prediction with OpenVINO™ `__ + * `Training to Deployment with TensorFlow and OpenVINO™ `__ + * `Speech to Text with OpenVINO™ `__ + * `Convert and Optimize YOLOv7 with OpenVINO™ `__ + * `Quantize Data2Vec Speech Recognition Model using NNCF PTQ API `__ - * See alternative: `Quantize Speech Recognition Models with accuracy control using NNCF PTQ API `__ + * See alternative: `Quantize Speech Recognition Models with accuracy control using NNCF PTQ API `__ - * `Semantic segmentation with LRASPP MobileNet v3 and OpenVINO `__ - * `Video Recognition using SlowFast and OpenVINO™ `__ + * `Semantic segmentation with LRASPP MobileNet v3 and OpenVINO `__ + * `Video Recognition using SlowFast and OpenVINO™ `__ - * See alternative: `Live Action Recognition with OpenVINO™ `__ + * See alternative: `Live Action Recognition with OpenVINO™ `__ - * `Semantic Segmentation with OpenVINO™ using Segmenter `__ - * `Programming Language Classification with OpenVINO `__ - * `Stable Diffusion Text-to-Image Demo `__ + * `Semantic Segmentation with OpenVINO™ using Segmenter `__ + * `Programming Language Classification with OpenVINO `__ + * `Stable Diffusion Text-to-Image Demo `__ - * See alternative: `Stable Diffusion v2.1 using Optimum-Intel OpenVINO and multiple Intel Hardware `__ + * See alternative: `Stable Diffusion v2.1 using Optimum-Intel OpenVINO and multiple Intel Hardware `__ - * `Text-to-Image Generation with Stable Diffusion v2 and OpenVINO™ `__ + * `Text-to-Image Generation with Stable Diffusion v2 and OpenVINO™ `__ - * See alternative: `Stable Diffusion v2.1 using Optimum-Intel OpenVINO and multiple Intel Hardware `__ + * See alternative: `Stable Diffusion v2.1 using Optimum-Intel OpenVINO and multiple Intel Hardware `__ - * `Image generation with Segmind Stable Diffusion 1B (SSD-1B) model and OpenVINO `__ - * `Data Preparation for 2D Medical Imaging `__ - * `Train a Kidney Segmentation Model with MONAI and PyTorch Lightning `__ - * `Live Inference and Benchmark CT-scan Data with OpenVINO™ `__ + * `Image generation with Segmind Stable Diffusion 1B (SSD-1B) model and OpenVINO `__ + * `Data Preparation for 2D Medical Imaging `__ + * `Train a Kidney Segmentation Model with MONAI and PyTorch Lightning `__ + * `Live Inference and Benchmark CT-scan Data with OpenVINO™ `__ - * See alternative: `Quantize a Segmentation Model and Show Live Inference `__ + * See alternative: `Quantize a Segmentation Model and Show Live Inference `__ - * `Live Style Transfer with OpenVINO™ `__ + * `Live Style Transfer with OpenVINO™ `__ diff --git a/docs/articles_en/about-openvino/release-notes-openvino/system-requirements.rst b/docs/articles_en/about-openvino/release-notes-openvino/system-requirements.rst index ed2533e1cb405c..a02d133453c8b9 100644 --- a/docs/articles_en/about-openvino/release-notes-openvino/system-requirements.rst +++ b/docs/articles_en/about-openvino/release-notes-openvino/system-requirements.rst @@ -19,6 +19,8 @@ CPU .. tab-item:: Supported Hardware + * Intel® Xeon® 6 processor (preview) + * Intel Atom® Processor X Series * Intel Atom® processor with Intel® SSE4.2 support * Intel® Pentium® processor N4200/5, N3350/5, N3450/5 with Intel® HD Graphics * 6th - 14th generation Intel® Core™ processors @@ -28,17 +30,16 @@ CPU .. tab-item:: Supported Operating Systems - * Ubuntu 24.04 long-term support (LTS), 64-bit (Kernel 6.8+) * Ubuntu 22.04 long-term support (LTS), 64-bit (Kernel 5.15+) * Ubuntu 20.04 long-term support (LTS), 64-bit (Kernel 5.15+) * Ubuntu 18.04 long-term support (LTS) with limitations, 64-bit (Kernel 5.4+) - * Windows 10 - * Windows 11 + * Windows 10, 64-bit + * Windows 11, 64-bit * macOS 12.6 and above, 64-bit and ARM64 * CentOS 7 * Red Hat Enterprise Linux 8, 64-bit * openSUSE Tumbleweed, 64-bit and ARM64 - * Ubuntu 18 ARM64 + * Ubuntu 20.04 ARM64 GPU ########## @@ -58,7 +59,6 @@ GPU .. tab-item:: Supported Operating Systems - * Ubuntu 24.04 long-term support (LTS), 64-bit * Ubuntu 22.04 long-term support (LTS), 64-bit * Ubuntu 20.04 long-term support (LTS), 64-bit * Windows 10, 64-bit @@ -88,7 +88,6 @@ Intel® Neural Processing Unit .. tab-item:: Operating Systems for NPU - * Ubuntu 24.04 long-term support (LTS), 64-bit * Ubuntu 22.04 long-term support (LTS), 64-bit * Windows 11, 64-bit (22H2, 23H2) @@ -107,7 +106,6 @@ Operating systems and developer environment .. tab-item:: Linux OS - * Ubuntu 24.04 with Linux kernel 6.8+ * Ubuntu 22.04 with Linux kernel 5.15+ * Ubuntu 20.04 with Linux kernel 5.15+ * Red Hat Enterprise Linux 8 with Linux kernel 5.4 @@ -122,11 +120,9 @@ Operating systems and developer environment * `GNU Compiler Collection (GCC) `__ 7.5 and above * `CMake `__ 3.13 or higher - Higher versions of kernel might be required for 10th Gen Intel® Core™ Processors, 11th Gen - Intel® Core™ Processors, 11th Gen Intel® Core™ Processors S-Series Processors, 12th Gen - Intel® Core™ Processors, 13th Gen Intel® Core™ Processors, 14th Gen - Intel® Core™ Processors, Intel® Core™ Ultra Processors, 4th Gen Intel® Xeon® Scalable Processors - or 5th Gen Intel® Xeon® Scalable Processors to support CPU, GPU, GNA or hybrid-cores CPU capabilities. + Higher versions of kernel might be required for 10th Gen Intel® Core™ Processors and above, + Intel® Core™ Ultra Processors, 4th Gen Intel® Xeon® Scalable Processors and above + to support CPU, GPU, NPU or hybrid-cores CPU capabilities. .. tab-item:: Windows 10 and 11 @@ -150,9 +146,9 @@ Operating systems and developer environment .. tab-item:: DL frameworks versions: - * TensorFlow 1.15, 2.13.1 + * TensorFlow 1.15.5, 2.16.1 * ONNX 1.15 - * PaddlePaddle 2.5 + * PaddlePaddle 2.6 This package can be installed on other versions of DL Frameworks but only the versions specified here are fully validated. @@ -160,12 +156,7 @@ Operating systems and developer environment .. note:: - OpenVINO Python binaries and binaries on Windows, CentOS 7, and macOS (x86) are built - with oneTBB libraries, and others on Ubuntu and RedHat systems are built with - legacy TBB which is released by OS distribution. OpenVINO can be built from source - with either oneTBB or legacy TBB on all the systems listed here. System - compatibility and performance are improved on Hybrid CPUs - such as 12th Gen Intel Core and above. + OpenVINO Python binaries are built with and redistribute oneTBB libraries. @@ -203,7 +194,7 @@ OpenCL and the OpenCL logo are trademarks of Apple Inc. used by permission by Kh Other names and brands may be claimed as the property of others. -Copyright © 2023, Intel Corporation. All rights reserved. +Copyright © 2024, Intel Corporation. All rights reserved. For more complete information about compiler optimizations, see our Optimization Notice. diff --git a/docs/articles_en/assets/images/optional.png b/docs/articles_en/assets/images/optional.png new file mode 100644 index 00000000000000..2c77f9288d42a0 --- /dev/null +++ b/docs/articles_en/assets/images/optional.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b78713020891a6655e3860338b7db430840560d8f2860162eca3d00b84533f24 +size 36362 diff --git a/docs/articles_en/assets/images/or-branches.png b/docs/articles_en/assets/images/or-branches.png new file mode 100644 index 00000000000000..c71766873ae116 --- /dev/null +++ b/docs/articles_en/assets/images/or-branches.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:353c5b7f35a8f9f6c2c3074485a5a4e51f8c6ee776214444666da4037cc44832 +size 45347 diff --git a/docs/articles_en/assets/images/or_branches.png b/docs/articles_en/assets/images/or_branches.png new file mode 100644 index 00000000000000..765b0622eebfd3 --- /dev/null +++ b/docs/articles_en/assets/images/or_branches.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:696672a8c38260147f2fa68d266866ab4390ef919fa092cea1149b0eb529c68e +size 58557 diff --git a/docs/articles_en/assets/images/python-api.png b/docs/articles_en/assets/images/python-api.png new file mode 100644 index 00000000000000..191ff8bace1962 --- /dev/null +++ b/docs/articles_en/assets/images/python-api.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d378718b968f6a40555b31bf54af83fac978c3bfc07e234f9b1370229222cccd +size 31092 diff --git a/docs/articles_en/assets/images/simple_pattern_example.png b/docs/articles_en/assets/images/simple_pattern_example.png new file mode 100644 index 00000000000000..b9fb29d34f6f4d --- /dev/null +++ b/docs/articles_en/assets/images/simple_pattern_example.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b3511899521a49c1bfdc569e16b8ad1e12157fe107d2b86d25ee04ff67a85b2 +size 43066 diff --git a/docs/articles_en/assets/snippets/__init__.py b/docs/articles_en/assets/snippets/__init__.py new file mode 100644 index 00000000000000..6238ede4469d18 --- /dev/null +++ b/docs/articles_en/assets/snippets/__init__.py @@ -0,0 +1,6 @@ +# Copyright (C) 2018-2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +from .utils import get_image +from .utils import get_model +from .utils import get_path_to_model diff --git a/docs/snippets/main.py b/docs/articles_en/assets/snippets/main.py similarity index 100% rename from docs/snippets/main.py rename to docs/articles_en/assets/snippets/main.py diff --git a/docs/articles_en/assets/snippets/multi_threading.py b/docs/articles_en/assets/snippets/multi_threading.py index 6994b26a0d6552..edefa760318f1a 100644 --- a/docs/articles_en/assets/snippets/multi_threading.py +++ b/docs/articles_en/assets/snippets/multi_threading.py @@ -37,7 +37,7 @@ # ! [ov:intel_cpu:multi_threading:part0] # ! [ov:intel_cpu:multi_threading:part1] -# Disable CPU threads pinning for inference when the system supports it +# Disable CPU thread pinning for inference when the system supports it compiled_model_4 = core.compile_model( model=model, device_name=device_name, diff --git a/docs/articles_en/assets/snippets/ov_model_snippets.cpp b/docs/articles_en/assets/snippets/ov_model_snippets.cpp index 0ca99b765643e1..31ba0bc8028edd 100644 --- a/docs/articles_en/assets/snippets/ov_model_snippets.cpp +++ b/docs/articles_en/assets/snippets/ov_model_snippets.cpp @@ -171,8 +171,8 @@ if (m->match(node->output(0))) { bool openvino_api_examples(std::shared_ptr node) { { // ! [ov:ports_example] -// Let's suppose that node is opset8::Convolution operation -// as we know opset8::Convolution has two input ports (data, weights) and one output port +// Let's suppose that node is of ov::op::v0::Convolution type. +// As we know ov::op::v0::Convolution has two input ports (data, weights) and one output port. ov::Input data = node->input(0); ov::Input weights = node->input(1); ov::Output output = node->output(0); @@ -181,7 +181,7 @@ ov::Output output = node->output(0); auto pshape = data.get_partial_shape(); auto el_type = data.get_element_type(); -// Getting parent for input port +// Getting parent for input port i.e. Output mapped by the input ov::Output parent_output; parent_output = data.get_source_output(); @@ -216,15 +216,15 @@ return true; // ! [ov:replace_node] bool ov_replace_node(std::shared_ptr node) { - // Step 1. Verify that node has opset8::Negative type - auto neg = std::dynamic_pointer_cast(node); + // Step 1. Verify that node is of type ov::op::v0::Negative + auto neg = std::dynamic_pointer_cast(node); if (!neg) { return false; } - // Step 2. Create opset8::Multiply operation where the first input is negative operation input and second as Constant with -1 value - auto mul = std::make_shared(neg->input_value(0), - ov::opset8::Constant::create(neg->get_element_type(), ov::Shape{1}, {-1})); + // Step 2. Create ov::op::v1::Multiply operation with the first input being the output going into Negative and second as Constant with -1 value + auto mul = std::make_shared(neg->input_value(0), + ov::op::v0::Constant::create(neg->get_element_type(), ov::Shape{1}, {-1})); mul->set_friendly_name(neg->get_friendly_name()); ov::copy_runtime_info(neg, mul); @@ -233,18 +233,18 @@ bool ov_replace_node(std::shared_ptr node) { ov::replace_node(neg, mul); return true; - // Step 4. Negative operation will be removed automatically because all consumers was moved to Multiply operation + // Step 4. Negative operation will be removed automatically because all consumers were moved to Multiply operation } // ! [ov:replace_node] bool ov_manual_replace_node(std::shared_ptr node) { -auto neg = std::dynamic_pointer_cast(node); +auto neg = std::dynamic_pointer_cast(node); if (!neg) { return false; } -auto mul = std::make_shared(neg->input_value(0), - ov::opset8::Constant::create(neg->get_element_type(), ov::Shape{1}, {-1})); +auto mul = std::make_shared(neg->input_value(0), + ov::op::v0::Constant::create(neg->get_element_type(), ov::Shape{1}, {-1})); mul->set_friendly_name(neg->get_friendly_name()); ov::copy_runtime_info(neg, mul); @@ -262,8 +262,8 @@ void insert_example(std::shared_ptr node) { // Get all consumers for node auto consumers = node->output(0).get_target_inputs(); - // Step 2. Create new node. Let it be opset8::Relu. - auto new_node = std::make_shared(node); + // Step 2. Create new node ov::op::v0::Relu. + auto new_node = std::make_shared(node); // Step 3. Reconnect all consumers to new_node for (auto input : consumers) { @@ -277,7 +277,7 @@ void insert_example_with_copy(std::shared_ptr node) { // Make a node copy auto node_copy = node->clone_with_new_inputs(node->input_values()); // Create new node - auto new_node = std::make_shared(node_copy); + auto new_node = std::make_shared(node_copy); ov::replace_node(node, new_node); } // ! [ov:insert_node_with_copy] @@ -290,12 +290,12 @@ bool success = ov::replace_output_update_name(node->output(0), node->input_value } void replace_friendly_name() { -auto div = std::make_shared(); +auto div = std::make_shared(); // ! [ov:replace_friendly_name] // Replace Div operation with Power and Multiply sub-graph and set original friendly name to Multiply operation -auto pow = std::make_shared(div->input(1).get_source_output(), +auto pow = std::make_shared(div->input(1).get_source_output(), ov::op::v0::Constant::create(div->get_input_element_type(1), ov::Shape{1}, {-1})); -auto mul = std::make_shared(div->input(0).get_source_output(), pow); +auto mul = std::make_shared(div->input(0).get_source_output(), pow); mul->set_friendly_name(div->get_friendly_name()); ov::replace_node(div, mul); // ! [ov:replace_friendly_name] @@ -304,10 +304,10 @@ ov::replace_node(div, mul); void constant_subgraph() { // ! [ov:constant_subgraph] // After ConstantFolding pass Power will be replaced with Constant -auto input = std::make_shared(ov::element::f32, ov::Shape{1}); -auto pow = std::make_shared(ov::opset8::Constant::create(ov::element::f32, ov::Shape{1}, {2}), - ov::opset8::Constant::create(ov::element::f32, ov::Shape{1}, {3})); -auto mul = std::make_shared(input /* not constant input */, pow); +auto input = std::make_shared(ov::element::f32, ov::Shape{1}); +auto pow = std::make_shared(ov::op::v0::Constant::create(ov::element::f32, ov::Shape{1}, {2}), + ov::op::v0::Constant::create(ov::element::f32, ov::Shape{1}, {3})); +auto mul = std::make_shared(input /* not constant input */, pow); // ! [ov:constant_subgraph] } diff --git a/docs/articles_en/assets/snippets/ov_patterns.cpp b/docs/articles_en/assets/snippets/ov_patterns.cpp new file mode 100644 index 00000000000000..0382468a01c0e7 --- /dev/null +++ b/docs/articles_en/assets/snippets/ov_patterns.cpp @@ -0,0 +1,251 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 + +// ! [ov:imports] +#include + +#include "common_test_utils/matcher.hpp" +#include "openvino/op/abs.hpp" +#include "openvino/op/add.hpp" +#include "openvino/op/matmul.hpp" +#include "openvino/op/parameter.hpp" +#include "openvino/op/relu.hpp" +#include "openvino/op/sigmoid.hpp" +#include "openvino/pass/pattern/op/optional.hpp" +#include "openvino/pass/pattern/op/or.hpp" +#include "openvino/pass/pattern/op/wrap_type.hpp" +#include "transformations/utils/utils.hpp" + +using namespace ov; +using namespace ov::pass; +using namespace std; +// ! [ov:imports] + +// ! [ov:create_simple_model_and_pattern] +TEST(pattern, simple_model_and_pattern) { + // Create a sample model + PartialShape shape{2, 2}; + auto model_param1 = std::make_shared(element::i32, shape); + auto model_param2 = std::make_shared(element::i32, shape); + auto model_add = std::make_shared(model_param1->output(0), model_param2->output(0)); + auto model_param3 = std::make_shared(element::i32, shape); + auto model_mul = std::make_shared(model_add->output(0), model_param3->output(0), false, false); + auto model_abs = std::make_shared(model_mul->output(0)); + auto model_relu = std::make_shared(model_abs->output(0)); + auto model_result = std::make_shared(model_relu->output(0)); + + // Create a sample model + auto pattern_mul = std::make_shared(pattern::any_input(), pattern::any_input(), false, false); + auto pattern_abs = std::make_shared(pattern_mul->output(0)); + auto pattern_relu = std::make_shared(pattern_abs->output(0)); + + // Create a matcher and try to match the nodes + TestMatcher tm; + + // Should perfectly match + ASSERT_TRUE(tm.match(pattern_relu, model_relu)); +} +// ! [ov:create_simple_model_and_pattern] + + +// ! [ov:create_simple_model_and_pattern_wrap_type] +TEST(pattern, simple_model_and_pattern_wrap_type) { + // Create a sample model + PartialShape shape{2, 2}; + auto model_param1 = std::make_shared(element::i32, shape); + auto model_param2 = std::make_shared(element::i32, shape); + auto model_add = std::make_shared(model_param1->output(0), model_param2->output(0)); + auto model_param3 = std::make_shared(element::i32, shape); + auto model_mul = std::make_shared(model_add->output(0), model_param3->output(0), false, false); + auto model_abs = std::make_shared(model_mul->output(0)); + auto model_relu = std::make_shared(model_abs->output(0)); + auto model_result = std::make_shared(model_relu->output(0)); + + // Create a sample model + auto pattern_mul = ov::pass::pattern::wrap_type({pattern::any_input(), pattern::any_input()}); + auto pattern_abs = ov::pass::pattern::wrap_type({pattern_mul->output(0)}); + auto pattern_relu = ov::pass::pattern::wrap_type({pattern_abs->output(0)}); + + // Create a matcher and try to match the nodes + TestMatcher tm; + + // Should perfectly match + ASSERT_TRUE(tm.match(pattern_relu, model_relu)); +} +// ! [ov:create_simple_model_and_pattern_wrap_type] + + +// ! [ov:wrap_type_list] +TEST(pattern, wrap_type_list) { + // Create a sample model + PartialShape shape{2, 2}; + auto model_param1 = std::make_shared(element::i32, shape); + auto model_param2 = std::make_shared(element::i32, shape); + auto model_add = std::make_shared(model_param1->output(0), model_param2->output(0)); + auto model_param3 = std::make_shared(element::i32, shape); + auto model_mul = std::make_shared(model_add->output(0), model_param3->output(0), false, false); + auto model_abs = std::make_shared(model_mul->output(0)); + auto model_relu = std::make_shared(model_abs->output(0)); + auto model_result = std::make_shared(model_relu->output(0)); + auto model_sig = std::make_shared(model_abs->output(0)); + auto model_result1 = std::make_shared(model_sig->output(0)); + + // Create a sample model + auto pattern_mul = ov::pass::pattern::wrap_type({pattern::any_input(), pattern::any_input()}); + auto pattern_abs = ov::pass::pattern::wrap_type({pattern_mul->output(0)}); + auto pattern_relu = ov::pass::pattern::wrap_type({pattern_abs->output(0)}); + + // Create a matcher and try to match the nodes + TestMatcher tm; + + // The same pattern perfectly matches 2 different nodes + ASSERT_TRUE(tm.match(pattern_relu, model_relu)); + ASSERT_TRUE(tm.match(pattern_relu, model_sig)); +} +// ! [ov:wrap_type_list] + +void patterns_misc() { +// ! [ov:any_input] + auto pattern_mul = ov::pass::pattern::wrap_type({pattern::any_input(), pattern::any_input()}); + auto pattern_abs = ov::pass::pattern::wrap_type({pattern_mul->output(0)}); + auto pattern_relu = ov::pass::pattern::wrap_type({pattern_abs->output(0)}); +// ! [ov:any_input] + +// ! [ov:wrap_type_predicate] + ov::pass::pattern::wrap_type({pattern::any_input()}, pattern::consumers_count(2)); +// ! [ov:wrap_type_predicate] + + +// ! [ov:any_input_predicate] + auto pattern_mul = ov::pass::pattern::wrap_type({pattern::any_input([](const Output& value){ + return value.get_shape().size() == 4;}), + pattern::any_input([](const Output& value){ + return value.get_shape().size() == 4;})}); + auto pattern_abs = ov::pass::pattern::wrap_type({pattern_mul->output(0)}); + auto pattern_relu = ov::pass::pattern::wrap_type({pattern_abs->output(0)}); +// ! [ov:any_input_predicate] + + +// ! [ov:optional_predicate] + auto pattern_sig_opt = ov::pass::pattern::optional(pattern_relu, pattern::consumers_count(2)); +// ! [ov:optional_predicate] +} + + +// ! [ov:pattern_or] +TEST(pattern, pattern_or) { + // Create a sample model + PartialShape shape{2, 2}; + auto model_param1 = std::make_shared(element::i32, shape); + auto model_param2 = std::make_shared(element::i32, shape); + auto model_add = std::make_shared(model_param1->output(0), model_param2->output(0)); + auto model_param3 = std::make_shared(element::i32, shape); + auto model_mul = std::make_shared(model_add->output(0), model_param3->output(0), false, false); + auto model_abs = std::make_shared(model_mul->output(0)); + auto model_relu = std::make_shared(model_abs->output(0)); + auto model_result = std::make_shared(model_relu->output(0)); + + // Create a red branch + auto red_pattern_add = ov::pass::pattern::wrap_type({pattern::any_input(), pattern::any_input()}); + auto red_pattern_relu = ov::pass::pattern::wrap_type({red_pattern_add->output(0)}); + auto red_pattern_sigmoid = ov::pass::pattern::wrap_type({red_pattern_relu->output(0)}); + + // Create a blue branch + auto blue_pattern_mul = ov::pass::pattern::wrap_type({pattern::any_input(), pattern::any_input()}); + auto blue_pattern_abs = ov::pass::pattern::wrap_type({blue_pattern_mul->output(0)}); + auto blue_pattern_relu = ov::pass::pattern::wrap_type({blue_pattern_abs->output(0)}); + + // Create Or node + auto pattern_or = std::make_shared(OutputVector{red_pattern_sigmoid->output(0), blue_pattern_relu->output(0)}); + + // Create a matcher and try to match the nodes + TestMatcher tm; + + // The same pattern perfectly matches 2 different nodes + ASSERT_TRUE(tm.match(pattern_or, model_relu)); +} +// ! [ov:pattern_or] + + +// ! [ov:pattern_optional_middle] +TEST(pattern, pattern_optional_middle) { + // Create a sample model + PartialShape shape{2, 2}; + auto model_param1 = std::make_shared(element::i32, shape); + auto model_param2 = std::make_shared(element::i32, shape); + auto model_add = std::make_shared(model_param1->output(0), model_param2->output(0)); + auto model_param3 = std::make_shared(element::i32, shape); + auto model_mul = std::make_shared(model_add->output(0), model_param3->output(0), false, false); + auto model_abs = std::make_shared(model_mul->output(0)); + auto model_relu = std::make_shared(model_abs->output(0)); + auto model_result = std::make_shared(model_relu->output(0)); + + // Create a sample pattern with an Optional node in the middle + auto pattern_mul = ov::pass::pattern::wrap_type({pattern::any_input(), pattern::any_input()}); + auto pattern_abs = ov::pass::pattern::wrap_type({pattern_mul->output(0)}); + auto pattern_sig_opt = ov::pass::pattern::optional({pattern_abs->output(0)}); + auto pattern_relu = ov::pass::pattern::wrap_type({pattern_sig_opt->output(0)}); + + // Create a matcher and try to match the nodes + TestMatcher tm; + + // Should perfectly match + ASSERT_TRUE(tm.match(pattern_relu, model_relu)); +} +// ! [ov:pattern_optional_middle] + + +// ! [ov:pattern_optional_top] +TEST(pattern, pattern_optional_top) { + // Create a sample model + PartialShape shape{2, 2}; + auto model_param1 = std::make_shared(element::i32, shape); + auto model_param2 = std::make_shared(element::i32, shape); + auto model_add = std::make_shared(model_param1->output(0), model_param2->output(0)); + auto model_param3 = std::make_shared(element::i32, shape); + auto model_mul = std::make_shared(model_add->output(0), model_param3->output(0), false, false); + auto model_abs = std::make_shared(model_mul->output(0)); + auto model_relu = std::make_shared(model_abs->output(0)); + auto model_result = std::make_shared(model_relu->output(0)); + + // Create a sample pattern an optional top node + auto pattern_sig_opt = ov::pass::pattern::optional(pattern::any_input()); + auto pattern_mul = ov::pass::pattern::wrap_type({pattern_sig_opt, pattern::any_input()}); + auto pattern_abs = ov::pass::pattern::wrap_type({pattern_mul->output(0)}); + auto pattern_relu = ov::pass::pattern::wrap_type({pattern_abs->output(0)}); + + // Create a matcher and try to match the nodes + TestMatcher tm; + + // Should perfectly match + ASSERT_TRUE(tm.match(pattern_relu, model_relu)); +} +// ! [ov:pattern_optional_top] + + +// ! [ov:pattern_optional_root] +TEST(pattern, pattern_optional_root) { + // Create a sample model + PartialShape shape{2, 2}; + auto model_param1 = std::make_shared(element::i32, shape); + auto model_param2 = std::make_shared(element::i32, shape); + auto model_add = std::make_shared(model_param1->output(0), model_param2->output(0)); + auto model_param3 = std::make_shared(element::i32, shape); + auto model_mul = std::make_shared(model_add->output(0), model_param3->output(0), false, false); + auto model_abs = std::make_shared(model_mul->output(0)); + auto model_relu = std::make_shared(model_abs->output(0)); + auto model_result = std::make_shared(model_relu->output(0)); + + // Create a sample pattern an optional top node + auto pattern_mul = ov::pass::pattern::wrap_type({pattern::any_input(), pattern::any_input()}); + auto pattern_abs = ov::pass::pattern::wrap_type({pattern_mul->output(0)}); + auto pattern_relu = ov::pass::pattern::wrap_type({pattern_abs->output(0)}); + auto pattern_sig_opt = ov::pass::pattern::optional(pattern_relu); + + // Create a matcher and try to match the nodes + TestMatcher tm; + + // Should perfectly match + ASSERT_TRUE(tm.match(pattern_relu, model_relu)); +} +// ! [ov:pattern_optional_root] \ No newline at end of file diff --git a/docs/articles_en/assets/snippets/ov_patterns.py b/docs/articles_en/assets/snippets/ov_patterns.py new file mode 100644 index 00000000000000..ac9536a3e7a67a --- /dev/null +++ b/docs/articles_en/assets/snippets/ov_patterns.py @@ -0,0 +1,217 @@ +# Copyright (C) 2018-2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +# ! [ov:imports] +import pytest +from openvino import PartialShape +from openvino.runtime import opset13 as ops +from openvino.runtime.passes import Matcher, WrapType, Or, AnyInput, Optional +# ! [ov:imports] +from openvino.runtime.passes import ( + consumers_count, + has_static_dim, + has_static_dims, + has_static_shape, + has_static_rank, + type_matches, + type_matches_any, +) + +# ! [ov:create_simple_model_and_pattern] +def simple_model_and_pattern(): + # Create a sample model + model_param1 = ops.parameter(PartialShape([2, 2])) + model_param2 = ops.parameter(PartialShape([2, 2])) + model_add = ops.add(model_param1, model_param2) + model_param3 = ops.parameter(PartialShape([2, 2])) + model_mul = ops.matmul(model_add, model_param3, False, False) + model_abs = ops.abs(model_mul) + model_relu = ops.relu(model_abs) + model_result = ops.result(model_relu) + + # Create a sample pattern + pattern_mul = ops.matmul(AnyInput(), AnyInput(), False, False) + pattern_abs = ops.abs(pattern_mul) + pattern_relu = ops.relu(pattern_abs) + + # Create a matcher and try to match the nodes + matcher = Matcher(pattern_relu, "FindPattern") + + # Should perfectly match + assert matcher.match(model_relu) +# ! [ov:create_simple_model_and_pattern] + +# ! [ov:create_simple_model_and_pattern_wrap_type] +def simple_model_and_pattern_wrap_type(): + model_param1 = ops.parameter(PartialShape([2, 2])) + model_param2 = ops.parameter(PartialShape([2, 2])) + model_add = ops.add(model_param1, model_param2) + model_param3 = ops.parameter(PartialShape([2, 2])) + model_mul = ops.matmul(model_add, model_param3, False, False) + model_abs = ops.abs(model_mul) + model_relu = ops.relu(model_abs) + model_result = ops.result(model_relu) + + # Create a sample pattern + pattern_mul = WrapType("opset13.MatMul", [AnyInput(), AnyInput()]) + pattern_abs = WrapType("opset13.Abs", pattern_mul) + pattern_relu = WrapType("opset13.Relu", pattern_abs) + + # Create a matcher and try to match the nodes + matcher = Matcher(pattern_relu, "FindPattern") + + # Should perfectly match + assert matcher.match(model_relu) +# ! [ov:create_simple_model_and_pattern_wrap_type] + +# ! [ov:wrap_type_list] +def wrap_type_list(): + model_param1 = ops.parameter(PartialShape([2, 2])) + model_param2 = ops.parameter(PartialShape([2, 2])) + model_add = ops.add(model_param1, model_param2) + model_param3 = ops.parameter(PartialShape([2, 2])) + model_mul = ops.matmul(model_add, model_param3, False, False) + model_abs = ops.abs(model_mul) + model_relu = ops.relu(model_abs) + model_result = ops.result(model_relu) + model_sig = ops.sigmoid(model_abs) # Note that we've added a Sigmoid node after Abs + model_result1 = ops.result(model_sig) + + # Create a sample pattern + pattern_mul = WrapType("opset13.MatMul", [AnyInput(), AnyInput()]) + pattern_abs = WrapType("opset13.Abs", pattern_mul) + pattern_relu = WrapType(["opset13.Relu", "opset13.Sigmoid"], pattern_abs) + + # Create a matcher and try to match the nodes + matcher = Matcher(pattern_relu, "FindPattern") + + # The same pattern perfectly matches 2 different nodes + assert matcher.match(model_relu) + assert matcher.match(model_sig) +# ! [ov:wrap_type_list] + +def any_input(): +# ! [ov:any_input] + # Create a pattern with a MatMul node taking any inputs. + pattern_mul = WrapType("opset13.MatMul", [AnyInput(), AnyInput()]) + pattern_abs = WrapType("opset13.Abs", pattern_mul) + pattern_relu = WrapType("opset13.Relu", pattern_abs) +# ! [ov:any_input] + +def wrap_type_predicate(): +# ! [ov:wrap_type_predicate] + WrapType("opset13.Relu", AnyInput(), consumers_count(2)) +# ! [ov:wrap_type_predicate] + +# ! [ov:any_input_predicate] + # Create a pattern with an MatMul node taking any input that has a rank 4. + pattern_mul = WrapType("opset13.MatMul", [AnyInput(lambda output: len(output.get_shape()) == 4), AnyInput(lambda output: len(output.get_shape()) == 4)]) + pattern_abs = WrapType("opset13.Abs", pattern_mul) + pattern_relu = WrapType("opset13.Relu", pattern_abs) +# ! [ov:any_input_predicate] + +# ! [ov:pattern_or] +def pattern_or(): + model_param1 = ops.parameter(PartialShape([2, 2])) + model_param2 = ops.parameter(PartialShape([2, 2])) + model_add = ops.add(model_param1, model_param2) + model_param3 = ops.parameter(PartialShape([2, 2])) + model_mul = ops.matmul(model_add, model_param3, False, False) + model_abs = ops.abs(model_mul) + model_relu = ops.relu(model_abs) + model_result = ops.result(model_relu) + + # Create a red branch + red_pattern_add = WrapType("opset13.Add", [AnyInput(), AnyInput()]) + red_pattern_relu = WrapType("opset13.Relu", red_pattern_add) + red_pattern_sigmoid = WrapType(["opset13.Sigmoid"], red_pattern_relu) + + # Create a blue branch + blue_pattern_mul = WrapType("opset13.MatMul", [AnyInput(), AnyInput()]) + blue_pattern_abs = WrapType("opset13.Abs", blue_pattern_mul) + blue_pattern_relu = WrapType(["opset13.Relu"], blue_pattern_abs) + + #Create Or node + pattern_or = Or([red_pattern_sigmoid, blue_pattern_relu]) + + # Create a matcher and try to match the nodes + matcher = Matcher(pattern_or, "FindPattern") + + # The same pattern perfectly matches 2 different nodes + assert matcher.match(model_relu) +# ! [ov:pattern_or] + +# ! [ov:pattern_optional_middle] +def pattern_optional_middle(): + model_param1 = ops.parameter(PartialShape([2, 2])) + model_param2 = ops.parameter(PartialShape([2, 2])) + model_add = ops.add(model_param1, model_param2) + model_param3 = ops.parameter(PartialShape([2, 2])) + model_mul = ops.matmul(model_add, model_param3, False, False) + model_abs = ops.abs(model_mul) + model_relu = ops.relu(model_abs) + model_result = ops.result(model_relu) + + # Create a sample pattern with an Optional node in the middle + pattern_mul = WrapType("opset13.MatMul", [AnyInput(), AnyInput()]) + pattern_abs = WrapType("opset13.Abs", pattern_mul) + pattern_sig_opt = Optional(["opset13.Sigmoid"], pattern_abs) + pattern_relu = WrapType("opset13.Relu", pattern_sig_opt) + + # Create a matcher and try to match the nodes + matcher = Matcher(pattern_relu, "FindPattern") + + # Should perfectly match + assert matcher.match(model_relu) +# ! [ov:pattern_optional_middle] + +# ! [ov:pattern_optional_top] +def pattern_optional_top(): + model_param1 = ops.parameter(PartialShape([2, 2])) + model_param2 = ops.parameter(PartialShape([2, 2])) + model_add = ops.add(model_param1, model_param2) + model_param3 = ops.parameter(PartialShape([2, 2])) + model_mul = ops.matmul(model_add, model_param3, False, False) + model_abs = ops.abs(model_mul) + model_relu = ops.relu(model_abs) + model_result = ops.result(model_relu) + + # Create a sample pattern an optional top node + pattern_sig_opt = Optional(["opset13.Sigmoid"], AnyInput()) + pattern_mul = WrapType("opset13.MatMul", [pattern_sig_opt, AnyInput()]) + pattern_abs = WrapType("opset13.Abs", pattern_mul) + pattern_relu = WrapType("opset13.Relu", pattern_abs) + + matcher = Matcher(pattern_relu, "FindPattern") + + # Should perfectly match even though there's no Sigmoid going into MatMul + assert matcher.match(model_relu) +# ! [ov:pattern_optional_top] + +# ! [ov:pattern_optional_root] +def pattern_optional_root(): + model_param1 = ops.parameter(PartialShape([2, 2])) + model_param2 = ops.parameter(PartialShape([2, 2])) + model_add = ops.add(model_param1, model_param2) + model_param3 = ops.parameter(PartialShape([2, 2])) + model_mul = ops.matmul(model_add, model_param3, False, False) + model_abs = ops.abs(model_mul) + model_relu = ops.relu(model_abs) + model_result = ops.result(model_relu) + + # Create a sample pattern + pattern_mul = WrapType("opset13.MatMul", [AnyInput(), AnyInput()]) + pattern_abs = WrapType("opset13.Abs", pattern_mul) + pattern_relu = WrapType("opset13.Relu", pattern_abs) + pattern_sig_opt = Optional(["opset13.Sigmoid"], pattern_relu) + + matcher = Matcher(pattern_sig_opt, "FindPattern") + + # Should perfectly match even though there's no Sigmoid as root + assert matcher.match(model_relu) +# ! [ov:pattern_optional_root] + +# ! [ov:optional_predicate] + pattern_sig_opt = Optional(["opset13.Sigmoid"], pattern_relu, consumers_count(1)) +# ! [ov:optional_predicate] diff --git a/docs/articles_en/assets/snippets/utils.py b/docs/articles_en/assets/snippets/utils.py new file mode 100644 index 00000000000000..22dd1ae1273f6a --- /dev/null +++ b/docs/articles_en/assets/snippets/utils.py @@ -0,0 +1,65 @@ +# Copyright (C) 2018-2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +import tempfile +import numpy as np +from sys import platform + +import openvino as ov +import openvino.runtime.opset13 as ops + + +def get_dynamic_model(): + param = ops.parameter(ov.PartialShape([-1, -1]), name="input") + return ov.Model(ops.relu(param), [param]) + + +def get_model(input_shape = None, input_dtype=np.float32) -> ov.Model: + if input_shape is None: + input_shape = [1, 3, 32, 32] + param = ops.parameter(input_shape, input_dtype, name="input") + relu = ops.relu(param, name="relu") + relu.output(0).get_tensor().set_names({"result"}) + model = ov.Model([relu], [param], "test_model") + + assert model is not None + return model + + +def get_advanced_model(): + data = ops.parameter(ov.Shape([1, 3, 64, 64]), ov.Type.f32, name="input") + data1 = ops.parameter(ov.Shape([1, 3, 64, 64]), ov.Type.f32, name="input2") + axis_const = ops.constant(1, dtype=ov.Type.i64) + split = ops.split(data, axis_const, 3) + relu = ops.relu(split.output(1)) + relu.output(0).get_tensor().set_names({"result"}) + return ov.Model([split.output(0), relu.output(0), split.output(2)], [data, data1], "model") + + +def get_image(shape = (1, 3, 32, 32), dtype = "float32"): + np.random.seed(42) + return np.random.rand(*shape).astype(dtype) + + +def get_path_to_extension_library(): + library_path="" + if platform == "win32": + library_path="openvino_template_extension.dll" + else: + library_path="libopenvino_template_extension.so" + return library_path + + +def get_path_to_model(input_shape = None): + if input_shape is None: + input_shape = [1, 3, 32, 32] + path_to_xml = tempfile.NamedTemporaryFile(suffix="_model.xml").name + model = get_model(input_shape) + ov.serialize(model, path_to_xml) + return path_to_xml + + +def get_temp_dir(): + temp_dir = tempfile.TemporaryDirectory() + return temp_dir diff --git a/docs/articles_en/documentation/legacy-features.rst b/docs/articles_en/documentation/legacy-features.rst index dd5938398beccd..489e6e77fe4191 100644 --- a/docs/articles_en/documentation/legacy-features.rst +++ b/docs/articles_en/documentation/legacy-features.rst @@ -59,6 +59,9 @@ offering. until they are fully removed. | :doc:`See the Open Model ZOO documentation ` | `Check the OMZ GitHub project `__ +| As for public model databases, `Hugging Face `__ has + become the recommended model source for OpenVINO. + @@ -107,14 +110,6 @@ Discontinued: :language: cpp :fragment: [export_compiled_model] -.. dropdown:: DL Workbench - - | *New solution:* DevCloud version - | *Old solution:* local distribution discontinued in OpenVINO 2022.3 - | The stand-alone version of DL Workbench, a GUI tool for previewing and benchmarking - deep learning models, has been discontinued. You can use its cloud version: - | `Intel® Developer Cloud for the Edge `__. - .. dropdown:: TensorFlow integration (OVTF) | *New solution:* Direct model support and OpenVINO Converter (OVC) diff --git a/docs/articles_en/documentation/legacy-features/install-dev-tools.rst b/docs/articles_en/documentation/legacy-features/install-dev-tools.rst index 6466eb8711a381..de2b35529aeafc 100644 --- a/docs/articles_en/documentation/legacy-features/install-dev-tools.rst +++ b/docs/articles_en/documentation/legacy-features/install-dev-tools.rst @@ -8,14 +8,18 @@ Install OpenVINO™ Development Tools :description: Learn how to install OpenVINO™ Development Tools on Windows, Linux, and macOS operating systems, using a PyPi package. -OpenVINO Development Tools is a set of utilities that make it easy to develop and optimize models and applications for OpenVINO. It provides the following tools: +OpenVINO Development Tools is a set of utilities that make it easy to develop and +optimize models and applications for OpenVINO. It provides the following tools: * Model conversion API * Benchmark Tool * Accuracy Checker and Annotation Converter * Model Downloader and other Open Model Zoo tools -The instructions on this page show how to install OpenVINO Development Tools. If you are a Python developer, it only takes a few simple steps to install the tools with PyPI. If you are developing in C/C++, OpenVINO Runtime must be installed separately before installing OpenVINO Development Tools. +The instructions on this page show how to install OpenVINO Development Tools. If you are a +Python developer, it only takes a few simple steps to install the tools with PyPI. If you +are developing in C/C++, OpenVINO Runtime must be installed separately before installing +OpenVINO Development Tools. In both cases, Python 3.8 - 3.11 needs to be installed on your machine before starting. @@ -28,16 +32,28 @@ In both cases, Python 3.8 - 3.11 needs to be installed on your machine before st For Python Developers ##################### -If you are a Python developer, follow the steps in the :ref:`Installing OpenVINO Development Tools ` section on this page to install it. Installing OpenVINO Development Tools will also install OpenVINO Runtime as a dependency, so you don’t need to install OpenVINO Runtime separately. This option is recommended for new users. +If you are a Python developer, follow the steps in the +:ref:`Installing OpenVINO Development Tools ` section on this page to +install it. Installing OpenVINO Development Tools will also install OpenVINO Runtime as +a dependency, so you don’t need to install OpenVINO Runtime separately. This option is +recommended for new users. .. _cpp_developers: For C/C++ Developers ####################### -If you are a C/C++ developer, you must first install OpenVINO Runtime separately to set up the C/C++ libraries, sample code, and dependencies for building applications with OpenVINO. These files are not included with the PyPI distribution. See the :doc:`Selector Tool <../../get-started/install-openvino>` page to install OpenVINO Runtime from an archive file for your operating system. +If you are a C/C++ developer, you must first install OpenVINO Runtime separately to set +up the C/C++ libraries, sample code, and dependencies for building applications with +OpenVINO. These files are not included with the PyPI distribution. See the +:doc:`Selector Tool <../../get-started/install-openvino>` page to install OpenVINO Runtime +from an archive file for your operating system. -Once OpenVINO Runtime is installed, you may install OpenVINO Development Tools for access to tools like ``mo``, Model Downloader, Benchmark Tool, and other utilities that will help you optimize your model and develop your application. Follow the steps in the :ref:`Installing OpenVINO Development Tools ` section on this page to install it. +Once OpenVINO Runtime is installed, you may install OpenVINO Development Tools for access +to tools like ``mo``, Model Downloader, Benchmark Tool, and other utilities that will help +you optimize your model and develop your application. Follow the steps in the +:ref:`Installing OpenVINO Development Tools ` section on this page +to install it. .. _install_dev_tools: @@ -45,14 +61,16 @@ Installing OpenVINO™ Development Tools ###################################### Follow these step-by-step instructions to install OpenVINO Development Tools on your computer. -There are two options to install OpenVINO Development Tools: installation into an existing environment with a deep learning framework that was used -for model training or creation; or installation into a new environment. +There are two options to install OpenVINO Development Tools: installation into an existing +environment with a deep learning framework that was used for model training or creation; +or installation into a new environment. Installation into an Existing Environment with the Source Deep Learning Framework +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -To install OpenVINO Development Tools (see the :ref:`Install the Package ` section of this article) into an existing environment -with the deep learning framework used for the model training or creation, run the following command: +To install OpenVINO Development Tools (see the :ref:`Install the Package ` +section of this article) into an existing environment with the deep learning framework used +for the model training or creation, run the following command: .. code-block:: sh @@ -62,13 +80,16 @@ with the deep learning framework used for the model training or creation, run th Installation in a New Environment +++++++++++++++++++++++++++++++++ -If you do not have an environment with a deep learning framework for the input model or you encounter any compatibility issues between OpenVINO -and your version of deep learning framework, you may install OpenVINO Development Tools with validated versions of frameworks into a new environment. +If you do not have an environment with a deep learning framework for the input model or you +encounter any compatibility issues between OpenVINO and your version of deep learning +framework, you may install OpenVINO Development Tools with validated versions of +frameworks into a new environment. Step 1. Set Up Python Virtual Environment ----------------------------------------- -Create a virtual Python environment to avoid dependency conflicts. To create a virtual environment, use the following command: +Create a virtual Python environment to avoid dependency conflicts. To create a virtual +environment, use the following command: .. tab-set:: @@ -117,7 +138,8 @@ Activate the newly created Python virtual environment by issuing this command: Step 3. Set Up and Update PIP to the Highest Version ---------------------------------------------------- -Make sure `pip` is installed in your environment and upgrade it to the latest version by issuing the following command: +Make sure `pip` is installed in your environment and upgrade it to the latest version by +issuing the following command: .. code-block:: sh @@ -129,7 +151,8 @@ Make sure `pip` is installed in your environment and upgrade it to the latest ve Step 4. Install the Package --------------------------- -To install and configure the components of the development package together with validated versions of specific frameworks, use the commands below. +To install and configure the components of the development package together with validated +versions of specific frameworks, use the commands below. .. code-block:: sh @@ -137,9 +160,11 @@ To install and configure the components of the development package together with where the ``extras`` parameter specifies the source deep learning framework for the input model -and is one or more of the following values separated with "," : ``onnx``, ``pytorch``, ``tensorflow``, ``tensorflow2``. +and is one or more of the following values separated with "," : ``onnx``, ``pytorch``, +``tensorflow``, ``tensorflow2``. -For example, to install and configure dependencies required for working with TensorFlow 2.x and ONNX models, use the following command: +For example, to install and configure dependencies required for working with TensorFlow 2.x +and ONNX models, use the following command: .. code-block:: sh @@ -148,9 +173,14 @@ For example, to install and configure dependencies required for working with Ten .. note:: - Model conversion API support for TensorFlow 1.x environment has been deprecated. Use the ``tensorflow2`` parameter to install a TensorFlow 2.x environment that can convert both TensorFlow 1.x and 2.x models. If your model isn't compatible with the TensorFlow 2.x environment, use the `tensorflow` parameter to install the TensorFlow 1.x environment. The TF 1.x environment is provided only for legacy compatibility reasons. + Model conversion API support for TensorFlow 1.x environment has been deprecated. Use the + ``tensorflow2`` parameter to install a TensorFlow 2.x environment that can convert both + TensorFlow 1.x and 2.x models. If your model isn't compatible with the TensorFlow 2.x + environment, use the `tensorflow` parameter to install the TensorFlow 1.x environment. + The TF 1.x environment is provided only for legacy compatibility reasons. -For more details on the openvino-dev PyPI package, see `pypi.org `__ . +For more details on the openvino-dev PyPI package, see +`pypi.org `__ . Step 5. Test the Installation ------------------------------ @@ -161,9 +191,13 @@ To verify the package is properly installed, run the command below (this may tak mo -h -You will see the help message for ``mo`` if installation finished successfully. If you get an error, refer to the :doc:`Troubleshooting Guide <../../get-started/troubleshooting-install-config>` for possible solutions. +You will see the help message for ``mo`` if installation finished successfully. If you get an +error, refer to the :doc:`Troubleshooting Guide <../../get-started/troubleshooting-install-config>` +for possible solutions. -Congratulations! You finished installing OpenVINO Development Tools with C/C++ capability. Now you can start exploring OpenVINO's functionality through example C/C++ applications. See the "What's Next?" section to learn more! +Congratulations! You finished installing OpenVINO Development Tools with C/C++ capability. +Now you can start exploring OpenVINO's functionality through example C/C++ applications. +See the "What's Next?" section to learn more! What's Next? ############ @@ -176,9 +210,12 @@ Get started with Python .. image:: ../../assets/images/get_started_with_python.gif :width: 400 -Try the `Python Quick Start Example <../../notebooks/vision-monodepth-with-output.html>`__ to estimate depth in a scene using an OpenVINO monodepth model in a Jupyter Notebook inside your web browser. +Try the `Python Quick Start Example <../../notebooks/vision-monodepth-with-output.html>`__ +to estimate depth in a scene using an OpenVINO monodepth model in a Jupyter Notebook +inside your web browser. -Visit the :doc:`Tutorials <../../learn-openvino/interactive-tutorials-python>` page for more Jupyter Notebooks to get you started with OpenVINO, such as: +Visit the :doc:`Tutorials <../../learn-openvino/interactive-tutorials-python>` page for more +Jupyter Notebooks to get you started with OpenVINO, such as: * `OpenVINO Python API Tutorial <../../notebooks/openvino-api-with-output.html>`__ * `Basic image classification program with Hello Image Classification <../../notebooks/hello-world-with-output.html>`__ @@ -191,9 +228,11 @@ Get started with C++ :width: 400 -Try the :doc:`C++ Quick Start Example <../../learn-openvino/openvino-samples/get-started-demos>` for step-by-step instructions on building and running a basic image classification C++ application. +Try the :doc:`C++ Quick Start Example <../../learn-openvino/openvino-samples/get-started-demos>` +for step-by-step instructions on building and running a basic image classification C++ application. -Visit the :doc:`Samples <../../learn-openvino/openvino-samples>` page for other C++ example applications to get you started with OpenVINO, such as: +Visit the :doc:`Samples <../../learn-openvino/openvino-samples>` page for other C++ +example applications to get you started with OpenVINO, such as: * :doc:`Basic object detection with the Hello Reshape SSD C++ sample <../../learn-openvino/openvino-samples/hello-reshape-ssd>` * :doc:`Object classification sample <../../learn-openvino/openvino-samples/hello-classification>` @@ -201,10 +240,20 @@ Visit the :doc:`Samples <../../learn-openvino/openvino-samples>` page for other Learn OpenVINO Development Tools ++++++++++++++++++++++++++++++++ -* Explore a variety of pre-trained deep learning models in the :doc:`Open Model Zoo ` and deploy them in demo applications to see how they work. -* Want to import a model from another framework and optimize its performance with OpenVINO? Visit the :doc:`Convert a Model ` page. -* Accelerate your model's speed even further with quantization and other compression techniques using :doc:`Neural Network Compression Framework (NNCF) <../../openvino-workflow/model-optimization-guide/quantizing-models-post-training>`. -* Benchmark your model's inference speed with one simple command using the :doc:`Benchmark Tool <../../learn-openvino/openvino-samples/benchmark-tool>`. +* Explore a variety of pre-trained deep learning models in the + :doc:`Open Model Zoo ` and deploy them in demo applications to see how they work. + + .. important:: + + Due to the deprecation of Open Model Zoo, models in the OpenVINO IR format are now + published on `Hugging Face `__. + +* Want to import a model from another framework and optimize its performance with OpenVINO? + Visit the :doc:`Convert a Model ` page. +* Accelerate your model's speed even further with quantization and other compression techniques + using :doc:`Neural Network Compression Framework (NNCF) <../../openvino-workflow/model-optimization-guide/quantizing-models-post-training>`. +* Benchmark your model's inference speed with one simple command using the + :doc:`Benchmark Tool <../../learn-openvino/openvino-samples/benchmark-tool>`. Additional Resources #################### diff --git a/docs/articles_en/documentation/legacy-features/model-zoo.rst b/docs/articles_en/documentation/legacy-features/model-zoo.rst index 281e217f75c9c4..f2b26f2a983efc 100644 --- a/docs/articles_en/documentation/legacy-features/model-zoo.rst +++ b/docs/articles_en/documentation/legacy-features/model-zoo.rst @@ -29,9 +29,29 @@ Model Zoo ../../omz_model_api_ovms_adapter -Open Model Zoo for OpenVINO™ toolkit delivers a wide variety of free, pre-trained deep learning models and demo applications that provide full application templates to help you implement deep learning in Python, C++, or OpenCV Graph API (G-API). Models and demos are available in the `Open Model Zoo GitHub repo `__ and licensed under Apache License Version 2.0. - -Browse through over 200 neural network models, both :doc:`public <../../omz_models_group_public>` and from :doc:`Intel <../../omz_models_group_intel>`, and pick the right one for your solution. Types include object detection, classification, image segmentation, handwriting recognition, text to speech, pose estimation, and others. The Intel models have already been converted to work with OpenVINO™ toolkit, while public models can easily be converted using the :doc:`OpenVINO Model Conversion API <../../openvino-workflow/model-preparation>` utility. - -Get started with simple :doc:`step-by-step procedures <../../learn-openvino/openvino-samples/get-started-demos>` to learn how to build and run demo applications or discover the :doc:`full set of demos <../../omz_demos>` and adapt them for implementing specific deep learning scenarios in your applications. +.. note:: + + Since the deprecation of Open Model Zoo, OpenVINO has significantly extended its presence on the + `Hugging Face `__ model repository. It is currently + the recommended source of optimized OpenVINO IR models. + +Open Model Zoo for OpenVINO™ toolkit delivers a wide variety of free, pre-trained deep learning +models and demo applications that provide full application templates to help you implement deep +learning in Python, C++, or OpenCV Graph API (G-API). Models and demos are available in the +`Open Model Zoo GitHub repo `__ +and licensed under Apache License Version 2.0. + +Browse through over 200 neural network models, both +:doc:`public <../../omz_models_group_public>` and from +:doc:`Intel <../../omz_models_group_intel>`, and pick the right one for your solution. +Types include object detection, classification, image segmentation, handwriting recognition, +text to speech, pose estimation, and others. The Intel models have already been converted +to work with OpenVINO™ toolkit, while public models can easily be converted using the +:doc:`OpenVINO Model Conversion API <../../openvino-workflow/model-preparation>` utility. + +Get started with simple +:doc:`step-by-step procedures <../../learn-openvino/openvino-samples/get-started-demos>` +to learn how to build and run demo applications or discover the +:doc:`full set of demos <../../omz_demos>` and adapt them for implementing specific deep +learning scenarios in your applications. diff --git a/docs/articles_en/documentation/legacy-features/transition-legacy-conversion-api/legacy-conversion-api/[legacy]-model-optimizer-faq.rst b/docs/articles_en/documentation/legacy-features/transition-legacy-conversion-api/legacy-conversion-api/[legacy]-model-optimizer-faq.rst index 6cc2de65a8a9a0..fb7e3004258253 100644 --- a/docs/articles_en/documentation/legacy-features/transition-legacy-conversion-api/legacy-conversion-api/[legacy]-model-optimizer-faq.rst +++ b/docs/articles_en/documentation/legacy-features/transition-legacy-conversion-api/legacy-conversion-api/[legacy]-model-optimizer-faq.rst @@ -53,7 +53,7 @@ For example, to add the description of the ``CustomReshape`` layer, which is an python3 generate_caffe_pb2.py --input_proto /src/caffe/proto/caffe.proto - where ``PATH_TO_CUSTOM_CAFFE` is the path to the root directory of custom Caffe. + where ``PATH_TO_CUSTOM_CAFFE`` is the path to the root directory of custom Caffe. 3. Now, Model Optimizer is able to load the model into memory and start working with your extensions if there are any. @@ -927,7 +927,7 @@ Q102. What does the message "Operation _contrib_box_nms is not supported ..." me .. _question-103: -Q103. What does the message "ModelOptimizer is not able to parse *.caffemodel" mean? +Q103. What does the message "ModelOptimizer is not able to parse "\*.caffemodel" mean? ##################################################################################################################################################### **A:** If a ``*.caffemodel`` file exists and is correct, the error occurred possibly because of the use of Python protobuf implementation. In some cases, error messages may appear during model parsing, for example: "``utf-8`` codec can't decode byte 0xe0 in position 4: invalid continuation byte in field: mo_caffe.SpatialTransformerParameter.transform_type". You can either use a newer Python version (3.8 - 3.11) or build the ``cpp`` implementation of ``protobuf`` yourself for your version of Python. For the complete instructions about building ``protobuf`` from sources, see the appropriate section in the :doc:`Converting Models with Model Optimizer <../legacy-conversion-api>` guide. diff --git a/docs/articles_en/documentation/legacy-features/transition-legacy-conversion-api/legacy-conversion-api/[legacy]-supported-model-formats/[legacy]-conversion-tutorials/convert-tensorflow-deep-speech.rst b/docs/articles_en/documentation/legacy-features/transition-legacy-conversion-api/legacy-conversion-api/[legacy]-supported-model-formats/[legacy]-conversion-tutorials/convert-tensorflow-deep-speech.rst index 71c28a5db9205d..45157106165732 100644 --- a/docs/articles_en/documentation/legacy-features/transition-legacy-conversion-api/legacy-conversion-api/[legacy]-supported-model-formats/[legacy]-conversion-tutorials/convert-tensorflow-deep-speech.rst +++ b/docs/articles_en/documentation/legacy-features/transition-legacy-conversion-api/legacy-conversion-api/[legacy]-supported-model-formats/[legacy]-conversion-tutorials/convert-tensorflow-deep-speech.rst @@ -42,8 +42,8 @@ To download the model, follow the instruction below: 2. Download the `TensorFlow MetaGraph with pre-trained weights `__. 3. Unpack it with a file archiver application. -Freezing the Model into a *.pb File -################################### +Freezing the Model into a "\*.pb File" +###################################### After unpacking the archives above, you have to freeze the model. This requires TensorFlow version 1, which is not available under Python 3.8, so you need Python 3.7 or lower. @@ -75,7 +75,7 @@ a fixed input length shape, thus the model is not reshapable. Refer to the :doc:`Using Shape Inference <../../../../../../openvino-workflow/running-inference/changing-input-shape>` guide. The second is that the frozen model still has two variables: ``previous_state_c`` and ``previous_state_h``, figure -with the frozen *.pb model is below. It means that the model keeps training these variables at each inference. +with the frozen \*.pb model is below. It means that the model keeps training these variables at each inference. .. image:: ../../../../../../assets/images/DeepSpeech-0.8.2.png diff --git a/docs/articles_en/documentation/legacy-features/transition-legacy-conversion-api/legacy-conversion-api/[legacy]-supported-model-formats/[legacy]-conversion-tutorials/convert-tensorflow-efficient-det.rst b/docs/articles_en/documentation/legacy-features/transition-legacy-conversion-api/legacy-conversion-api/[legacy]-supported-model-formats/[legacy]-conversion-tutorials/convert-tensorflow-efficient-det.rst index 47a6342ff7c989..daebe992d8441c 100644 --- a/docs/articles_en/documentation/legacy-features/transition-legacy-conversion-api/legacy-conversion-api/[legacy]-supported-model-formats/[legacy]-conversion-tutorials/convert-tensorflow-efficient-det.rst +++ b/docs/articles_en/documentation/legacy-features/transition-legacy-conversion-api/legacy-conversion-api/[legacy]-supported-model-formats/[legacy]-conversion-tutorials/convert-tensorflow-efficient-det.rst @@ -5,7 +5,7 @@ Converting TensorFlow EfficientDet Models .. meta:: - :description: Learn how to convert an EfficientDet model + :description: Learn how to convert an EfficientDet model from TensorFlow to the OpenVINO Intermediate Representation. @@ -14,7 +14,7 @@ Converting TensorFlow EfficientDet Models The code described here has been **deprecated!** Do not use it to avoid working with a legacy solution. It will be kept for some time to ensure backwards compatibility, but **you should not use** it in contemporary applications. This guide describes a deprecated conversion method. The guide on the new and recommended method can be found in the :doc:`Python tutorials <../../../../../../learn-openvino/interactive-tutorials-python>`. - + This tutorial explains how to convert EfficientDet public object detection models to the Intermediate Representation (IR). .. _efficientdet-to-ir: @@ -55,9 +55,15 @@ The attribute ``image_size`` specifies the shape to be defined for the model con The color channel order (RGB or BGR) of an input data should match the channel order of the model training dataset. If they are different, perform the ``RGB<->BGR`` conversion specifying the command-line parameter: ``--reverse_input_channels``. Otherwise, inference results may be incorrect. For more information about the parameter, refer to the **When to Reverse Input Channels** section of the :doc:`Converting a Model to Intermediate Representation (IR) <../../[legacy]-setting-input-shapes>` guide. -OpenVINO toolkit provides samples that can be used to infer EfficientDet model. +OpenVINO toolkit provides samples that can be used to infer EfficientDet model. For more information, refer to the :doc:`Open Model Zoo Demos <../../../../../../omz_demos>`. +.. important:: + + Due to the deprecation of Open Model Zoo, models in the OpenVINO IR format have are now + published on `Hugging Face `__. + + Interpreting Results of the TensorFlow Model and the IR ####################################################### diff --git a/docs/articles_en/documentation/legacy-features/transition-legacy-conversion-api/legacy-conversion-api/[legacy]-supported-model-formats/[legacy]-conversion-tutorials/convert-tensorflow-object-detection.rst b/docs/articles_en/documentation/legacy-features/transition-legacy-conversion-api/legacy-conversion-api/[legacy]-supported-model-formats/[legacy]-conversion-tutorials/convert-tensorflow-object-detection.rst index 3eb5a93e6e0feb..8b1a04029962b1 100644 --- a/docs/articles_en/documentation/legacy-features/transition-legacy-conversion-api/legacy-conversion-api/[legacy]-supported-model-formats/[legacy]-conversion-tutorials/convert-tensorflow-object-detection.rst +++ b/docs/articles_en/documentation/legacy-features/transition-legacy-conversion-api/legacy-conversion-api/[legacy]-supported-model-formats/[legacy]-conversion-tutorials/convert-tensorflow-object-detection.rst @@ -89,6 +89,11 @@ Speech Recognition, Natural Language Processing and others. Refer to the links b * :doc:`OpenVINO Samples <../../../../../../learn-openvino/openvino-samples>` * :doc:`Open Model Zoo Demos <../../../../model-zoo>` +.. important:: + + Due to the deprecation of Open Model Zoo, models in the OpenVINO IR format are now + published on `Hugging Face `__. + Feeding Input Images to the Samples ################################### diff --git a/docs/articles_en/documentation/legacy-features/transition-legacy-conversion-api/legacy-conversion-api/[legacy]-supported-model-formats/[legacy]-conversion-tutorials/convert-tensorflow-yolo.rst b/docs/articles_en/documentation/legacy-features/transition-legacy-conversion-api/legacy-conversion-api/[legacy]-supported-model-formats/[legacy]-conversion-tutorials/convert-tensorflow-yolo.rst index 10946ce84100da..bf1668a89007c9 100644 --- a/docs/articles_en/documentation/legacy-features/transition-legacy-conversion-api/legacy-conversion-api/[legacy]-supported-model-formats/[legacy]-conversion-tutorials/convert-tensorflow-yolo.rst +++ b/docs/articles_en/documentation/legacy-features/transition-legacy-conversion-api/legacy-conversion-api/[legacy]-supported-model-formats/[legacy]-conversion-tutorials/convert-tensorflow-yolo.rst @@ -236,7 +236,7 @@ To convert DarkNet YOLOv1 and YOLOv2 models to the OpenVINO format, follow these Installing DarkFlow --------------------------------------------------------------- ++++++++++++++++++++++ You need DarkFlow to convert YOLOv1 and YOLOv2 models to TensorFlow. To install DarkFlow: @@ -259,7 +259,7 @@ You need DarkFlow to convert YOLOv1 and YOLOv2 models to TensorFlow. To install Converting a DarkNet YOLOv1 or YOLOv2 Model to TensorFlow --------------------------------------------------------------- ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ To convert YOLOv1 or YOLOv2 model to TensorFlow, go to the root directory of the cloned DarkFlow repository, place the previously downloaded \*.cfg and \*.weights files in the current directory and run the following command: @@ -292,7 +292,7 @@ in ``built_graph`` subdirectory of the cloned DarkFlow repository. File ``.pb`` is a TensorFlow representation of the YOLO model. Converting a TensorFlow YOLOv1 or YOLOv2 Model to the IR ---------------------------------------------------------- +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Converted TensorFlow YOLO model is missing ``Region`` layer and its parameters. Original YOLO ``Region`` layer parameters are stored in the configuration ``/.cfg`` file under the ``[region]`` title. diff --git a/docs/articles_en/documentation/legacy-features/transition-legacy-conversion-api/legacy-model-optimizer-extensibility/[legacy]-graph-traversal-and-modification.rst b/docs/articles_en/documentation/legacy-features/transition-legacy-conversion-api/legacy-model-optimizer-extensibility/[legacy]-graph-traversal-and-modification.rst index b2be35f4452832..f742b4531f93d0 100644 --- a/docs/articles_en/documentation/legacy-features/transition-legacy-conversion-api/legacy-model-optimizer-extensibility/[legacy]-graph-traversal-and-modification.rst +++ b/docs/articles_en/documentation/legacy-features/transition-legacy-conversion-api/legacy-model-optimizer-extensibility/[legacy]-graph-traversal-and-modification.rst @@ -71,7 +71,7 @@ the ``mo.graph.port.Port`` class). The ``Port`` object has several attributes: * ``node`` - the instance of the ``Node`` object the port belongs to. * ``idx`` - the port number. Input and output ports are numbered independently, starting from ``0``. Thus, -:doc:`ReLU <../../../openvino-ir-format/operation-sets/operation-specs/activation/relu-1>` operation has one input port (with index ``0``) and one output port (with index ``0``). + :doc:`ReLU <../../../openvino-ir-format/operation-sets/operation-specs/activation/relu-1>` operation has one input port (with index ``0``) and one output port (with index ``0``). * ``type`` - the type of the port. Could be equal to either ``"in"`` or ``"out"``. * ``data`` - the object that should be used to get attributes of the corresponding data node. This object has methods ``get_shape()`` / ``set_shape()`` and ``get_value()`` / ``set_value()`` to get/set shape/value of the corresponding data node. For example, ``in_port.data.get_shape()`` returns an input shape of a tensor connected to input port ``in_port`` (``in_port.type == 'in'``), ``out_port.data.get_value()`` returns a value of a tensor produced from output port ``out_port`` (``out_port.type == 'out'``). diff --git a/docs/articles_en/documentation/openvino-ecosystem/datumaro.rst b/docs/articles_en/documentation/openvino-ecosystem/datumaro.rst index c6b257ae3f17ca..e582b9359d227c 100644 --- a/docs/articles_en/documentation/openvino-ecosystem/datumaro.rst +++ b/docs/articles_en/documentation/openvino-ecosystem/datumaro.rst @@ -21,8 +21,6 @@ Plus, enjoy `Jupyter notebooks /cldnn_global_custom_kernels/cldnn_global_custom_kernels.xml`` file. -* Call the ``ov::Core::set_property()`` method from your application with the ``"CONFIG_FILE"`` key and the configuration file name as a value before loading the network that uses custom operations to the plugin: +* Include a section with your kernels into the automatically-loaded + ``/cldnn_global_custom_kernels/cldnn_global_custom_kernels.xml`` file. +* Call the ``ov::Core::set_property()`` method from your application with the + ``"CONFIG_FILE"`` key and the configuration file name as a value before loading + the network that uses custom operations to the plugin: .. tab-set:: @@ -35,14 +42,21 @@ There are two options for using the custom operation configuration file: :fragment: [part0] -All OpenVINO samples, except the trivial ``hello_classification``, and most Open Model Zoo demos -feature a dedicated command-line option ``-c`` to load custom kernels. For example, to load custom operations for the classification sample, run the command below: +All OpenVINO samples, except the trivial ``hello_classification``, and most Open +Model Zoo demos feature a dedicated command-line option ``-c`` to load custom kernels. +For example, to load custom operations for the classification sample, run the command below: .. code-block:: cpp $ ./classification_sample -m /bvlc_alexnet_fp16.xml -i ./validation_set/daily/227x227/apron.bmp -d GPU -c /custom_layer_example.xml +.. important:: + + Due to the deprecation of Open Model Zoo, models in the OpenVINO IR format are now + published on `Hugging Face `__. + + .. _config-file-format: Configuration File Format @@ -80,7 +94,8 @@ The ``CustomLayer`` node contains the entire configuration for a single custom o - Description * - ``name`` - (1) - - The name of the operation type to be used. This name should be identical to the type used in the IR. + - The name of the operation type to be used. This name should be identical to + the type used in the IR. * - ``type`` - (1) - Must be ``SimpleGPU`` . @@ -111,7 +126,8 @@ The ``Source`` node points to a single OpenCL source file. - Description * - ``filename`` - (1) - - Name of the file containing OpenCL source code. The path is relative to your executable. Multiple source nodes will have their sources concatenated in order. + - Name of the file containing OpenCL source code. The path is relative to your + executable. Multiple source nodes will have their sources concatenated in order. **Sub-nodes**: None @@ -129,16 +145,19 @@ the sources during compilation (JIT). - Description * - ``name`` - (1) - - The name of the defined JIT. For static constants, this can include the value as well, which is taken as a string. + - The name of the defined JIT. For static constants, this can include the value + as well, which is taken as a string. * - ``param`` - (0/1) - This parameter value is used as the value of this JIT definition. * - ``type`` - (0/1) - - The parameter type. Accepted values: ``int`` , ``float`` , and ``int[]`` , ``float[]`` for arrays. + - The parameter type. Accepted values: ``int`` , ``float`` , and ``int[]`` , + ``float[]`` for arrays. * - ``default`` - (0/1) - - The default value to be used if the specified parameters are missing from the operation in the OpenVINO IR. + - The default value to be used if the specified parameters are missing from the + operation in the OpenVINO IR. **Sub-nodes:** None @@ -197,7 +216,8 @@ The ``Tensor`` node configures a single input or output tensor. - 0-based index in the operation input/output ports in the OpenVINO IR * - ``format`` - (0/1) - - Data layout declaration for the tensor. Accepted values: ``BFYX`` , ``BYXF`` , ``YXFB`` , ``FYXB`` , and same values in all lowercase. Default value: ``BFYX``. + - Data layout declaration for the tensor. Accepted values: ``BFYX`` , ``BYXF`` , + ``YXFB`` , ``FYXB`` , and same values in all lowercase. Default value: ``BFYX``. CompilerOptions Node and Sub-Node Structure +++++++++++++++++++++++++++++++++++++++++++ @@ -231,10 +251,14 @@ queuing an OpenCL program for execution. - Description * - ``global`` ``local`` - (0/1) (0/1) - - An array of up to three integers or formulas for defining OpenCL work-sizes to be used during execution. The formulas can use the values of the B,F,Y,X dimensions and contain the operators: +,-,/,\*,%. All operators are evaluated in integer arithmetic. Default value: ``global=”B\*F\*Y\*X” local=””`` + - An array of up to three integers or formulas for defining OpenCL work-sizes to + be used during execution. The formulas can use the values of the B,F,Y,X + dimensions and contain the operators: +,-,/,\*,%. All operators are evaluated + in integer arithmetic. Default value: ``global=”B\*F\*Y\*X” local=””`` * - ``dim`` - (0/1) - - A tensor to take the work-size from. Accepted values: ``input N`` , ``output`` , where ``N`` is an index of input tensor starting with 0. Default value: ``output`` + - A tensor to take the work-size from. Accepted values: ``input N`` , ``output`` , + where ``N`` is an index of input tensor starting with 0. Default value: ``output`` **Sub-nodes**: None @@ -242,7 +266,8 @@ Example Configuration File ########################## The following code sample provides an example configuration file in XML -format. For information on the configuration file structure, see the `Configuration File Format <#config-file-format>`__. +format. For information on the configuration file structure, see the +`Configuration File Format <#config-file-format>`__. .. code-block:: xml :force: @@ -291,21 +316,27 @@ For an example, see `Example Kernel <#example-kernel>`__. * - ``_TYPE`` - The datatype of the tensor: ``float`` , ``half`` , or ``char`` * - ``_FORMAT_`` - - The format of the tensor, BFYX, BYXF, YXFB , FYXB, or ANY. The format is concatenated to the defined name. You can use the tensor format to define codepaths in your code with ``#ifdef/#endif`` . + - The format of the tensor, BFYX, BYXF, YXFB , FYXB, or ANY. The format is + concatenated to the defined name. You can use the tensor format to define + codepaths in your code with ``#ifdef/#endif`` . * - ``_LOWER_PADDING`` - - An array of padding elements used for the tensor dimensions before they start. Always ordered as BFYX. + - An array of padding elements used for the tensor dimensions before they start. + Always ordered as BFYX. * - ``_LOWER_PADDING_SIZE`` - The size of the ``_LOWER_PADDING`` array * - ``_UPPER_PADDING`` - - An array of padding elements used for the tensor dimensions after they end. Always ordered as BFYX. + - An array of padding elements used for the tensor dimensions after they end. + Always ordered as BFYX. * - ``_UPPER_PADDING_SIZE`` - The size of the ``_UPPER_PADDING`` array * - ``_PITCHES`` - - The offset (in elements) between adjacent elements in each dimension. Always ordered as BFYX. + - The offset (in elements) between adjacent elements in each dimension. + Always ordered as BFYX. * - ``_PITCHES_SIZE`` - The size of the ``_PITCHES`` array * - ``_OFFSET`` - - The number of elements from the start of the tensor to the first valid element, bypassing the lower padding. + - The number of elements from the start of the tensor to the first valid element, + bypassing the lower padding. All ```` values are automatically defined for every tensor bound to this operation, such as ``INPUT0``, ``INPUT1``, and ``OUTPUT0``, as shown @@ -345,7 +376,9 @@ Example Kernel .. _debugging-tips: .. note:: - As described in the previous section, all items such as the ``INPUT0_TYPE`` are actually defined as OpenCL (pre-)compiler inputs by OpenVINO for efficiency reasons. See the `Debugging Tips <#debugging-tips>`__ below for information on debugging the results. + As described in the previous section, all items such as the ``INPUT0_TYPE`` are + actually defined as OpenCL (pre-)compiler inputs by OpenVINO for efficiency reasons. + See the `Debugging Tips <#debugging-tips>`__ below for information on debugging the results. Debugging Tips ############## @@ -359,5 +392,6 @@ your output can be truncated to fit the buffer. Also, because of buffering, you actually get an entire buffer of output when the execution ends. -For more information, refer to the `printf Function `__. +For more information, refer to the +`printf Function `__. diff --git a/docs/articles_en/documentation/openvino-extensibility/custom-openvino-operations.rst b/docs/articles_en/documentation/openvino-extensibility/custom-openvino-operations.rst index aafcfe23538281..cb9035a93a7033 100644 --- a/docs/articles_en/documentation/openvino-extensibility/custom-openvino-operations.rst +++ b/docs/articles_en/documentation/openvino-extensibility/custom-openvino-operations.rst @@ -16,7 +16,7 @@ Operation Class To add your custom operation, create a new class that extends ``ov::Op``, which is in turn derived from ``ov::Node``, the base class for all graph operations in OpenVINO™. To add ``ov::Op``, include the next file: -.. doxygensnippet:: ./src/core/template_extension/new/identity.hpp +.. doxygensnippet:: src/core/template_extension/identity.hpp :language: cpp :fragment: [op:common_include] @@ -45,7 +45,7 @@ OpenVINO™ operation contains two constructors: * Default constructor, which enables you to create an operation without attributes * Constructor that creates and validates an operation with specified inputs and attributes -.. doxygensnippet:: ./src/core/template_extension/new/identity.cpp +.. doxygensnippet:: src/core/template_extension/identity.cpp :language: cpp :fragment: [op:ctor] @@ -54,7 +54,7 @@ OpenVINO™ operation contains two constructors: ``ov::Node::validate_and_infer_types`` method validates operation attributes and calculates output shapes using attributes of the operation. -.. doxygensnippet:: ./src/core/template_extension/new/identity.cpp +.. doxygensnippet:: src/core/template_extension/identity.cpp :language: cpp :fragment: [op:validate] @@ -63,7 +63,7 @@ OpenVINO™ operation contains two constructors: ``ov::Node::clone_with_new_inputs`` method creates a copy of the operation with new inputs. -.. doxygensnippet:: ./src/core/template_extension/new/identity.cpp +.. doxygensnippet:: src/core/template_extension/identity.cpp :language: cpp :fragment: [op:copy] @@ -72,7 +72,7 @@ OpenVINO™ operation contains two constructors: ``ov::Node::visit_attributes`` method enables you to visit all operation attributes. -.. doxygensnippet:: ./src/core/template_extension/new/identity.cpp +.. doxygensnippet:: src/core/template_extension/identity.cpp :language: cpp :fragment: [op:visit_attributes] @@ -81,7 +81,7 @@ OpenVINO™ operation contains two constructors: ``ov::Node::evaluate`` method enables you to apply constant folding to an operation. -.. doxygensnippet:: ./src/core/template_extension/new/identity.cpp +.. doxygensnippet:: src/core/template_extension/identity.cpp :language: cpp :fragment: [op:evaluate] diff --git a/docs/articles_en/documentation/openvino-extensibility/openvino-plugin-library/advanced-guides/low-precision-transformations/step1-prerequisites.rst b/docs/articles_en/documentation/openvino-extensibility/openvino-plugin-library/advanced-guides/low-precision-transformations/step1-prerequisites.rst index 135dc2516c0276..5c36e8b509ebd6 100644 --- a/docs/articles_en/documentation/openvino-extensibility/openvino-plugin-library/advanced-guides/low-precision-transformations/step1-prerequisites.rst +++ b/docs/articles_en/documentation/openvino-extensibility/openvino-plugin-library/advanced-guides/low-precision-transformations/step1-prerequisites.rst @@ -15,10 +15,12 @@ Step 1. Prerequisites Transformations PullReshapeThroughDequantization PullTransposeThroughDequantization LinOpSequenceFusion + ConvertSubtractConstant Prerequisites transformations are optional. The transformations prepare a model before running other low precision transformations. The transformations do not operate with dequantization operations or update precisions. Prerequisites transformations include: * :doc:`PullReshapeThroughDequantization ` * :doc:`PullTransposeThroughDequantization ` * :doc:`LinOpSequenceFusion ` +* :doc:`ConvertSubtractConstant ` diff --git a/docs/articles_en/documentation/openvino-extensibility/openvino-plugin-library/advanced-guides/low-precision-transformations/step2-markup.rst b/docs/articles_en/documentation/openvino-extensibility/openvino-plugin-library/advanced-guides/low-precision-transformations/step2-markup.rst index 073ccc1c3d0da4..cfdd53d348a259 100644 --- a/docs/articles_en/documentation/openvino-extensibility/openvino-plugin-library/advanced-guides/low-precision-transformations/step2-markup.rst +++ b/docs/articles_en/documentation/openvino-extensibility/openvino-plugin-library/advanced-guides/low-precision-transformations/step2-markup.rst @@ -26,6 +26,7 @@ Step 2. Markup Transformations CreatePrecisionsDependentAttribute PropagateThroughPrecisionPreserved PropagateToInput + PropagateSharedValue UpdateSharedPrecisionPreserved This step defines the optimal ``FakeQuantize`` decomposition precisions for the best inference performance via operations markup with runtime attribute instances. Attributes are created for input and output ports and operations. Transformations do not change the operation output port precisions. A model markup low precision logic is decomposed and implemented into the following common markup transformations. The order of transformations is important: @@ -36,8 +37,9 @@ This step defines the optimal ``FakeQuantize`` decomposition precisions for the 4. :doc:`MarkupPerTensorQuantization ` 5. :doc:`MarkupAvgPoolPrecisionPreserved ` 6. :doc:`PropagatePrecisions ` -7. :doc:`AlignQuantizationIntervals ` -8. :doc:`AlignQuantizationParameters ` +7. :doc:`PropagateSharedValue ` +8. :doc:`AlignQuantizationIntervals ` +9. :doc:`AlignQuantizationParameters ` .. list-table:: :header-rows: 1 diff --git a/docs/articles_en/documentation/openvino-extensibility/openvino-plugin-library/advanced-guides/low-precision-transformations/step3-main.rst b/docs/articles_en/documentation/openvino-extensibility/openvino-plugin-library/advanced-guides/low-precision-transformations/step3-main.rst index bd5c449877ad87..121d19001f2a51 100644 --- a/docs/articles_en/documentation/openvino-extensibility/openvino-plugin-library/advanced-guides/low-precision-transformations/step3-main.rst +++ b/docs/articles_en/documentation/openvino-extensibility/openvino-plugin-library/advanced-guides/low-precision-transformations/step3-main.rst @@ -20,7 +20,7 @@ Step 3. Main Transformations ConcatTransformation ConvolutionTransformation ConvolutionBackpropDataTransformation - DepthToSpaceTransformation + DepthToSpaceTransformation FakeQuantizeDecompositionTransformation FakeQuantizeTransformation InterpolateTransformation @@ -29,6 +29,7 @@ Step 3. Main Transformations MatMulTransformation MaxPoolTransformation MultiplyPartialTransformation + MultiplyTransformation MVNTransformation NormalizeL2Transformation PadTransformation @@ -44,6 +45,7 @@ Step 3. Main Transformations ShuffleChannelsTransformation SplitTransformation StridedSliceTransformation + SubtractTransformation TransposeTransformation UnsqueezeTransformation VariadicSplitTransformation @@ -67,6 +69,7 @@ Main transformations are the majority of low precision transformations. Transfor * :doc:`MatMulTransformation ` * :doc:`MaxPoolTransformation ` * :doc:`MultiplyPartialTransformation ` +* :doc:`MultiplyTransformation ` * :doc:`MVNTransformation ` * :doc:`NormalizeL2Transformation ` * :doc:`PadTransformation ` @@ -82,6 +85,7 @@ Main transformations are the majority of low precision transformations. Transfor * :doc:`ShuffleChannelsTransformation ` * :doc:`SplitTransformation ` * :doc:`StridedSliceTransformation ` +* :doc:`SubtractTransformation ` * :doc:`TransposeTransformation ` * :doc:`UnsqueezeTransformation ` * :doc:`VariadicSplitTransformation ` diff --git a/docs/articles_en/documentation/openvino-extensibility/openvino-plugin-library/plugin-testing.rst b/docs/articles_en/documentation/openvino-extensibility/openvino-plugin-library/plugin-testing.rst index 3705f0e4a285d1..ff3bdf878d1b13 100644 --- a/docs/articles_en/documentation/openvino-extensibility/openvino-plugin-library/plugin-testing.rst +++ b/docs/articles_en/documentation/openvino-extensibility/openvino-plugin-library/plugin-testing.rst @@ -5,7 +5,7 @@ Plugin Testing .. meta:: - :description: Use the openvino::funcSharedTests library, which includes + :description: Use the openvino::funcSharedTests library, which includes a predefined set of functional tests and utilities to verify a plugin. @@ -22,13 +22,13 @@ Test definitions are split into tests class declaration (see ``src/tests/functio 2. **Single layer tests** (``single_layer_tests`` sub-folder). This groups of tests checks that a particular single layer can be inferenced on a device. An example of test instantiation based on test definition from ``openvino::funcSharedTests`` library: * From the declaration of convolution test class we can see that it's a parametrized GoogleTest based class with the ``convLayerTestParamsSet`` tuple of parameters: - -.. doxygensnippet:: src/tests/functional/shared_test_classes/include/shared_test_classes/single_layer/convolution.hpp + +.. doxygensnippet:: src/tests/functional/shared_test_classes/include/shared_test_classes/single_op/convolution.hpp :language: cpp :fragment: [test_convolution:definition] * Based on that, define a set of parameters for ``Template`` plugin functional test instantiation: - + .. doxygensnippet:: src/plugins/template/tests/functional/shared_tests_instances/single_layer_tests/convolution.cpp :language: cpp :fragment: [test_convolution:declare_parameters] @@ -41,7 +41,7 @@ Test definitions are split into tests class declaration (see ``src/tests/functio 3. **Sub-graph tests** (``subgraph_tests`` sub-folder). This group of tests is designed to tests small patterns or combination of layers. E.g. when a particular topology is being enabled in a plugin e.g. TF ResNet-50, there is no need to add the whole topology to test tests. In opposite way, a particular repetitive subgraph or pattern can be extracted from ``ResNet-50`` and added to the tests. The instantiation of the sub-graph tests is done in the same way as for single layer tests. -.. note:: +.. note:: Such sub-graphs or patterns for sub-graph tests should be added to ``openvino::ov_models`` library first (this library is a pre-defined set of small ``ov::Model``) and re-used in sub-graph tests after. @@ -55,7 +55,7 @@ Test definitions are split into tests class declaration (see ``src/tests/functio To use these tests for your own plugin development, link the ``openvino::funcSharedTests`` library to your test binary and instantiate required test cases with desired parameters values. .. note:: - + A plugin may contain its own tests for use cases that are specific to hardware or need to be extensively tested. To build test binaries together with other build artifacts, use the ``make all`` command. For details, see :doc:`Build Plugin Using CMake `. @@ -67,7 +67,7 @@ OpenVINO Plugin tests are open for contribution. Add common test case definitions applicable for all plugins to the ``openvino::funcSharedTests`` target within the OpenVINO repository. Then, any other plugin supporting corresponding functionality can instantiate the new test. .. note:: - + When implementing a new subgraph test, add new single-layer tests for each operation of the subgraph if such test does not exist. diff --git a/docs/articles_en/documentation/openvino-extensibility/transformation-api.rst b/docs/articles_en/documentation/openvino-extensibility/transformation-api.rst index 5e28a22e69ab98..c179e628f6d0db 100644 --- a/docs/articles_en/documentation/openvino-extensibility/transformation-api.rst +++ b/docs/articles_en/documentation/openvino-extensibility/transformation-api.rst @@ -16,25 +16,26 @@ Overview of Transformations API transformation-api/model-pass transformation-api/matcher-pass transformation-api/graph-rewrite-pass + transformation-api/patterns-python-api -OpenVINO Transformation mechanism allows to develop transformation passes to modify ``ov::Model``. You can use this mechanism to apply additional optimizations to the original Model or transform unsupported subgraphs and operations to new operations which are supported by the plugin. -This guide contains all necessary information that you need to start implementing OpenVINO™ transformations. +OpenVINO Transformation mechanism allows to develop transformation passes to modify ``ov::Model``. You can use this mechanism to apply additional optimizations to the original Model or transform unsupported subgraphs and operations to new operations supported by the plugin. +This guide contains all the necessary information to start implementing OpenVINO™ transformations. Working with Model ################## -Before the moving to transformation part it is needed to say several words about functions which allow to modify ``ov::Model``. -This chapter extends the :doc:`model representation guide <../../openvino-workflow/running-inference/integrate-openvino-with-your-application/model-representation>` and shows an API that allows us to manipulate with ``ov::Model``. +Before moving to the transformation part, it is important to say a few words about the functions which allow modifying ``ov::Model``. +This section extends the :doc:`model representation guide <../../openvino-workflow/running-inference/integrate-openvino-with-your-application/model-representation>` and introduces an API for ``ov::Model`` manipulation. Working with node input and output ports ++++++++++++++++++++++++++++++++++++++++ -First of all let's talk about ``ov::Node`` input/output ports. Each OpenVINO™ operation has input and output ports except cases when operation has ``Parameter`` or ``Constant`` type. +Each OpenVINO operation has ``ov::Node`` input and output ports, except for ``Parameter`` and ``Constant`` types. +The terms ``node`` and ``operation`` are used interchangeably in OpenVINO, but this article will maintain consistency in their use. -Every port belongs to its node, so using a port we can access parent node, get shape and type for particular input/output, get all consumers in case of output port, and get producer node in case of input port. -With output port we can set inputs for newly created operations. +Every port is associated with a node, allowing access to the node it belongs to, including its shape, type, all consumers for output ports and the producer node for input ports. -Lets look at the code example. +Take a look at the code example: .. doxygensnippet:: docs/articles_en/assets/snippets/ov_model_snippets.cpp :language: cpp @@ -47,7 +48,7 @@ OpenVINO™ provides two ways for node replacement: via OpenVINO™ helper funct Let's start with OpenVINO™ helper functions. The most popular function is ``ov::replace_node(old_node, new_node)``. -We will review real replacement case where Negative operation is replaced with Multiply. +Let's review a replacement case where a Negative operation is replaced with Multiply. .. image:: ../../assets/images/ov_replace_node.png @@ -55,7 +56,7 @@ We will review real replacement case where Negative operation is replaced with M :language: cpp :fragment: [ov:replace_node] -``ov::replace_node`` has a constraint that number of output ports for both of ops must be the same; otherwise, it raises an exception. +``ov::replace_node`` has a constraint that number of output ports for both nodes must be the same. Otherwise, the attempt to replace the nodes will result in an exception. The alternative way to do the same replacement is the following: @@ -63,7 +64,7 @@ The alternative way to do the same replacement is the following: :language: cpp :fragment: [ov:manual_replace] -Another transformation example is insertion. +Another transformation example is insertion. Let's insert an additional Relu node. .. image:: ../../assets/images/ov_insert_node.png @@ -71,7 +72,7 @@ Another transformation example is insertion. :language: cpp :fragment: [ov:insert_node] -The alternative way to the insert operation is to make a node copy and use ``ov::replace_node()``: +The alternative way of inserting a node is to make a copy of the node and use ``ov::replace_node()``: .. doxygensnippet:: docs/articles_en/assets/snippets/ov_model_snippets.cpp :language: cpp @@ -80,15 +81,15 @@ The alternative way to the insert operation is to make a node copy and use ``ov: Node elimination ++++++++++++++++ -Another type of node replacement is its elimination. +Another type of node replacement is elimination of a node. -To eliminate operation, OpenVINO™ has special method that considers all limitations related to OpenVINO™ Runtime. +To eliminate a node, OpenVINO provides a method that considers all limitations of the OpenVINO Runtime. .. doxygensnippet:: docs/articles_en/assets/snippets/ov_model_snippets.cpp :language: cpp :fragment: [ov:eliminate_node] -``ov::replace_output_update_name()`` in case of successful replacement it automatically preserves friendly name and runtime info. +If the replacement is successful, ``ov::replace_output_update_name()`` automatically preserves the friendly name and runtime info. .. _transformations_types: @@ -99,7 +100,7 @@ OpenVINO™ Runtime has three main transformation types: * :doc:`Model pass ` - straightforward way to work with ``ov::Model`` directly * :doc:`Matcher pass ` - pattern-based transformation approach -* :doc:`Graph rewrite pass ` - container for matcher passes needed for efficient execution +* :doc:`Graph rewrite pass ` - container for matcher passes used for efficient execution .. image:: ../../assets/images/transformations_structure.png @@ -116,36 +117,36 @@ Transformation library has two internal macros to support conditional compilatio Transformation writing essentials ################################# -When developing a transformation, you need to follow these transformation rules: +To develop a transformation, follow these transformation rules: 1. Friendly Names +++++++++++++++++ -Each ``ov::Node`` has an unique name and a friendly name. In transformations we care only about friendly name because it represents the name from the model. -To avoid losing friendly name when replacing node with other node or subgraph, set the original friendly name to the latest node in replacing subgraph. See the example below. +Each ``ov::Node`` has a unique name and a friendly name. In transformations, only the friendly name matters because it represents the name from the model's perspective. +To prevent losing the friendly name when replacing a node with another node or a subgraph, the original friendly name is set to the last node in the replacing subgraph. See the example below. .. doxygensnippet:: docs/articles_en/assets/snippets/ov_model_snippets.cpp :language: cpp :fragment: [ov:replace_friendly_name] -In more advanced cases, when replaced operation has several outputs and we add additional consumers to its outputs, we make a decision how to set friendly name by arrangement. +In more complicated cases, when a replaced operation has several outputs and additional consumers are added to its outputs, the decision on how to set the friendly name is determined by an agreement. 2. Runtime Info +++++++++++++++ -Runtime info is a map ``std::map`` located inside ``ov::Node`` class. It represents additional attributes in ``ov::Node``. -These attributes can be set by users or by plugins and when executing transformation that changes ``ov::Model`` we need to preserve these attributes as they will not be automatically propagated. +Runtime info is a map ``std::map`` located inside the ``ov::Node`` class. It represents additional attributes of the ``ov::Node``. +These attributes, which can be set by users or plugins, need to be preserved when executing a transformation that changes ``ov::Model``, as they are not automatically propagated. In most cases, transformations have the following types: 1:1 (replace node with another node), 1:N (replace node with a sub-graph), N:1 (fuse sub-graph into a single node), N:M (any other transformation). -Currently, there is no mechanism that automatically detects transformation types, so we need to propagate this runtime information manually. See the examples below. +Currently, there is no mechanism that automatically detects transformation types, so this runtime information needs to be propagated manually. See the example below: .. doxygensnippet:: docs/articles_en/assets/snippets/ov_model_snippets.cpp :language: cpp :fragment: [ov:copy_runtime_info] -When transformation has multiple fusions or decompositions, ``ov::copy_runtime_info`` must be called multiple times for each case. +When a transformation has multiple fusions or decompositions, ``ov::copy_runtime_info`` must be called multiple times for each case. -.. note:: ``copy_runtime_info`` removes ``rt_info`` from destination nodes. If you want to keep it, you need to specify them in source nodes like this: ``copy_runtime_info({a, b, c}, {a, b})`` +.. note:: ``copy_runtime_info`` removes ``rt_info`` from destination nodes. If you want to keep it, specify them in source nodes as following: ``copy_runtime_info({a, b, c}, {a, b})`` 3. Constant Folding +++++++++++++++++++ @@ -172,21 +173,21 @@ Common mistakes in transformations In transformation development process: -* Do not use deprecated OpenVINO™ API. Deprecated methods has the ``OPENVINO_DEPRECATED`` macros in its definition. -* Do not pass ``shared_ptr`` as an input for other node if type of node is unknown or it has multiple outputs. Use explicit output port. -* If you replace node with another node that produces different shape, remember that new shape will not be propagated until the first ``validate_nodes_and_infer_types`` call for ``ov::Model``. If you are using ``ov::pass::Manager``, it will automatically call this method after each transformation execution. +* Do not use deprecated OpenVINO™ API. Deprecated methods are marked with ``OPENVINO_DEPRECATED`` macro in their definition. +* Do not pass ``shared_ptr`` as input for another node if the type of the node is unknown or if it has multiple outputs. Instead, use explicit output ports. +* If you replace a node with another node that produces different shape, note that the new shape will not be propagated until the first ``validate_nodes_and_infer_types`` call for ``ov::Model``. If you are using ``ov::pass::Manager``, it will automatically call this method after each transformation execution. * Do not forget to call the ``ov::pass::ConstantFolding`` pass if your transformation creates constant subgraphs. * Use latest OpSet if you are not developing downgrade transformation pass. -* When developing a callback for ``ov::pass::MatcherPass``, do not change nodes that come after the root node in topological order. +* When developing a callback for ``ov::pass::MatcherPass``, do not change nodes that come after the root node in the topological order. .. _using_pass_manager: Using pass manager ################## -``ov::pass::Manager`` is a container class that can store the list of transformations and execute them. The main idea of this class is to have high-level representation for grouped list of transformations. -It can register and apply any `transformation pass <#transformations_types>`__ on model. -In addition, ``ov::pass::Manager`` has extended debug capabilities (find more information in the `how to debug transformations <#how_to_debug_transformations>`__ section). +``ov::pass::Manager`` is a container class that can store a list of transformations and execute them. The main idea of this class is to have a high-level representation for grouped list of transformations. +It can register and apply any `transformation pass <#transformations_types>`__ on a model. +In addition, ``ov::pass::Manager`` has extended debug capabilities (find more information in the `how to debug transformations <#how-to-debug-transformations>`__ section). The example below shows basic usage of ``ov::pass::Manager`` diff --git a/docs/articles_en/documentation/openvino-extensibility/transformation-api/patterns-python-api.rst b/docs/articles_en/documentation/openvino-extensibility/transformation-api/patterns-python-api.rst new file mode 100644 index 00000000000000..bf113ad8ffb460 --- /dev/null +++ b/docs/articles_en/documentation/openvino-extensibility/transformation-api/patterns-python-api.rst @@ -0,0 +1,271 @@ + +Transformation Patterns with OpenVINO API +================================================== + +.. meta:: + :description: Learn how to apply additional model optimizations or transform + unsupported subgraphs and operations using OpenVINO™ Transformations API. + +Pattern matching is an essential component of OpenVINO™ transformations. Before performing any transformation on a subgraph of a graph, it is necessary to find that subgraph in the graph. +Patterns serve as a searching utility to identify nodes intended for transformations. This article covers the basics of pattern +creation using OpenVINO™ API and helpful utilities to facilitate working with them. While this guide focuses on creating patterns, if you want to learn more about ``MatcherPass``, refer to the :doc:`OpenVINO Matcher Pass article <./matcher-pass>`. Note that some examples may be intentionally simplified for ease of understanding. + +Before proceeding further, it is necessary to add some imports. These imports include the operations to be used and additional utilities described in this guide. +Add the following lines to your file: + +.. tab-set:: + + .. tab-item:: Python + :sync: py + + .. doxygensnippet:: docs/articles_en/assets/snippets/ov_patterns.py + :language: python + :fragment: [ov:imports] + + .. tab-item:: C++ + :sync: cpp + + .. doxygensnippet:: docs/articles_en/assets/snippets/ov_patterns.cpp + :language: cpp + :fragment: [ov:imports] + +Pattern Creation ++++++++++++++++++++++ + +A pattern is a simplified model comprised of nodes aimed to be matched. It lacks some features of a model and cannot function as one. + +Consider a straightforward pattern consisting of three nodes to be found in a given model. + +.. image:: ./../../../assets/images/simple_pattern_example.png + +Let's create the model and the pattern: + +.. tab-set:: + + .. tab-item:: Python + :sync: py + + .. doxygensnippet:: docs/articles_en/assets/snippets/ov_patterns.py + :language: python + :fragment: [ov:create_simple_model_and_pattern] + + .. tab-item:: C++ + :sync: cpp + + .. doxygensnippet:: docs/articles_en/assets/snippets/ov_patterns.cpp + :language: cpp + :fragment: [ov:create_simple_model_and_pattern] + +.. note:: This example uses testing utilities that directly compare given sequences of nodes. In reality, the process of finding a pattern within a model is more complicated. However, to focus only on patterns and their functionality, these details are intentionally omitted. + +Although the code is functional, in OpenVINO, patterns are typically not created using the same nodes as those used for creating the model. Instead, wrappers are preferred, providing additional functionality. +For the given case, ``WrapType`` is used and the code looks as following: + +.. tab-set:: + + .. tab-item:: Python + :sync: py + + .. doxygensnippet:: docs/articles_en/assets/snippets/ov_patterns.py + :language: python + :fragment: [ov:create_simple_model_and_pattern_wrap_type] + + .. tab-item:: C++ + :sync: cpp + + .. doxygensnippet:: docs/articles_en/assets/snippets/ov_patterns.cpp + :language: cpp + :fragment: [ov:create_simple_model_and_pattern_wrap_type] + +1. WrapType +++++++++++++++++++++++++++++++++++++++++ + +``WrapType`` is a wrapper used to store one or many types to match them. As demonstrated earlier, it is possible to specify a single type in ``WrapType`` and use it for matching. +However, you can also list all possible types for a given node, for example: + +.. tab-set:: + + .. tab-item:: Python + :sync: py + + .. doxygensnippet:: docs/articles_en/assets/snippets/ov_patterns.py + :language: python + :fragment: [ov:wrap_type_list] + + .. tab-item:: C++ + :sync: cpp + + .. doxygensnippet:: docs/articles_en/assets/snippets/ov_patterns.cpp + :language: cpp + :fragment: [ov:wrap_type_list] + +Note that ``pattern_sig`` is created with the list ``["opset13.Relu", "opset13.Sigmoid"]``, meaning it can be either a ``Relu`` or a ``Sigmoid``. +This feature enables matching the same pattern against different nodes. Essentially, ``WrapType`` can represent "one of listed" types. ``WrapType`` supports specifying more than two types. + +To add additional checks for your node, create a predicate by providing a function or a lambda. This function will be executed during matching, performing the additional validation specified in the logic of the function. For example, you might want to check the consumers count of a given node: + +.. tab-set:: + + .. tab-item:: Python + :sync: py + + .. doxygensnippet:: docs/articles_en/assets/snippets/ov_patterns.py + :language: python + :fragment: [ov:wrap_type_predicate] + + .. tab-item:: C++ + :sync: cpp + + .. doxygensnippet:: docs/articles_en/assets/snippets/ov_patterns.cpp + :language: cpp + :fragment: [ov:wrap_type_predicate] + +2. AnyInput +++++++++++++++++++++++++++++++++++++++++ +``AnyInput`` is used when there is no need to specify a particular input for a given node. + +.. tab-set:: + + .. tab-item:: Python + :sync: py + + .. doxygensnippet:: docs/articles_en/assets/snippets/ov_patterns.py + :language: python + :fragment: [ov:any_input] + + .. tab-item:: C++ + :sync: cpp + + .. doxygensnippet:: docs/articles_en/assets/snippets/ov_patterns.cpp + :language: cpp + :fragment: [ov:any_input] + +You can also create ``AnyInput()`` with a predicate, if you want additional checks for you input. It will look similar to ``WrapType`` with a lambda or a function. For example, to ensure that the input has a rank of 4: + +.. tab-set:: + + .. tab-item:: Python + :sync: py + + .. doxygensnippet:: docs/articles_en/assets/snippets/ov_patterns.py + :language: python + :fragment: [ov:any_input_predicate] + + .. tab-item:: C++ + :sync: cpp + + .. doxygensnippet:: docs/articles_en/assets/snippets/ov_patterns.cpp + :language: cpp + :fragment: [ov:any_input_predicate] + +3. Or +++++++++++++++++++++++++++++++++++++++++ +``Or`` functions similar to ``WrapType``, however, while ``WrapType`` can only match one of the types provided in the list, ``Or`` is used to match different branches of nodes. +Suppose the goal is to match the model against two different sequences of nodes. The ``Or`` type +facilitates this by creating two different branches (``Or`` supports more than two branches), looking as follows: + +.. image:: ./../../../assets/images/or_branches.png + +The red branch will not match, but it will work perfectly for the blue one. +Here is how it looks in code: + +.. tab-set:: + + .. tab-item:: Python + :sync: py + + .. doxygensnippet:: docs/articles_en/assets/snippets/ov_patterns.py + :language: python + :fragment: [ov:pattern_or] + + .. tab-item:: C++ + :sync: cpp + + .. doxygensnippet:: docs/articles_en/assets/snippets/ov_patterns.cpp + :language: cpp + :fragment: [ov:pattern_or] + +Note that matching will succeed for the first matching branch and the remaining ones will not be checked. + +4. Optional +++++++++++++++++++++++++++++++++++++++++ +``Optional`` is a bit tricky. It allows specifying whether a node might be present or absent in the model. Under the hood, +the pattern will create two branches using ``Or``: one with the optional node present and another one without it. Here is what it would look like with the ``Optional`` +unfolding into two branches: + +.. image:: ./../../../assets/images/optional.png + +The code for our model looks as follows: + +.. tab-set:: + + .. tab-item:: Python + :sync: py + + .. doxygensnippet:: docs/articles_en/assets/snippets/ov_patterns.py + :language: python + :fragment: [ov:pattern_optional_middle] + + .. tab-item:: C++ + :sync: cpp + + .. doxygensnippet:: docs/articles_en/assets/snippets/ov_patterns.cpp + :language: cpp + :fragment: [ov:pattern_optional_middle] + +The ``Optional`` does not necessarily have to be in the middle of the pattern. It can be a top node and a root node. + + +Top node: + +.. tab-set:: + + .. tab-item:: Python + :sync: py + + .. doxygensnippet:: docs/articles_en/assets/snippets/ov_patterns.py + :language: python + :fragment: [ov:pattern_optional_top] + + .. tab-item:: C++ + :sync: cpp + + .. doxygensnippet:: docs/articles_en/assets/snippets/ov_patterns.cpp + :language: cpp + :fragment: [ov:pattern_optional_top] + +Root node: + +.. tab-set:: + + .. tab-item:: Python + :sync: py + + .. doxygensnippet:: docs/articles_en/assets/snippets/ov_patterns.py + :language: python + :fragment: [ov:pattern_optional_root] + + .. tab-item:: C++ + :sync: cpp + + .. doxygensnippet:: docs/articles_en/assets/snippets/ov_patterns.cpp + :language: cpp + :fragment: [ov:pattern_optional_root] + +``Optional`` also supports adding a predicate the same way ``WrapType`` and ``AnyInput`` do: + +.. tab-set:: + + .. tab-item:: Python + :sync: py + + .. doxygensnippet:: docs/articles_en/assets/snippets/ov_patterns.py + :language: python + :fragment: [ov:optional_predicate] + + .. tab-item:: C++ + :sync: cpp + + .. doxygensnippet:: docs/articles_en/assets/snippets/ov_patterns.cpp + :language: cpp + :fragment: [ov:optional_predicate] \ No newline at end of file diff --git a/docs/articles_en/documentation/openvino-ir-format/intermediate-representation-int8-inference.rst b/docs/articles_en/documentation/openvino-ir-format/intermediate-representation-int8-inference.rst index a0cc4488ef15cc..4c9c3ab6f05c76 100644 --- a/docs/articles_en/documentation/openvino-ir-format/intermediate-representation-int8-inference.rst +++ b/docs/articles_en/documentation/openvino-ir-format/intermediate-representation-int8-inference.rst @@ -21,7 +21,7 @@ Such a model is called a Low Precision IR and can be generated in two ways: * By :doc:`quantizing regular IR with the Neural Network Compression Framework (NNCF) <../../openvino-workflow/model-optimization>` * Using model conversion of a model pre-trained for Low Precision inference: TensorFlow models (``.pb`` model file with ``FakeQuantize`` operations), quantized TensorFlow Lite models and ONNX quantized models. -TensorFlow and ONNX quantized models can be prepared by `Neural Network Compression Framework `__. + TensorFlow and ONNX quantized models can be prepared by `Neural Network Compression Framework `__. For an operation to be executed in INT8, it must have `FakeQuantize` operations as inputs. For more details, see the :doc:`specification of FakeQuantize operation `. diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/broadcast-rules.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/broadcast-rules.rst index bfa55a5a798ce6..c15b31b9051e20 100644 --- a/docs/articles_en/documentation/openvino-ir-format/operation-sets/broadcast-rules.rst +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/broadcast-rules.rst @@ -5,8 +5,8 @@ Broadcast Rules For Elementwise Operations .. meta:: - :description: Learn about Numpy and PDPD broadcast types in OpenVINO, that - enable performing element-wise operation for inputs of arbitrary + :description: Learn about Numpy and PDPD broadcast types in OpenVINO, that + enable performing element-wise operation for inputs of arbitrary number of dimensions. The purpose of this document is to provide a set of common rules which are applicable for ops using broadcasting. @@ -40,81 +40,82 @@ Rules Numpy examples ############## -* ``A: Shape(,) -> scalar`` - ``B: Shape(,) -> scalar`` +* ``A: Shape(,) -> scalar`` + ``B: Shape(,) -> scalar`` ``Result: Shape(,) -> scalar`` -* ``A: Shape(2, 3)`` - ``B: Shape( 1)`` +* ``A: Shape(2, 3)`` + ``B: Shape( 1)`` ``Result: Shape(2, 3)`` -* ``A: Shape( 3)`` - ``B: Shape(2, 3)`` +* ``A: Shape( 3)`` + ``B: Shape(2, 3)`` ``Result: Shape(2, 3)`` -* ``A: Shape(2, 3, 5)`` - ``B: Shape(,) -> scalar`` +* ``A: Shape(2, 3, 5)`` + ``B: Shape(,) -> scalar`` ``Result: Shape(2, 3, 5)`` -* ``A: Shape(2, 1, 5)`` +* ``A: Shape(2, 1, 5)`` ``B: Shape(1, 4, 5)`` ``Result: Shape(2, 4, 5)`` -* ``A: Shape( 6, 5)`` - ``B: Shape(2, 1, 5)`` +* ``A: Shape( 6, 5)`` + ``B: Shape(2, 1, 5)`` ``Result: Shape(2, 6, 5)`` -* ``A: Shape(2, 1, 5)`` - ``B: Shape( 4, 1)`` - ``Result: Shape(2, 4, 5)`` +* ``A: Shape(2, 1, 5)`` + ``B: Shape( 4, 1)`` + ``Result: Shape(2, 4, 5)`` -* ``A: Shape(3, 2, 1, 4)`` - ``B: Shape( 5, 4)`` +* ``A: Shape(3, 2, 1, 4)`` + ``B: Shape( 5, 4)`` ``Result: Shape(3, 2, 5, 4)`` -* ``A: Shape( 1, 5, 3)`` - ``B: Shape(5, 2, 1, 3)`` +* ``A: Shape( 1, 5, 3)`` + ``B: Shape(5, 2, 1, 3)`` ``Result: Shape(5, 2, 5, 3)`` -* ``A: Shape(3)`` - ``B: Shape(2)`` +* ``A: Shape(3)`` + ``B: Shape(2)`` ``Result: broadcast won't happen due to dimensions mismatch`` -* ``A: Shape(3, 1, 5)`` - ``B: Shape(4, 4, 5)`` +* ``A: Shape(3, 1, 5)`` + ``B: Shape(4, 4, 5)`` ``Result: broadcast won't happen due to dimensions mismatch on the leftmost axis`` PDPD examples ############# -* ``A: Shape(2, 3, 4, 5)`` - ``B: Shape( 3, 4 ) with axis = 1`` +* ``A: Shape(2, 3, 4, 5)`` + ``B: Shape( 3, 4 ) with axis = 1`` ``Result: Shape(2, 3, 4, 5)`` -* ``A: Shape(2, 3, 4, 5)`` - ``B: Shape( 3, 1 ) with axis = 1`` +* ``A: Shape(2, 3, 4, 5)`` + ``B: Shape( 3, 1 ) with axis = 1`` ``Result: Shape(2, 3, 4, 5)`` -* ``A: Shape(2, 3, 4, 5)`` - ``B: Shape( 4, 5) with axis=-1(default) or axis=2`` +* ``A: Shape(2, 3, 4, 5)`` + ``B: Shape( 4, 5) with axis=-1(default) or axis=2`` ``Result: Shape(2, 3, 4, 5)`` -* ``A: Shape(2, 3, 4, 5)`` - ``B: Shape(1, 3 ) with axis = 0`` +* ``A: Shape(2, 3, 4, 5)`` + ``B: Shape(1, 3 ) with axis = 0`` ``Result: Shape(2, 3, 4, 5)`` -* ``A: Shape(2, 3, 4, 5)`` - ``B: Shape(,)`` - ``Result: Shape(2, 3, 4, 5)`` +* ``A: Shape(2, 3, 4, 5)`` + ``B: Shape(,)`` + ``Result: Shape(2, 3, 4, 5)`` -* ``A: Shape(2, 3, 4, 5)`` +* ``A: Shape(2, 3, 4, 5)`` ``B: Shape( 5) with axis=-1(default) or axis = 3`` ``Result: Shape(2, 3, 4, 5)`` -* ``A: Shape(8, 1, 6, 1)`` - ``B: Shape( 7, 1, 5) with axis = 1`` +* ``A: Shape(8, 1, 6, 1)`` + ``B: Shape( 7, 1, 5) with axis = 1`` ``Result: broadcast won't happen due to dimensions mismatch, only B to A broadcast is supported for PDPD`` -Bidirectional Broadcast Rules + +Bidirectional Broadcast Rules ############################# Description @@ -138,23 +139,23 @@ Rules Bidirectional examples ++++++++++++++++++++++ -* ``A: Shape(5)`` - ``B: Shape(1)`` +* ``A: Shape(5)`` + ``B: Shape(1)`` ``Result: Shape(5)`` -* ``A: Shape(2, 3)`` - ``B: Shape( 3)`` +* ``A: Shape(2, 3)`` + ``B: Shape( 3)`` ``Result: Shape(2, 3)`` -* ``A: Shape(3, 1)`` - ``B: Shape(3, 4)`` +* ``A: Shape(3, 1)`` + ``B: Shape(3, 4)`` ``Result: Shape(3, 4)`` -* ``A: Shape(3, 4)`` - ``B: Shape(,) -> scalar`` +* ``A: Shape(3, 4)`` + ``B: Shape(,) -> scalar`` ``Result: Shape(3, 4)`` -* ``A: Shape( 3, 1)`` - ``B: Shape(2, 1, 6)`` +* ``A: Shape( 3, 1)`` + ``B: Shape(2, 1, 6)`` ``Result: Shape(2, 3, 6)`` diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs.rst index 250ef955bb41a8..8f37b9dbdf8e9f 100644 --- a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs.rst +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs.rst @@ -22,6 +22,8 @@ Operation Specifications Assign-6 Atan-1 Atanh-3 + AUGRUCell + AUGRUSequence AvgPool-1 AvgPool-14 BatchNormInference-1 @@ -107,7 +109,7 @@ Operation Specifications Interpolate-1 Interpolate-4 Interpolate-11 - Inverse-14 + Inverse-14 IsFinite-10 IsInf-10 IsNaN-10 @@ -130,6 +132,7 @@ Operation Specifications MaxPool-1 MaxPool-8 MaxPool-14 + Shape Calculation Rules for Pooling Operators Maximum-1 Minimum-1 Mish-4 diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/arithmetic/tanh-1.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/arithmetic/tanh-1.rst index 57a4e6f28091e8..bd2184251d0a26 100644 --- a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/arithmetic/tanh-1.rst +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/arithmetic/tanh-1.rst @@ -5,7 +5,7 @@ Tanh .. meta:: - :description: Learn about Tanh-1 - an element-wise, arithmetic operation, which + :description: Learn about Tanh-1 - an element-wise, arithmetic operation, which can be performed on a single tensor in OpenVINO. **Versioned name**: *Tanh-1* @@ -34,7 +34,7 @@ For each element from the input tensor calculates corresponding element in the o **Outputs**: * **1**: The result of element-wise *Tanh* operation. A tensor of type *T* and the same shape as input tensor. -**Types** + **Types** * *T*: any numeric type. @@ -44,7 +44,7 @@ For each element from the input tensor calculates corresponding element in the o *Example 1* .. code-block:: xml - :force: + :force: diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/condition/if-8.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/condition/if-8.rst index 7dd1a2d1659a0a..f53693792133f8 100644 --- a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/condition/if-8.rst +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/condition/if-8.rst @@ -5,46 +5,46 @@ If .. meta:: - :description: Learn about If-8 - an element-wise, condition operation, which + :description: Learn about If-8 - an element-wise, condition operation, which can be performed on multiple tensors in OpenVINO. **Versioned name**: *If-8* **Category**: *Condition* -**Short description**: *If* operation contains two internal networks(subgraphs) such as ``then_body`` and ``else_body``, -and performs one of them depending on ``cond`` value. If ``cond`` is ``True``, ``then_body`` is executed. If ``cond`` is ``False``, -the operation executes the ``else_body`` subgraph. +**Short description**: *If* operation contains two internal networks(subgraphs) such as ``then_body`` and ``else_body``, +and performs one of them depending on ``cond`` value. If ``cond`` is ``True``, ``then_body`` is executed. If ``cond`` is ``False``, +the operation executes the ``else_body`` subgraph. **Detailed description** -*If* must not contain empty subgraphs. Each of them must have at least one operation ``Result``. +*If* must not contain empty subgraphs. Each of them must have at least one operation ``Result``. Also the number of outputs from *If* always must be greater than zero and equal to the number of outputs from each subgraph. **If attributes**: * **Subgraphs**: - ``then_body``/``else_body`` are subgraphs that are executed depending on the ``cond`` value. - The subgraph is described operation by operation as a typical IR network. + ``then_body``/``else_body`` are subgraphs that are executed depending on the ``cond`` value. + The subgraph is described operation by operation as a typical IR network. The subgraph has inputs (``Parameter`` operations) and outputs (``Result`` operations). - * **Subgraph's inputs** - inputs to the subgraph which associated with *If* inputs via *port_map*. + * **Subgraph's inputs** - inputs to the subgraph which associated with *If* inputs via *port_map*. The subgraph can have any number of inputs (even zero). * **Subgraph's outputs** - outputs from the subgraph which associated with *If* outputs via *port_map*. - The subgraph must contain at least one output. Each *If* output is associated with one output from the subgraph. - Therefore the number of ``then_body`` outputs is equal to the number of outputs from *If* and + The subgraph must contain at least one output. Each *If* output is associated with one output from the subgraph. + Therefore the number of ``then_body`` outputs is equal to the number of outputs from *If* and the number of ``else_body`` outputs. The type of the subgraph output and the type of the associated output from *If* must be equal. * **Port maps**: - *port_map* is a set of rules to map input or output data tensors of *If* operation onto the subgraph data tensors. - The ``port_map`` entries can be ``input`` and ``output``. Each entry describes a corresponding mapping rule. + *port_map* is a set of rules to map input or output data tensors of *If* operation onto the subgraph data tensors. + The ``port_map`` entries can be ``input`` and ``output``. Each entry describes a corresponding mapping rule. *If* has two *port_maps*: ``then_port_map`` for ``then_body`` and ``else_port_map`` for ``else_body``. * **Port map attributes**: @@ -59,18 +59,18 @@ Also the number of outputs from *If* always must be greater than zero and equal * *internal_layer_id* - * **Description**: *internal_layer_id* is a ``Parameter`` or ``Result`` operation ID inside - the subgraph to map to. + * **Description**: *internal_layer_id* is a ``Parameter`` or ``Result`` operation ID inside + the subgraph to map to. * **Range of values**: IDs of the ``Parameter`` or ``Result`` operations in the subgraph * **Type**: ``unsigned int`` * **Default value**: None - * **Required**: *yes* + * **Required**: *yes* **If Inputs** -* **cond**: A scalar or 1D tensor with 1 element of ``boolean`` type specifying which subgraph to execute. -``True`` value means to execute the ``then_body``, ``False`` - ``else_body``. *Required*. +* **cond**: A scalar or 1D tensor with 1 element of ``boolean`` type specifying which subgraph to execute. + ``True`` value means to execute the ``then_body``, ``False`` - ``else_body``. *Required*. * **Multiple other inputs**: Tensors of different types and shapes. *Optional*. diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/detection/experimental-detectron-detection-output-6.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/detection/experimental-detectron-detection-output-6.rst index 4f494d910fa014..3e9ba293596d2a 100644 --- a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/detection/experimental-detectron-detection-output-6.rst +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/detection/experimental-detectron-detection-output-6.rst @@ -5,8 +5,8 @@ ExperimentalDetectronDetectionOutput .. meta:: - :description: Learn about ExperimentalDetectronDetectionOutput-6 - an object - detection operation, which can be performed on four required + :description: Learn about ExperimentalDetectronDetectionOutput-6 - an object + detection operation, which can be performed on four required input tensors in OpenVINO. **Versioned name**: *ExperimentalDetectronDetectionOutput-6* @@ -42,7 +42,7 @@ refined boxes according to the formulas: ``ctr_y = y0 + 0.5f * box_h`` * ``dx``, ``dy``, ``d_log_w`` and ``d_log_h`` are deltas calculated according to the formulas below, and ``deltas_tensor`` is a -second input: + second input: ``dx = deltas_tensor[roi_idx, 4 * class_idx + 0] / deltas_weights[0]`` @@ -104,7 +104,7 @@ second input: * **Description**: *class_agnostic_box_regression* attribute is a flag that specifies whether to delete background classes or not. * **Range of values**: - + * ``true`` means background classes should be deleted * ``false`` means background classes should not be deleted * **Type**: ``boolean`` @@ -149,7 +149,7 @@ second input: .. code-block:: xml :force: - + diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/detection/experimental-detectron-generate-proposals-single-image-6.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/detection/experimental-detectron-generate-proposals-single-image-6.rst index d3850dbb4296ec..4a3b947603c15c 100644 --- a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/detection/experimental-detectron-generate-proposals-single-image-6.rst +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/detection/experimental-detectron-generate-proposals-single-image-6.rst @@ -5,8 +5,8 @@ ExperimentalDetectronGenerateProposalsSingleImage .. meta:: - :description: Learn about ExperimentalDetectronGenerateProposalsSingleImage-6 - - an object detection operation, which can be performed on four + :description: Learn about ExperimentalDetectronGenerateProposalsSingleImage-6 - + an object detection operation, which can be performed on four required input tensors. **Versioned name**: *ExperimentalDetectronGenerateProposalsSingleImage-6* @@ -64,7 +64,7 @@ ExperimentalDetectronGenerateProposalsSingleImage * **1**: A 1D tensor of type *T* with 3 elements ``[image_height, image_width, scale_height_and_width]`` providing input image size info. **Required.** * **2**: A 2D tensor of type *T* with shape ``[height * width * number_of_channels, 4]`` providing anchors. **Required.** * **3**: A 3D tensor of type *T* with shape ``[number_of_channels * 4, height, width]`` providing deltas for anchors. -Height and width for third and fourth inputs should be equal. **Required.** + Height and width for third and fourth inputs should be equal. **Required.** * **4**: A 3D tensor of type *T* with shape ``[number_of_channels, height, width]`` providing proposals scores. **Required.** **Outputs** @@ -80,7 +80,7 @@ Height and width for third and fourth inputs should be equal. **Required.** .. code-block:: xml :force: - + diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/detection/psroi-pooling-1.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/detection/psroi-pooling-1.rst index d3ac83442e3c2d..c5de7a037bbbe9 100644 --- a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/detection/psroi-pooling-1.rst +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/detection/psroi-pooling-1.rst @@ -5,7 +5,7 @@ PSROIPooling .. meta:: - :description: Learn about PSROIPooling-1 - an object detection operation, + :description: Learn about PSROIPooling-1 - an object detection operation, which can be performed on two required input tensors. **Versioned name**: *PSROIPooling-1* @@ -73,10 +73,10 @@ ROIs coordinates are specified in absolute values for the average mode and in no **Inputs**: -* **1**: 4D input tensor with shape ``[N, C, H, W]`` and type *T* with feature maps. **Required.** +* **1**: 4D input tensor with shape ``[N, C, H, W]`` and type *T* with feature maps. **Required.** -* **2**: 2D input tensor with shape ``[num_boxes, 5]``. It contains a list of five element tuples that describe a region of interest: ``[batch_id, x_1, y_1, x_2, y_2]``. **Required.** -Batch indices must be in the range of ``[0, N-1]``. +* **2**: 2D input tensor with shape ``[num_boxes, 5]``. It contains a list of five element tuples that describe a region of interest: ``[batch_id, x_1, y_1, x_2, y_2]``. **Required.** + Batch indices must be in the range of ``[0, N-1]``. **Outputs**: diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/detection/roi-align-rotated-15.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/detection/roi-align-rotated-15.rst index 1da1e33079c106..6663b048fdc6de 100644 --- a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/detection/roi-align-rotated-15.rst +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/detection/roi-align-rotated-15.rst @@ -1,4 +1,5 @@ .. {#openvino_docs_ops_detection_ROIAlignRotated_15} + ROIAlignRotated =============== @@ -56,9 +57,9 @@ Each ROI box's center is shifted by [-0.5, -0.5] before pooling to achive better * *spatial_scale* * **Description**: *spatial_scale* is a multiplicative spatial scale factor to that is applied to the ROI box(height, weight and center vector) before pooling. - WARNING! - Spatial scale is also applied to the center point of the ROI box. It means that scaling does not only change the size of the ROI box, but also its position. - For example, if the spatial scale is 2.0, ROI box center is [0.5, 0.5], box width is 1.0 and box height is 1.0, then after scaling the ROI box center will be [1.0, 1.0], box width will be 2.0 and box height will be 2.0. + WARNING! + Spatial scale is also applied to the center point of the ROI box. It means that scaling does not only change the size of the ROI box, but also its position. + For example, if the spatial scale is 2.0, ROI box center is [0.5, 0.5], box width is 1.0 and box height is 1.0, then after scaling the ROI box center will be [1.0, 1.0], box width will be 2.0 and box height will be 2.0. * **Range of values**: a positive floating-point number * **Type**: ``float`` * **Required**: *yes* diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/detection/roi-pooling-1.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/detection/roi-pooling-1.rst index 26377ca0aebfcc..7deb444d388ac8 100644 --- a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/detection/roi-pooling-1.rst +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/detection/roi-pooling-1.rst @@ -5,7 +5,7 @@ ROIPooling .. meta:: - :description: Learn about ROIPooling-1 - an object detection operation, + :description: Learn about ROIPooling-1 - an object detection operation, which can be performed on two required input tensors. **Versioned name**: *ROIPooling-1* @@ -61,10 +61,10 @@ The box height and width have different representation based on **method** attri **Inputs**: -* **1**: 4D input tensor of shape ``[N, C, H, W]`` with feature maps of type *T*. **Required.** +* **1**: 4D input tensor of shape ``[N, C, H, W]`` with feature maps of type *T*. **Required.** -* **2**: 2D input tensor of shape ``[NUM_ROIS, 5]`` describing region of interest box consisting of 5 element tuples of type *T*: ``[batch_id, x_1, y_1, x_2, y_2]``. **Required.** -Batch indices must be in the range of ``[0, N-1]``. +* **2**: 2D input tensor of shape ``[NUM_ROIS, 5]`` describing region of interest box consisting of 5 element tuples of type *T*: ``[batch_id, x_1, y_1, x_2, y_2]``. **Required.** + Batch indices must be in the range of ``[0, N-1]``. **Outputs**: diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/image/interpolate-1.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/image/interpolate-1.rst index 470fa5b7099006..6968d556734d20 100644 --- a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/image/interpolate-1.rst +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/image/interpolate-1.rst @@ -51,7 +51,7 @@ Interpolate * *pads_begin* * **Description**: *pads_beg* specify the number of pixels to add to the beginning of the image being interpolated. -This is a scalar that specifies padding for each spatial dimension. + This is a scalar that specifies padding for each spatial dimension. * **Range of values**: list of non-negative integer numbers * **Type**: ``int`` * **Default value**: 0 @@ -60,7 +60,7 @@ This is a scalar that specifies padding for each spatial dimension. * *pads_end* * **Description**: *pads_end* specify the number of pixels to add to the beginning of the image being interpolated. -This is a scalar that specifies padding for each spatial dimension. + This is a scalar that specifies padding for each spatial dimension. * **Range of values**: list of non-negative integer numbers * **Type**: ``int`` * **Default value**: 0 diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/image/interpolate-4.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/image/interpolate-4.rst index 7572f7c1bc97ac..202bcde7c0c60c 100644 --- a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/image/interpolate-4.rst +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/image/interpolate-4.rst @@ -22,7 +22,12 @@ Interpolate * **Range of values**: one of ``nearest``, ``linear``, ``linear_onnx``, ``cubic`` * **Type**: string * **Required**: *yes* - **Note**: Only 2D, 3D, 4D, 5D tensors with ``axes = {0, 1}``, ``axes = {0, 1, 2}``, ``axes = {2, 3}``, ``axes = {2, 3, 4}`` respectively are supported for ``"mode" == "linear_onnx"``. + + .. note:: + + Only 2D, 3D, 4D, 5D tensors with ``axes = {0, 1}``, ``axes = {0, 1, 2}``, + ``axes = {2, 3}``, ``axes = {2, 3, 4}`` respectively are supported for + ``"mode" == "linear_onnx"``. * *shape_calculation_mode* diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/infrastructure/loop-5.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/infrastructure/loop-5.rst index 375833ec7ee18e..0e1e206d821ad0 100644 --- a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/infrastructure/loop-5.rst +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/infrastructure/loop-5.rst @@ -72,7 +72,7 @@ There are several combinations of these two inputs ``(trip_count, execution cond Loop operation description in the IR has regular sections: ``input`` and ``output``. They connect Loop body to the outer graph and specify condition(s). Loop operation description in the IR also has several special sections: ``body``, ``port_map`` and ``back_edges`` similar to the ones from the TensorIterator operation but having some important features described below. -1. The body operation getting an input from the main graph should have an entry in the ``port_map`` section of the Loop operation. These edges connect input ports of the Loop with the body ``Parameter``s. +1. The body operation getting an input from the main graph should have an entry in the ``port_map`` section of the Loop operation. These edges connect input ports of the Loop with the body ``Parameter``\ s. 2. Input tensors to the Loop can be sliced along a specified axis, the Loop can iterates over all sliced parts. The corresponding ``input`` entry in the ``port_map`` should have ``axis`` attribute specifying the axis to slice. Therefore, inputs to the Loop operation corresponding to ``input`` entries in the ``port_map`` without ``axis`` attribute are used "as is" (without slicing). 3. The body operation producing tensor to be used in the subsequent iterations (like in RNN models) should have a back edge described in the ``back_edges`` section of the operation. The back edge connects the respective body ``Parameter`` and ``Result`` operations. For such a case the Loop operation node provides input for the first iteration, while corresponding Loop operation output produces the tensor computed during the last iteration. 4. Output tensors produced by a particular body operation across all iterations can be concatenated and returned as a Loop operation output (this is a "scan output" according to the ONNX* Loop operation `specification `__ ). The corresponding ``output`` entry in the ``port_map`` should have ``axis`` attribute specifying the axis to concatenate. Therefore, outputs from operations corresponding to ``output`` entries in the ``port_map`` without ``axis`` attribute are returned "as is" (without concatenation). @@ -114,7 +114,7 @@ Loop operation description in the IR also has several special sections: ``body`` * *axis* * **Description**: if *axis* is specified for ``output`` entry, then it is an axis to concatenate the body ``Result`` output across all iterations. - If *axis* is specified for ``input`` entry, then it is an axis to iterate through, it triggers the slicing of the input tensor. + If *axis* is specified for ``input`` entry, then it is an axis to iterate through, it triggers the slicing of the input tensor. * **Range of values**: an integer. Negative value means counting dimension from the end. * **Type**: ``int`` diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/movement/gather-7.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/movement/gather-7.rst index ebe248309a122d..cd1026cb052b18 100644 --- a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/movement/gather-7.rst +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/movement/gather-7.rst @@ -13,7 +13,7 @@ Gather **Category**: *Data movement* **Short description**: *Gather* operation takes slices of data of the first input tensor according to the indices - specified with the second input tensor and axis from the third input. Semantics of this operation is identical to +specified with the second input tensor and axis from the third input. Semantics of this operation is identical to TensorFlow `Gather `__ operation. **Detailed description** diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/movement/gather-8.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/movement/gather-8.rst index b2bb5bf0235c60..9bc5dc50a51fb6 100644 --- a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/movement/gather-8.rst +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/movement/gather-8.rst @@ -195,7 +195,7 @@ Example 7 with indices out of the range: **Outputs** * **1**: The resulting tensor of type *T* that consists of elements from ``data`` tensor gathered by ``indices``. The shape -of the output tensor is ``data.shape[:axis] + indices.shape[batch_dims:] + data.shape[axis + 1:]`` + of the output tensor is ``data.shape[:axis] + indices.shape[batch_dims:] + data.shape[axis + 1:]`` **Types** diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/movement/scatter-elements-update-3.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/movement/scatter-elements-update-3.rst index 69eafbb10f7c26..175e838ab0b576 100644 --- a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/movement/scatter-elements-update-3.rst +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/movement/scatter-elements-update-3.rst @@ -33,20 +33,20 @@ For instance, in a 3D tensor case, the update corresponding to the ``[i][j][k]`` **Inputs**: -* **1**: ``data`` tensor of arbitrary rank ``r`` and of type *T*. **Required.** +* **1**: ``data`` tensor of arbitrary rank ``r`` and of type *T*. **Required.** -* **2**: ``indices`` tensor with indices of type *T_IND*. The rank of the tensor is equal to the rank of ``data`` tensor. All index values are expected to be within bounds ``[0, s - 1]`` along axis of size ``s``. If multiple indices point to the -same output location then the order of updating the values is undefined. If an index points to non-existing output -tensor element or is negative then exception is raised. **Required.** +* **2**: ``indices`` tensor with indices of type *T_IND*. The rank of the tensor is equal to the rank of ``data`` tensor. All index values are expected to be within bounds ``[0, s - 1]`` along axis of size ``s``. If multiple indices point to the + same output location then the order of updating the values is undefined. If an index points to non-existing output + tensor element or is negative then exception is raised. **Required.** -* **3**: ``updates`` tensor of shape equal to the shape of ``indices`` tensor and of type *T*. **Required.** +* **3**: ``updates`` tensor of shape equal to the shape of ``indices`` tensor and of type *T*. **Required.** -* **4**: ``axis`` tensor with scalar or 1D tensor with one element of type *T_AXIS* specifying axis for scatter. -The value can be in range ``[-r, r - 1]`` where ``r`` is the rank of ``data``. **Required.** +* **4**: ``axis`` tensor with scalar or 1D tensor with one element of type *T_AXIS* specifying axis for scatter. + The value can be in range ``[-r, r - 1]`` where ``r`` is the rank of ``data``. **Required.** **Outputs**: -* **1**: tensor with shape equal to ``data`` tensor of the type *T*. +* **1**: tensor with shape equal to ``data`` tensor of the type *T*. **Types** diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/pooling/pooling_shape_rules.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/pooling/pooling_shape_rules.rst index cc5105a4f11697..df46e4afb30c01 100644 --- a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/pooling/pooling_shape_rules.rst +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/pooling/pooling_shape_rules.rst @@ -1,6 +1,6 @@ .. {#openvino_docs_pooling_shape_rules} -Shape calculation rules for Pooling Operators +Shape Calculation Rules for Pooling Operators ============================================= .. meta:: diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/reduction/reduce-mean-1.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/reduction/reduce-mean-1.rst index 9aef4e981cf46f..768b440f626ea0 100644 --- a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/reduction/reduce-mean-1.rst +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/reduction/reduce-mean-1.rst @@ -107,7 +107,7 @@ Particular cases: -= + .. code-block:: xml :force: diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/shape/squeeze-1.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/shape/squeeze-1.rst index 1ccce710ee0f41..ae6db8fb5353a4 100644 --- a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/shape/squeeze-1.rst +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/shape/squeeze-1.rst @@ -18,7 +18,7 @@ Squeeze * If only the first input is provided, every dimension that is equal to 1 will be removed from it. * With the second input provided, each value is an index of a dimension from the first tensor that is to be removed. Specified dimension should be equal to 1, otherwise it will be ignored and copied as is. -Dimension indices can be specified directly, or by negative indices (counting dimensions from the end). + Dimension indices can be specified directly, or by negative indices (counting dimensions from the end). Note: Updated behavior since 2024.3, request of squeezing dimension not equal to 1 is expected to be ignored instead of causing an error. diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/sort/non-max-suppression-1.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/sort/non-max-suppression-1.rst index 328dd37d0f2c44..662dbd5d282e83 100644 --- a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/sort/non-max-suppression-1.rst +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/sort/non-max-suppression-1.rst @@ -5,8 +5,8 @@ NonMaxSuppression .. meta:: - :description: Learn about NonMaxSuppression-1 - a sorting and maximization - operation, which can be performed on two required and three + :description: Learn about NonMaxSuppression-1 - a sorting and maximization + operation, which can be performed on two required and three optional input tensors. **Versioned name**: *NonMaxSuppression-1* @@ -52,24 +52,24 @@ class must not exceed ``max_output_boxes_per_class``. **Inputs**: -* **1**: ``boxes`` - floating-point tensor of shape ``[num_batches, num_boxes, 4]`` with box coordinates. **Required.** +* **1**: ``boxes`` - floating-point tensor of shape ``[num_batches, num_boxes, 4]`` with box coordinates. **Required.** -* **2**: ``scores`` - floating-point tensor of shape ``[num_batches, num_classes, num_boxes]`` with box scores. **Required.** +* **2**: ``scores`` - floating-point tensor of shape ``[num_batches, num_classes, num_boxes]`` with box scores. **Required.** -* **3**: ``max_output_boxes_per_class`` - integer scalar tensor specifying maximum number of boxes to be selected per class. Optional with default value 0 meaning select no boxes. +* **3**: ``max_output_boxes_per_class`` - integer scalar tensor specifying maximum number of boxes to be selected per class. Optional with default value 0 meaning select no boxes. -* **4**: ``iou_threshold`` - floating-point scalar tensor specifying intersection over union threshold. Optional with default value 0 meaning keep all boxes. +* **4**: ``iou_threshold`` - floating-point scalar tensor specifying intersection over union threshold. Optional with default value 0 meaning keep all boxes. -* **5**: ``score_threshold`` - floating-point scalar tensor specifying minimum score to consider box for the processing. Optional with default value 0. +* **5**: ``score_threshold`` - floating-point scalar tensor specifying minimum score to consider box for the processing. Optional with default value 0. **Outputs**: -* **1**: ``selected_indices`` - integer tensor of shape ``[min(num_boxes, max_output_boxes_per_class * num_classes), 3]`` containing information about selected boxes as triplets ``[batch_index, class_index, box_index]``. -The output tensor is filled with -1s for output tensor elements if the total number of selected boxes is less than the output tensor size. +* **1**: ``selected_indices`` - integer tensor of shape ``[min(num_boxes, max_output_boxes_per_class * num_classes), 3]`` containing information about selected boxes as triplets ``[batch_index, class_index, box_index]``. + The output tensor is filled with -1s for output tensor elements if the total number of selected boxes is less than the output tensor size. **Example** -.. code-block:: cpp +.. code-block:: cpp diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/sort/non-max-suppression-4.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/sort/non-max-suppression-4.rst index 161a68b255b1c9..72a5631194f2cf 100644 --- a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/sort/non-max-suppression-4.rst +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/sort/non-max-suppression-4.rst @@ -60,20 +60,20 @@ class must not exceed ``max_output_boxes_per_class``. **Inputs**: -* **1**: ``boxes`` - tensor of type *T* and shape ``[num_batches, num_boxes, 4]`` with box coordinates. **Required.** +* **1**: ``boxes`` - tensor of type *T* and shape ``[num_batches, num_boxes, 4]`` with box coordinates. **Required.** -* **2**: ``scores`` - tensor of type *T* and shape ``[num_batches, num_classes, num_boxes]`` with box scores. **Required.** +* **2**: ``scores`` - tensor of type *T* and shape ``[num_batches, num_classes, num_boxes]`` with box scores. **Required.** -* **3**: ``max_output_boxes_per_class`` - scalar tensor of type *T_MAX_BOXES* specifying maximum number of boxes to be selected per class. Optional with default value 0 meaning select no boxes. +* **3**: ``max_output_boxes_per_class`` - scalar tensor of type *T_MAX_BOXES* specifying maximum number of boxes to be selected per class. Optional with default value 0 meaning select no boxes. -* **4**: ``iou_threshold`` - scalar tensor of type *T_THRESHOLDS* specifying intersection over union threshold. Optional with default value 0 meaning keep all boxes. +* **4**: ``iou_threshold`` - scalar tensor of type *T_THRESHOLDS* specifying intersection over union threshold. Optional with default value 0 meaning keep all boxes. -* **5**: ``score_threshold`` - scalar tensor of type *T_THRESHOLDS* specifying minimum score to consider box for the processing. Optional with default value 0. +* **5**: ``score_threshold`` - scalar tensor of type *T_THRESHOLDS* specifying minimum score to consider box for the processing. Optional with default value 0. **Outputs**: -* **1**: ``selected_indices`` - tensor of type *T_IND* and shape ``[min(num_boxes, max_output_boxes_per_class) * num_batches * num_classes, 3]`` containing information about selected boxes as triplets ``[batch_index, class_index, box_index]``. -The output tensor is filled with -1s for output tensor elements if the total number of selected boxes is less than the output tensor size. +* **1**: ``selected_indices`` - tensor of type *T_IND* and shape ``[min(num_boxes, max_output_boxes_per_class) * num_batches * num_classes, 3]`` containing information about selected boxes as triplets ``[batch_index, class_index, box_index]``. + The output tensor is filled with -1s for output tensor elements if the total number of selected boxes is less than the output tensor size. **Types** diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/sparse/embedding-bag-offsets-15.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/sparse/embedding-bag-offsets-15.rst new file mode 100644 index 00000000000000..9f0392c8e2d038 --- /dev/null +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/sparse/embedding-bag-offsets-15.rst @@ -0,0 +1,184 @@ +.. {#openvino_docs_ops_sparse_EmbeddingBagOffsets_15} + +EmbeddingBagOffsets +====================== + + +.. meta:: + :description: Learn about EmbeddingBagOffsets-15 - a sparse operation, which + can be performed on three required and two optional input tensors. + +**Versioned name**: *EmbeddingBagOffsets-15* + +**Category**: *Sparse* + +**Short description**: Computes sums or means of "bags" of embeddings, without instantiating the intermediate embeddings. + +**Detailed description**: + +Operation EmbeddingBagOffsets is an implementation of ``torch.nn.EmbeddingBag`` with indices and offsets inputs being 1D tensors. + +For each index in ``indices`` this operator gathers values from ``emb_table`` embedding table. Then values at indices in the range of the same bag (based on ``offset`` input) are reduced according to ``reduction`` attribute. + +Values in ``offsets`` define starting index in ``indices`` tensor of each "bag", +e.g. ``offsets`` with value ``[0, 3, 4, 4, 6]`` define 5 "bags" containing ``[3, 1, 0, 2, num_indices-6]`` elements corresponding to ``[indices[0:3], indices[3:4], empty_bag, indices[4:6], indices[6:]]`` slices of indices per bag. + +EmbeddingBagOffsets is an equivalent to following NumPy snippet: + +.. code-block:: py + + def embedding_bag_offsets( + emb_table: np.ndarray, + indices: np.ndarray, + offsets: np.ndarray, + default_index: Optional[int] = None, + per_sample_weights: Optional[np.ndarray] = None, + reduction: Literal["sum", "mean"] = "sum", + ): + assert ( + reduction == "sum" or per_sample_weights is None + ), "Attribute per_sample_weights is only supported in sum reduction." + if per_sample_weights is None: + per_sample_weights = np.ones_like(indices) + embeddings = [] + for emb_idx, emb_weight in zip(indices, per_sample_weights): + embeddings.append(emb_table[emb_idx] * emb_weight) + previous_offset = offsets[0] + bags = [] + offsets = np.append(offsets, len(indices)) + for bag_offset in offsets[1:]: + bag_size = bag_offset - previous_offset + if bag_size != 0: + embedding_bag = embeddings[previous_offset:bag_offset] + reduced_bag = np.add.reduce(embedding_bag) + if reduction == "mean": + reduced_bag = reduced_bag / bag_size + bags.append(reduced_bag) + else: + # Empty bag case + if default_index is not None and default_index != -1: + bags.append(emb_table[default_index]) + else: + bags.append(np.zeros(emb_table.shape[1:])) + previous_offset = bag_offset + return np.stack(bags, axis=0) + + +**Attributes**: + +* *reduction* + + * **Description**: reduction mode. + * **Range of values**: + + * sum - compute weighted sum, using corresponding values of ``per_sample_weights`` as weights if provided. + * mean - compute average of values in bag. Input ``per_sample_weights`` is not supported and will raise exception. + + * **Type**: ``string`` + * **Default value**: sum + * **Required**: *no* + +**Inputs**: + +* **1**: ``emb_table`` tensor containing the embedding lookup table of the module of shape ``[num_emb, emb_dim1, emb_dim2, ...]`` and of type *T*. **Required.** +* **2**: ``indices`` tensor of shape ``[num_indices]`` and of type *T_IND*. **Required.** +* **3**: ``offsets`` tensor of shape ``[batch]`` and of type *T_IND* containing the starting index positions of each "bag" in ``indices``. Maximum value of offsets cannot be greater than length of ``indices``. **Required.** +* **4**: ``default_index`` scalar of type *T_IND* containing default index in embedding table to fill empty "bags". If set to ``-1`` or not provided, empty "bags" are filled with zeros. Reverse indexing using negative values is not supported. **Optional.** +* **5**: ``per_sample_weights`` tensor of the same shape as ``indices`` and of type *T*. Supported only when *reduction* attribute is set to ``"sum"``. Each value in this tensor are multiplied with each value pooled from embedding table for each index. Optional, default is tensor of ones. **Optional.** + +**Outputs**: + +* **1**: tensor of shape ``[batch, emb_dim1, emb_dim2, ...]`` and of type *T* containing embeddings for each bag. + +**Types** + +* *T*: any numeric type. +* *T_IND*: ``int32`` or ``int64``. + +**Example** + +*Example 1: per_sample_weights are provided, default_index is set to 0 to fill empty bag with values gathered form emb_table on given index.* + +.. code-block:: xml + + + + + + 5 + 2 + + + 4 + + + 3 + + + + 4 + + + + + 3 + 2 + + + + +*Example 2: per_sample_weights are provided, default_index is set to -1 to fill empty bag with 0.* + +.. code-block:: xml + + + + + + 5 + 2 + + + 4 + + + 3 + + + + 4 + + + + + 3 + 2 + + + + +*Example 3: Example of reduction set to mean.* + +.. code-block:: xml + + + + + + 5 + 2 + + + 4 + + + 3 + + + + + 3 + 2 + + + diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/sparse/embedding-bag-offsets-sum-3.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/sparse/embedding-bag-offsets-sum-3.rst index 0a0cb67afb0f06..c3eb163b16d98f 100644 --- a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/sparse/embedding-bag-offsets-sum-3.rst +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/sparse/embedding-bag-offsets-sum-3.rst @@ -14,7 +14,48 @@ EmbeddingBagOffsetsSum **Short description**: Computes sums of "bags" of embeddings, without instantiating the intermediate embeddings. -**Detailed description**: This is the second case of the PyTorch `EmbeddingBag `__ , it has indices in two 1D tensors provided as 2nd and 3rd inputs. For each index in ``indices`` this operator gets values from ``data`` embedding table and sums all values belonging to each bag. Values in ``offsets`` define starting index in ``indices`` tensor of each "bag", e.g. ``offsets`` with value ``[0,3,4,4,6]`` define 5 "bags" containing ``[3,1,0,2,n-6]`` elements. +**Detailed description**: + +Operation EmbeddingBagOffsets is an implementation of ``torch.nn.EmbeddingBag`` with indices and offsets inputs being 1D tensors. + +For each index in ``indices`` this operator gathers values from ``emb_table`` embedding table. Then values at indices in the range of the same bag (based on ``offset`` input) are reduced according to ``reduction`` attribute. + +Values in ``offsets`` define starting index in ``indices`` tensor of each "bag", +e.g. ``offsets`` with value ``[0, 3, 4, 4, 6]`` define 5 "bags" containing ``[3, 1, 0, 2, num_indices-6]`` elements corresponding to ``[indices[0:3], indices[3:4], empty_bag, indices[4:6], indices[6:]]`` slices of indices per bag. + +EmbeddingBagOffsetsSum is an equivalent to following NumPy snippet: + +.. code-block:: py + + def embedding_bag_offsets( + emb_table: np.ndarray, + indices: np.ndarray, + offsets: np.ndarray, + default_index: Optional[int] = None, + per_sample_weights: Optional[np.ndarray] = None, + ): + if per_sample_weights is None: + per_sample_weights = np.ones_like(indices) + embeddings = [] + for emb_idx, emb_weight in zip(indices, per_sample_weights): + embeddings.append(emb_table[emb_idx] * emb_weight) + previous_offset = offsets[0] + bags = [] + offsets = np.append(offsets, len(indices)) + for bag_offset in offsets[1:]: + bag_size = bag_offset - previous_offset + if bag_size != 0: + embedding_bag = embeddings[previous_offset:bag_offset] + reduced_bag = np.add.reduce(embedding_bag) + bags.append(reduced_bag) + else: + # Empty bag case + if default_index is not None and default_index != -1: + bags.append(emb_table[default_index]) + else: + bags.append(np.zeros(emb_table.shape[1:])) + previous_offset = bag_offset + return np.stack(bags, axis=0) **Attributes**: EmbeddingBagOffsetsSum operation has no attributes. @@ -22,9 +63,9 @@ EmbeddingBagOffsetsSum * **1**: ``emb_table`` tensor containing the embedding lookup table of the module of shape ``[num_emb, emb_dim1, emb_dim2, ...]`` and of type *T*. **Required.** * **2**: ``indices`` tensor of shape ``[num_indices]`` and of type *T_IND*. **Required.** -* **3**: ``offsets`` tensor of shape ``[batch]`` and of type *T_IND* containing the starting index positions of each "bag" in ``indices``. **Required.** -* **4**: ``default_index`` scalar of type *T_IND* containing default index in embedding table to fill empty "bags". If not provided empty "bags" are filled with zeros. **Optional.** -* **5**: ``per_sample_weights`` tensor of the same shape as ``indices`` and of type *T*. Each value in this tensor are multiplied with each value pooled from embedding table for each index. Optional, default is tensor of ones. +* **3**: ``offsets`` tensor of shape ``[batch]`` and of type *T_IND* containing the starting index positions of each "bag" in ``indices``. Maximum value of offsets cannot be greater than length of ``indices``. **Required.** +* **4**: ``default_index`` scalar of type *T_IND* containing default index in embedding table to fill empty "bags". If set to ``-1`` or not provided, empty "bags" are filled with zeros. Reverse indexing using negative values is not supported. **Optional.** +* **5**: ``per_sample_weights`` tensor of the same shape as ``indices`` and of type *T*. Each value in this tensor are multiplied with each value pooled from embedding table for each index. Optional, default is tensor of ones. **Optional.** **Outputs**: @@ -37,7 +78,9 @@ EmbeddingBagOffsetsSum **Example** -.. code-block:: cpp +*Example 1: per_sample_weights are provided, default_index is set to 0 to fill empty bag with values gathered form emb_table on given index.* + +.. code-block:: xml @@ -52,7 +95,7 @@ EmbeddingBagOffsetsSum 3 - + 4 @@ -64,4 +107,31 @@ EmbeddingBagOffsetsSum +*Example 2: per_sample_weights are provided, default_index is set to -1 to fill empty bag with 0.* + +.. code-block:: xml + + + + 5 + 2 + + + 4 + + + 3 + + + + 4 + + + + + 3 + 2 + + + diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/sparse/embedding-bag-packed-15.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/sparse/embedding-bag-packed-15.rst new file mode 100644 index 00000000000000..2892d49759f667 --- /dev/null +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/sparse/embedding-bag-packed-15.rst @@ -0,0 +1,131 @@ +.. {#openvino_docs_ops_sparse_EmbeddingBagPacked_15} + +EmbeddingBagPacked +===================== + + +.. meta:: + :description: Learn about EmbeddingBagPacked-15 - a sparse operation, which + can be performed on two required and one optional input tensor. + +**Versioned name**: *EmbeddingBagPacked-15* + +**Category**: *Sparse* + +**Short description**: Computes sums or means of "bags" of embeddings, without instantiating the intermediate embeddings. + +**Detailed description**: + +Operation EmbeddingBagPacked is an implementation of ``torch.nn.EmbeddingBag`` with indices input being 2D tensor of shape ``[batch, indices_per_bag]``. +Operation is equivalent to *gather_op = Gather(emb_table, indices, axis=0)* followed by reduction: + + * *sum* - *ReduceSum(Multiply(gather_op, Unsqueeze(per_sample_weights, -1)), axis=1)*, + * *mean* - *ReduceMean(gather_op, axis=1)*. + +**Attributes**: + +* *reduction* + + * **Description**: reduction mode. + * **Range of values**: + + * sum - compute weighted sum, using corresponding values of ``per_sample_weights`` as weights if provided. + * mean - compute average of values in bag. Input ``per_sample_weights`` is not supported and will raise exception. + + * **Type**: ``string`` + * **Default value**: sum + * **Required**: *no* + +**Inputs**: + +* **1**: ``emb_table`` tensor containing the embedding lookup table of the module of shape ``[num_emb, emb_dim1, emb_dim2, ...]`` and of type *T*. **Required.** +* **2**: ``indices`` tensor of shape ``[batch, indices_per_bag]`` and of type *T_IND*. **Required.** +* **3**: ``per_sample_weights`` tensor of the same shape as ``indices`` and of type *T* supported only in ``sum`` mode. Each value in this tensor are multiplied with each value pooled from embedding table for each index. Optional, default is tensor of ones. **Optional.** + +**Outputs**: + +* **1**: tensor of shape ``[batch, emb_dim1, emb_dim2, ...]`` and of type *T* containing embeddings for each bag. + +**Types** + +* *T*: any numeric type. +* *T_IND*: ``int32`` or ``int64``. + +**Example** + +*Example 1: reduction set to sum, per_sample_weights are not provided.* + +.. code-block:: xml + + + + + + 5 + 2 + + + 3 + 2 + + + + + 3 + 2 + + + + +*Example 2: reduction set to sum and per_sample_weights are provided.* + +.. code-block:: xml + + + + + + 5 + 2 + + + 3 + 2 + + + 3 + 2 + + + + + 3 + 2 + + + + +*Example 3: reduction set to mean, per_sample_weights are not provided.* + +.. code-block:: xml + + + + + + 5 + 2 + + + 3 + 2 + + + + + 3 + 2 + + + + diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/sparse/embedding-bag-packed-sum-3.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/sparse/embedding-bag-packed-sum-3.rst index 9ef623ca7755eb..b6cad12be869ac 100644 --- a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/sparse/embedding-bag-packed-sum-3.rst +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/sparse/embedding-bag-packed-sum-3.rst @@ -14,7 +14,10 @@ EmbeddingBagPackedSum **Short description**: Computes sums of "bags" of embeddings, without instantiating the intermediate embeddings. -**Detailed description**: This is the first case of the PyTorch `EmbeddingBag `__ , it has indices in the tensor of format ``[batch, indices_per_bag]``. If 3rd input is not provided, this operation is equivalent to *Gather* followed by *ReduceSum(axis=0)*. However, *EmbeddingBagPackedSum* is much more time and memory efficient than using a chain of these operations. +**Detailed description**: + +Operation EmbeddingBagPackedSum is an implementation of ``torch.nn.EmbeddingBag`` in ``sum`` mode, which indices input being 2D tensor of shape ``[batch, indices_per_bag]``. +Operation is equivalent to *ReduceSum(Multiply(Gather(emb_table, indices, axis=0), Unsqueeze(per_sample_weights, -1)), axis=1)*. **Attributes**: EmbeddingBagPackedSum operation has no attributes. @@ -35,7 +38,7 @@ EmbeddingBagPackedSum **Example** -.. code-block:: cpp +.. code-block:: xml @@ -47,13 +50,13 @@ EmbeddingBagPackedSum 3 2 - + 3 2 - + 3 2 diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/type/string-tensor-pack-15.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/type/string-tensor-pack-15.rst new file mode 100644 index 00000000000000..77f44f8d687741 --- /dev/null +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/type/string-tensor-pack-15.rst @@ -0,0 +1,147 @@ +.. {#openvino_docs_ops_type_StringTensorPack_15} + +StringTensorPack +=================== + + +.. meta:: + :description: Learn about StringTensorPack-15 - operation which packs a concatenated batch of strings into a batched string tensor. + +**Versioned name**: *StringTensorPack-15* + +**Category**: *Type* + +**Short description**: *StringTensorPack* transforms a concatenated strings data (encoded as 1D tensor of u8 element type) into +a string tensor using *begins* and *ends* indices. + +**Detailed description** + +Consider inputs: + +* *begins* = [0, 5] +* *ends* = [5, 13] +* *symbols* = "IntelOpenVINO" + +*StringTensorPack* uses indices from ``begins`` and ``ends`` to transform concatenated string ``symbols`` into ``output``, +a string tensor. The ``output.shape`` is equal to ``begins.shape`` and ``ends.shape``, +and in this case ``output`` holds values ``["Intel", "OpenVINO"]``. + +When defining *begins* and *ends*, the notation ``[a, b)`` is used. This means that the range starts with ``a`` and includes all values up to, +but not including, ``b``. That is why in the example given the length of "IntelOpenVINO" is 12, but *ends* vector contains 13. The shapes of ``begins`` and ``ends`` are required to be equal. + +**Inputs** + +* **1**: ``begins`` - ND tensor of non-negative integer numbers of type *T_IDX*, containing indices of each string's beginnings. **Required.** + +* **2**: ``ends`` - ND tensor of non-negative integer numbers of type *T_IDX*, containing indices of each string's endings. **Required.** + +* **3**: ``symbols`` - 1D tensor of concatenated strings data encoded in utf-8 bytes, of type *u8*. **Required.** + +**Outputs** + +* **1**: ``output`` - ND string tensor of the same shape as *begins* and *ends*. + +**Types** + +* *T_IDX*: ``int32`` or ``int64``. + +**Examples** + +*Example 1: 1D begins and ends* + +.. code-block:: xml + :force: + + + + + 2 + + + 2 + + + 13 + + + + + 2 + + + + +*Example 2: input with an empty string* + +.. code-block:: xml + :force: + + + + + 2 + + + 2 + + + 13 + + + + + 5 + + + + +*Example 3: skipped symbols* + +.. code-block:: xml + :force: + + + + + 2 + + + 2 + + + 13 + + + + + 5 + + + + +*Example 4: 2D begins and ends* + +.. code-block:: xml + :force: + + + + + 2 + 2 + + + 2 + 2 + + + 21 + + + + + 2 + 2 + + + diff --git a/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/type/string-tensor-unpack-15.rst b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/type/string-tensor-unpack-15.rst new file mode 100644 index 00000000000000..8ae8a8f8234b21 --- /dev/null +++ b/docs/articles_en/documentation/openvino-ir-format/operation-sets/operation-specs/type/string-tensor-unpack-15.rst @@ -0,0 +1,133 @@ +.. {#openvino_docs_ops_type_StringTensorUnpack_15} + +StringTensorUnpack +=================== + + +.. meta:: + :description: Learn about StringTensorUnpack-15 - operation which unpacks a batch of strings into three tensors. + +**Versioned name**: *StringTensorUnpack-15* + +**Category**: *Type* + +**Short description**: *StringTensorUnpack* operation transforms a given batch of strings into three tensors - two storing begin +and end indices of the strings and another containing the concatenated string data, respectively. + +**Detailed description** + +Consider an ``input`` string tensor containing values ``["Intel", "OpenVINO"]``. + +The operator will transform the tensor into three outputs: + +* *begins* = [0, 5] + * ``begins[0]`` is equal to 0, because the first string starts at the beginning index. + * ``begins[1]`` is equal to 5, because length of the string "Intel" is equal to 5. + * ``begins.shape`` is equal to [2], because the ``input`` is a batch of 2 strings. + +* *ends* = [5, 13] + * ``ends[0]`` is equal to 5, because length of the string "Intel" is equal to 5. + * ``ends[1]`` is equal to 13, because length of the string "OpenVINO" is 8, and it needs to be summed up with length of the string "Intel". + * ``ends.shape`` is equal to ``[2]``, because the ``input`` is a batch of 2 strings. + +* *symbols* = "IntelOpenVINO" + * ``symbols`` contains concatenated string data encoded in utf-8 bytes, interpretable using ``begins`` and ``ends``. + * ``symbols.shape`` is equal to ``[13]``, because it's the length of concatenated ``input`` strings. + +When defining *begins* and *ends*, the notation ``[a, b)`` is used. This means that the range starts with ``a`` and includes all values up to, +but not including, ``b``. That is why in the example given the length of "IntelOpenVINO" is 12, but *ends* vector contains 13. + +**Inputs** + +* **1**: ``data`` - ND tensor of type *string*. **Required.** + +**Outputs** + +* **1**: ``begins`` - ND tensor of non-negative integer numbers of type *int32* and of the same shape as ``data`` input. + +* **2**: ``ends`` - ND tensor of non-negative integer numbers of type *int32* and of the same shape as ``data`` input. + +* **3**: ``symbols`` - 1D tensor of concatenated strings data encoded in utf-8 bytes, of type *u8* and size equal to the sum of the lengths of each string from the ``data`` input. + +**Examples** + +*Example 1: 1D input* + +For ``input = ["Intel", "OpenVINO"]`` + +.. code-block:: xml + :force: + + + + + 2 + + + + + 2 + + + 2 + + + 13 + + + + +*Example 2: input with an empty string* + +For ``input = ["OMZ", "", "GenAI", " ", "2024"]`` + +.. code-block:: xml + :force: + + + + + 5 + + + + + 2 + + + 2 + + + 13 + + + + +*Example 3: 2D input* + +For ``input = [["Intel", "OpenVINO"], ["OMZ", "GenAI"]]`` + +.. code-block:: xml + :force: + + + + + 2 + 2 + + + + + 2 + 2 + + + 2 + 2 + + + 21 + + + diff --git a/docs/articles_en/get-started.rst b/docs/articles_en/get-started.rst index c734bd704d2974..3c62dec7f261c2 100644 --- a/docs/articles_en/get-started.rst +++ b/docs/articles_en/get-started.rst @@ -59,8 +59,6 @@ See the :doc:`installation overview page ` for opt OpenVINO provides a wide array of examples and documentation showing how to work with models, run inference, and deploy applications. Step through the sections below to learn the basics of OpenVINO and explore its advanced optimization features. For further details, visit :doc:`OpenVINO documentation `. -OpenVINO users of all experience levels can try `Intel® DevCloud `__ , a free web-based service for developing, testing, and running OpenVINO applications on an online cluster of the latest Intel® hardware. - .. _openvino-basics: OpenVINO Basics diff --git a/docs/articles_en/get-started/install-openvino.rst b/docs/articles_en/get-started/install-openvino.rst index 70c53b5389138f..f8fefc4abacece 100644 --- a/docs/articles_en/get-started/install-openvino.rst +++ b/docs/articles_en/get-started/install-openvino.rst @@ -14,52 +14,63 @@ Install OpenVINO™ 2024.2 OpenVINO Runtime on Linux OpenVINO Runtime on Windows OpenVINO Runtime on macOS - + Create an OpenVINO Yocto Image + OpenVINO GenAI Flavor .. raw:: html - + -.. warning:: +OpenVINO 2024.2, described here, is not a Long-Term-Support version! +All currently supported versions are: - The OpenVINO™ Development Tools package has been deprecated and removed from the default - installation options. For new projects, the OpenVINO runtime package now includes - all necessary components. +* 2024.2 (development) +* 2023.3 (LTS) +* 2022.3 (LTS) - The OpenVINO Development Tools is still available for older versions of OpenVINO, - as well as the current one, from the GitHub repository and PyPI. :doc:`Learn more <../documentation/legacy-features/install-dev-tools>`. +.. dropdown:: Distributions and Device Support - -.. tip:: - - OpenVINO 2024.2, described here, is not a Long-Term-Support version! - All currently supported versions are: - - * 2024.2 (development) - * 2023.3 (LTS) - * 2022.3 (LTS) - - Moreover, different OpenVINO distributions may support slightly different sets of features. + Different OpenVINO distributions may support slightly different sets of features. Read installation guides for particular distributions for more details. + Refer to the :doc:`OpenVINO Release Policy <../../../about-openvino/release-notes-openvino/release-policy>` + to learn more about the release types. .. dropdown:: Distribution Comparison for OpenVINO 2024.2 =============== ========== ====== =============== ======== ============ ========== ========== ========== - Device Archives PyPI APT/YUM/ZYPPER Conda Homebrew vcpkg Conan npm + Device Archives PyPI APT/YUM/ZYPPER Conda Homebrew vcpkg Conan npm =============== ========== ====== =============== ======== ============ ========== ========== ========== - CPU V V V V V V V V - GPU V V V V V V V V - NPU V\* V\* V\ * n/a n/a n/a n/a V\* + CPU V V V V V V V V + GPU V V V V V V V V + NPU V\* V\* V\ * n/a n/a n/a n/a V\* =============== ========== ====== =============== ======== ============ ========== ========== ========== - | \* **Of the Linux systems, versions 22.04 and 24.04 include drivers for NPU.** + | \* **Of the Linux systems, version 22.04 includes drivers for NPU.** | **For Windows, CPU inference on ARM64 is not supported.** -| **Build OpenVINO from source** -| OpenVINO Toolkit source files are available on GitHub as open source. If you want to build your own version of OpenVINO for your platform, - follow the `OpenVINO Build Instructions `__. +.. dropdown:: Effortless GenAI integration with OpenVINO GenAI Flavor + + A new OpenVINO GenAI Flavor streamlines application development by providing + LLM-specific interfaces for easy integration of language models, handling tokenization and + text generation. For installation and usage instructions, proceed to + :doc:`Install OpenVINO GenAI Flavor <../learn-openvino/llm_inference_guide/genai-guide>` and + :doc:`Run LLMs with OpenVINO GenAI Flavor <../learn-openvino/llm_inference_guide/genai-guide>`. + +.. dropdown:: Deprecation of OpenVINO™ Development Tools Package + + The OpenVINO™ Development Tools package has been deprecated and removed from the default + installation options. For new projects, the OpenVINO runtime package now includes + all necessary components. + + The OpenVINO Development Tools is still available for older versions of OpenVINO, + as well as the current one, from the GitHub repository and PyPI. :doc:`Learn more <../documentation/legacy-features/install-dev-tools>`. + +.. dropdown:: Building OpenVINO from Source + + OpenVINO Toolkit source files are available on GitHub as open source. If you want to build your own version of OpenVINO for your platform, + follow the `OpenVINO Build Instructions `__. diff --git a/docs/articles_en/get-started/install-openvino/install-openvino-apt.rst b/docs/articles_en/get-started/install-openvino/install-openvino-apt.rst index 058b93f3b9dd75..004f74f74043fd 100644 --- a/docs/articles_en/get-started/install-openvino/install-openvino-apt.rst +++ b/docs/articles_en/get-started/install-openvino/install-openvino-apt.rst @@ -252,13 +252,8 @@ You can also try the following: * Learn more about :doc:`OpenVINO Workflow <../../../openvino-workflow>`. * To prepare your models for working with OpenVINO, see :doc:`Model Preparation <../../../openvino-workflow/model-preparation>`. -* See pre-trained deep learning models in our :doc:`Open Model Zoo <../../../documentation/legacy-features/model-zoo>`. +* See pre-trained deep learning models on `Hugging Face `__ * Learn more about :doc:`Inference with OpenVINO Runtime <../../../openvino-workflow/running-inference>`. * See sample applications in :doc:`OpenVINO toolkit Samples Overview <../../../learn-openvino/openvino-samples>`. * Take a glance at the OpenVINO `product home page `__ . - - - - - diff --git a/docs/articles_en/get-started/install-openvino/install-openvino-archive-linux.rst b/docs/articles_en/get-started/install-openvino/install-openvino-archive-linux.rst index 32b7b36ce5bba9..c64c27678b4fa4 100644 --- a/docs/articles_en/get-started/install-openvino/install-openvino-archive-linux.rst +++ b/docs/articles_en/get-started/install-openvino/install-openvino-archive-linux.rst @@ -25,14 +25,11 @@ Install OpenVINO™ Runtime on Linux from an Archive File Debian9 armhf V n/a n/a Ubuntu20 arm64 V n/a n/a CentOS7 x86_64 V V n/a - Ubuntu18 x86_64 V V n/a Ubuntu20 x86_64 V V V Ubuntu22 x86_64 V V V - Ubuntu24 x86_64 V V V RHEL8 x86_64 V V n/a =================== ===== ===== ===== - .. tab-set:: .. tab-item:: System Requirements @@ -61,11 +58,6 @@ Install OpenVINO™ Runtime on Linux from an Archive File * GCC 9.3.0 - .. tab-item:: Ubuntu 18.04 - :sync: ubuntu-18 - - * GCC 7.5.0 - .. tab-item:: RHEL 8 :sync: rhel-8 @@ -75,7 +67,7 @@ Install OpenVINO™ Runtime on Linux from an Archive File :sync: centos-7 * GCC 8.3.1 - Use the following instructions to install it: + Use the following instructions to install it: Install GCC 8.3.1 via devtoolset-8 @@ -92,10 +84,6 @@ Install OpenVINO™ Runtime on Linux from an Archive File gcc -v - - - - Installing OpenVINO Runtime ############################################################ @@ -120,7 +108,7 @@ Step 1: Download and Install the OpenVINO Core Components cd /Downloads -4. Download the `OpenVINO Runtime archive file for your system `_, extract the files, rename the extracted folder and move it to the desired path: +4. Download the `OpenVINO Runtime archive file for your system `_, extract the files, rename the extracted folder and move it to the desired path: .. tab-set:: @@ -135,9 +123,9 @@ Step 1: Download and Install the OpenVINO Core Components .. code-block:: sh - curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.1/linux/l_openvino_toolkit_ubuntu22_2024.1.0.15008.f4afc983258_x86_64.tgz --output openvino_2024.1.0.tgz - tar -xf openvino_2024.1.0.tgz - sudo mv l_openvino_toolkit_ubuntu24_2024.1.0.15008.f4afc983258_x86_64 /opt/intel/openvino_2024.1.0 + curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.2/linux/l_openvino_toolkit_ubuntu24_2024.2.0.15519.5c0f38f83f6_x86_64.tgz --output openvino_2024.2.0.tgz + tar -xf openvino_2024.2.0.tgz + sudo mv l_openvino_toolkit_ubuntu24_2024.2.0.15519.5c0f38f83f6_x86_64 /opt/intel/openvino_2024.2.0 .. tab-item:: Ubuntu 22.04 :sync: ubuntu-22 @@ -145,9 +133,9 @@ Step 1: Download and Install the OpenVINO Core Components .. code-block:: sh - curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.1/linux/l_openvino_toolkit_ubuntu22_2024.1.0.15008.f4afc983258_x86_64.tgz --output openvino_2024.1.0.tgz - tar -xf openvino_2024.1.0.tgz - sudo mv l_openvino_toolkit_ubuntu22_2024.1.0.15008.f4afc983258_x86_64 /opt/intel/openvino_2024.1.0 + curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.2/linux/l_openvino_toolkit_ubuntu22_2024.2.0.15519.5c0f38f83f6_x86_64.tgz --output openvino_2024.2.0.tgz + tar -xf openvino_2024.2.0.tgz + sudo mv l_openvino_toolkit_ubuntu22_2024.2.0.15519.5c0f38f83f6_x86_64 /opt/intel/openvino_2024.2.0 .. tab-item:: Ubuntu 20.04 :sync: ubuntu-20 @@ -155,20 +143,9 @@ Step 1: Download and Install the OpenVINO Core Components .. code-block:: sh - curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.1/linux/l_openvino_toolkit_ubuntu20_2024.1.0.15008.f4afc983258_x86_64.tgz --output openvino_2024.1.0.tgz - tar -xf openvino_2024.1.0.tgz - sudo mv l_openvino_toolkit_ubuntu20_2024.1.0.15008.f4afc983258_x86_64 /opt/intel/openvino_2024.1.0 - - - .. tab-item:: Ubuntu 18.04 - :sync: ubuntu-18 - - .. code-block:: sh - - - curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.1/linux/l_openvino_toolkit_ubuntu18_2024.1.0.15008.f4afc983258_x86_64.tgz --output openvino_2024.1.0.tgz - tar -xf openvino_2024.1.0.tgz - sudo mv l_openvino_toolkit_ubuntu18_2024.1.0.15008.f4afc983258_x86_64 /opt/intel/openvino_2024.1.0 + curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.2/linux/l_openvino_toolkit_ubuntu20_2024.2.0.15519.5c0f38f83f6_x86_64.tgz --output openvino_2024.2.0.tgz + tar -xf openvino_2024.2.0.tgz + sudo mv l_openvino_toolkit_ubuntu20_2024.2.0.15519.5c0f38f83f6_x86_64 /opt/intel/openvino_2024.2.0 .. tab-item:: RHEL 8 :sync: rhel-8 @@ -176,18 +153,18 @@ Step 1: Download and Install the OpenVINO Core Components .. code-block:: sh - curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.1/linux/l_openvino_toolkit_rhel8_2024.1.0.15008.f4afc983258_x86_64.tgz --output openvino_2024.1.0.tgz - tar -xf openvino_2024.1.0.tgz - sudo mv l_openvino_toolkit_rhel8_2024.1.0.15008.f4afc983258_x86_64 /opt/intel/openvino_2024.1.0 + curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.2/linux/l_openvino_toolkit_rhel8_2024.2.0.15519.5c0f38f83f6_x86_64.tgz --output openvino_2024.2.0.tgz + tar -xf openvino_2024.2.0.tgz + sudo mv l_openvino_toolkit_rhel8_2024.2.0.15519.5c0f38f83f6_x86_64 /opt/intel/openvino_2024.2.0 .. tab-item:: CentOS 7 :sync: centos-7 .. code-block:: sh - curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.1/linux/l_openvino_toolkit_centos7_2024.1.0.15008.f4afc983258_x86_64.tgz --output openvino_2024.1.0.tgz - tar -xf openvino_2024.1.0.tgz - sudo mv l_openvino_toolkit_centos7_2024.1.0.15008.f4afc983258_x86_64 /opt/intel/openvino_2024.1.0 + curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.2/linux/l_openvino_toolkit_centos7_2024.2.0.15519.5c0f38f83f6_x86_64.tgz --output openvino_2024.2.0.tgz + tar -xf openvino_2024.2.0.tgz + sudo mv l_openvino_toolkit_centos7_2024.2.0.15519.5c0f38f83f6_x86_64 /opt/intel/openvino_2024.2.0 .. tab-item:: ARM 64-bit @@ -195,26 +172,25 @@ Step 1: Download and Install the OpenVINO Core Components .. code-block:: sh - - curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.1/linux/l_openvino_toolkit_ubuntu20_2024.1.0.15008.f4afc983258_arm64.tgz -O openvino_2024.1.0.tgz - tar -xf openvino_2024.1.0.tgz - sudo mv l_openvino_toolkit_ubuntu20_2024.1.0.15008.f4afc983258_arm64 /opt/intel/openvino_2024.1.0 + curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.2/linux/l_openvino_toolkit_ubuntu20_2024.2.0.15519.5c0f38f83f6_arm64.tgz -O openvino_2024.2.0.tgz + tar -xf openvino_2024.2.0.tgz + sudo mv l_openvino_toolkit_ubuntu20_2024.2.0.15519.5c0f38f83f6_arm64 /opt/intel/openvino_2024.2.0 .. tab-item:: ARM 32-bit :sync: arm-32 .. code-block:: sh - curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.1/linux/l_openvino_toolkit_debian9_2024.1.0.15008.f4afc983258_armhf.tgz -O openvino_2024.1.0.tgz - tar -xf openvino_2024.1.0.tgz - sudo mv l_openvino_toolkit_debian9_2024.1.0.15008.f4afc983258_armhf /opt/intel/openvino_2024.1.0 + curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.2/linux/l_openvino_toolkit_debian9_2024.2.0.15519.5c0f38f83f6_armhf.tgz -O openvino_2024.2.0.tgz + tar -xf openvino_2024.2.0.tgz + sudo mv l_openvino_toolkit_debian9_2024.2.0.15519.5c0f38f83f6_armhf /opt/intel/openvino_2024.2.0 5. Install required system dependencies on Linux. To do this, OpenVINO provides a script in the extracted installation directory. Run the following command: .. code-block:: sh - cd /opt/intel/openvino_2024.1.0 + cd /opt/intel/openvino_2024.2.0 sudo -E ./install_dependencies/install_openvino_dependencies.sh 6. (Optional) Install *numpy* Python Library: @@ -223,11 +199,11 @@ Step 1: Download and Install the OpenVINO Core Components This step is required only when you decide to use Python API. - You can use the ``requirements.txt`` file from the ``/opt/intel/openvino_2024.1.0/python`` folder: + You can use the ``requirements.txt`` file from the ``/opt/intel/openvino_2024.2.0/python`` folder: .. code-block:: sh - cd /opt/intel/openvino_2024.1.0 + cd /opt/intel/openvino_2024.2.0 python3 -m pip install -r ./python/requirements.txt 7. For simplicity, it is useful to create a symbolic link as below: @@ -236,7 +212,7 @@ Step 1: Download and Install the OpenVINO Core Components cd /opt/intel - sudo ln -s openvino_2024.1.0 openvino_2024 + sudo ln -s openvino_2024.2.0 openvino_2024 .. note:: If you have already installed a previous release of OpenVINO 2024, a symbolic link to the ``openvino_2024`` folder may already exist. diff --git a/docs/articles_en/get-started/install-openvino/install-openvino-archive-macos.rst b/docs/articles_en/get-started/install-openvino/install-openvino-archive-macos.rst index 2421ec54c32354..03902a8edb7dc6 100644 --- a/docs/articles_en/get-started/install-openvino/install-openvino-archive-macos.rst +++ b/docs/articles_en/get-started/install-openvino/install-openvino-archive-macos.rst @@ -63,7 +63,7 @@ Step 1: Install OpenVINO Core Components cd /Downloads -4. Download the `OpenVINO Runtime archive file for macOS `__, extract the files, rename the extracted folder and move it to the desired path: +4. Download the `OpenVINO Runtime archive file for macOS `__, extract the files, rename the extracted folder and move it to the desired path: .. tab-set:: @@ -73,9 +73,9 @@ Step 1: Install OpenVINO Core Components .. code-block:: sh - curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.1/macos/m_openvino_toolkit_macos_12_6_2024.1.0.15008.f4afc983258_x86_64.tgz --output openvino_2024.1.0.tgz - tar -xf openvino_2024.1.0.tgz - sudo mv m_openvino_toolkit_macos_12_6_2024.1.0.15008.f4afc983258_x86_64 /opt/intel/openvino_2024.1.0 + curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.2/macos/m_openvino_toolkit_macos_12_6_2024.2.0.15519.5c0f38f83f6_x86_64.tgz --output openvino_2024.2.0.tgz + tar -xf openvino_2024.2.0.tgz + sudo mv m_openvino_toolkit_macos_12_6_2024.2.0.15519.5c0f38f83f6_x86_64 /opt/intel/openvino_2024.2.0 .. tab-item:: ARM, 64-bit :sync: arm-64 @@ -83,9 +83,9 @@ Step 1: Install OpenVINO Core Components .. code-block:: sh - curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.1/macos/m_openvino_toolkit_macos_12_6_2024.1.0.15008.f4afc983258_arm64.tgz --output openvino_2024.1.0.tgz - tar -xf openvino_2024.1.0.tgz - sudo mv m_openvino_toolkit_macos_12_6_2024.1.0.15008.f4afc983258_arm64 /opt/intel/openvino_2024.1.0 + curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.2/macos/m_openvino_toolkit_macos_12_6_2024.2.0.15519.5c0f38f83f6_arm64.tgz --output openvino_2024.2.0.tgz + tar -xf openvino_2024.2.0.tgz + sudo mv m_openvino_toolkit_macos_12_6_2024.2.0.15519.5c0f38f83f6_arm64 /opt/intel/openvino_2024.2.0 5. (Optional) Install *numpy* Python Library: @@ -94,11 +94,11 @@ Step 1: Install OpenVINO Core Components This step is required only when you decide to use Python API. - You can use the ``requirements.txt`` file from the ``/opt/intel/openvino_2024.1.0/python`` folder: + You can use the ``requirements.txt`` file from the ``/opt/intel/openvino_2024.2.0/python`` folder: .. code-block:: sh - cd /opt/intel/openvino_2024.1.0 + cd /opt/intel/openvino_2024.2.0 python3 -m pip install -r ./python/requirements.txt 6. For simplicity, it is useful to create a symbolic link as below: @@ -106,7 +106,7 @@ Step 1: Install OpenVINO Core Components .. code-block:: sh - sudo ln -s /opt/intel/openvino_2024.1.0 /opt/intel/openvino_2024 + sudo ln -s /opt/intel/openvino_2024.2.0 /opt/intel/openvino_2024 .. note:: diff --git a/docs/articles_en/get-started/install-openvino/install-openvino-archive-windows.rst b/docs/articles_en/get-started/install-openvino/install-openvino-archive-windows.rst index ffa5e612ab1571..c9e32e907837b0 100644 --- a/docs/articles_en/get-started/install-openvino/install-openvino-archive-windows.rst +++ b/docs/articles_en/get-started/install-openvino/install-openvino-archive-windows.rst @@ -78,19 +78,19 @@ Step 1: Download and Install OpenVINO Core Components ``C:\Program Files (x86)\Intel`` is the recommended folder. You may also use a different path if desired or if you don't have administrator privileges on your computer. -2. Download the `OpenVINO Runtime archive file for Windows `__ to your local ``Downloads`` folder. +2. Download the `OpenVINO Runtime archive file for Windows `__ to your local ``Downloads`` folder. If you prefer using command-lines, run the following commands in the command prompt window you opened: .. code-block:: sh cd /Downloads - curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.0/windows/w_openvino_toolkit_windows_2024.1.0.15008.f4afc983258_x86_64.zip --output openvino_2024.1.0.zip --output openvino_2024.0.0.zip + curl -L https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.2/windows/w_openvino_toolkit_windows_2024.2.0.15519.5c0f38f83f6_x86_64.zip --output openvino_2024.2.0.zip --output openvino_2024.2.0.zip .. note:: - A ``.sha256`` file is provided together with the archive file to validate your download process. To do that, download the ``.sha256`` file from the same repository and run ``CertUtil -hashfile openvino_2024.1.0.zip SHA256``. Compare the returned value in the output with what's in the ``.sha256`` file: if the values are the same, you have downloaded the correct file successfully; if not, create a Support ticket `here `__. + A ``.sha256`` file is provided together with the archive file to validate your download process. To do that, download the ``.sha256`` file from the same repository and run ``CertUtil -hashfile openvino_2024.2.0.zip SHA256``. Compare the returned value in the output with what's in the ``.sha256`` file: if the values are the same, you have downloaded the correct file successfully; if not, create a Support ticket `here `__. 3. Use your favorite tool to extract the archive file, rename the extracted folder, and move it to the ``C:\Program Files (x86)\Intel`` directory. @@ -99,9 +99,9 @@ Step 1: Download and Install OpenVINO Core Components .. code-block:: sh - tar -xf openvino_2024.1.0.zip - ren w_openvino_toolkit_windows_2024.1.0.15008.f4afc983258_x86_64 openvino_2024.1.0 - move openvino_2024.1.0 "C:\Program Files (x86)\Intel" + tar -xf openvino_2024.2.0.zip + ren w_openvino_toolkit_windows_2024.2.0.15519.5c0f38f83f6_x86_64 openvino_2024.2.0 + move openvino_2024.2.0 "C:\Program Files (x86)\Intel" 4. (Optional) Install *numpy* Python Library: @@ -110,11 +110,11 @@ Step 1: Download and Install OpenVINO Core Components This step is required only when you decide to use Python API. - You can use the ``requirements.txt`` file from the ``C:\Program Files (x86)\Intel\openvino_2024.1.0\python`` folder: + You can use the ``requirements.txt`` file from the ``C:\Program Files (x86)\Intel\openvino_2024.2.0\python`` folder: .. code-block:: sh - cd "C:\Program Files (x86)\Intel\openvino_2024.1.0" + cd "C:\Program Files (x86)\Intel\openvino_2024.2.0" python -m pip install -r .\python\requirements.txt @@ -123,7 +123,7 @@ Step 1: Download and Install OpenVINO Core Components .. code-block:: sh cd C:\Program Files (x86)\Intel - mklink /D openvino_2024 openvino_2024.1.0 + mklink /D openvino_2024 openvino_2024.2.0 .. note:: diff --git a/docs/articles_en/get-started/install-openvino/install-openvino-brew.rst b/docs/articles_en/get-started/install-openvino/install-openvino-brew.rst index a5f5abeaf34db9..11bd8d90b6b641 100644 --- a/docs/articles_en/get-started/install-openvino/install-openvino-brew.rst +++ b/docs/articles_en/get-started/install-openvino/install-openvino-brew.rst @@ -96,7 +96,14 @@ Now that you've installed OpenVINO Runtime, you can try the following things: * Learn more about :doc:`OpenVINO Workflow <../../../openvino-workflow>`. * To prepare your models for working with OpenVINO, see :doc:`Model Preparation <../../../openvino-workflow/model-preparation>`. -* See pre-trained deep learning models in our :doc:`Open Model Zoo <../../../documentation/legacy-features/model-zoo>`. +* See pre-trained deep learning models in our + :doc:`Open Model Zoo <../../../documentation/legacy-features/model-zoo>`. + + .. important:: + + Due to the deprecation of Open Model Zoo, models in the OpenVINO IR format are now + published on `Hugging Face `__. + * Learn more about :doc:`Inference with OpenVINO Runtime <../../../openvino-workflow/running-inference>`. * See sample applications in :doc:`OpenVINO toolkit Samples Overview <../../../learn-openvino/openvino-samples>`. * Check out the OpenVINO `product home page `__. diff --git a/docs/articles_en/get-started/install-openvino/install-openvino-docker-linux.rst b/docs/articles_en/get-started/install-openvino/install-openvino-docker-linux.rst index 6aa5240098da10..11c2425d231dd3 100644 --- a/docs/articles_en/get-started/install-openvino/install-openvino-docker-linux.rst +++ b/docs/articles_en/get-started/install-openvino/install-openvino-docker-linux.rst @@ -1,9 +1,6 @@ -.. {#openvino_docs_install_guides_installing_openvino_docker} - Install Intel® Distribution of OpenVINO™ toolkit from a Docker Image ======================================================================= - .. meta:: :description: Learn how to use a prebuilt Docker image or create an image manually to install OpenVINO™ Runtime on Linux and Windows operating systems. @@ -33,7 +30,7 @@ The Docker CI repository includes guides on how to To start using Dockerfiles the following conditions must be met: -- Linux OS or Windows (under :ref:`Windows Subsystem for Linux (WSL2) `) +- Linux OS or Windows (under :ref:`Windows Subsystem for Linux (WSL2) `) - Installed docker engine or compatible container engine - Permissions to run containers (sudo or docker group membership) @@ -42,10 +39,14 @@ To start using Dockerfiles the following conditions must be met: OpenVINO's `Docker `__ and :doc:`Bare Metal <../install-openvino>` distributions are identical, so the documentation applies to both. + Note that starting with OpenVINO 2024.4, Ubuntu docker images will no longer be provided + and will be replaced by Debian-based ones. + .. note:: OpenVINO development environment in a docker container is also available in the - `notebook repository `__. It can be implemented in + `notebook repository `__. + It can be implemented in `OpenShift RedHat OpenData Science (RHODS) `__. More information about Docker CI for Intel® Distribution of OpenVINO™ toolset can be found diff --git a/docs/articles_en/get-started/install-openvino/install-openvino-genai.rst b/docs/articles_en/get-started/install-openvino/install-openvino-genai.rst new file mode 100644 index 00000000000000..cd6714149e7211 --- /dev/null +++ b/docs/articles_en/get-started/install-openvino/install-openvino-genai.rst @@ -0,0 +1,113 @@ +Install OpenVINO™ GenAI +==================================== + +OpenVINO GenAI is a new flavor of OpenVINO, aiming to simplify running inference of generative AI models. +It hides the complexity of the generation process and minimizes the amount of code required. +You can now provide a model and input context directly to OpenVINO, which performs tokenization of the +input text, executes the generation loop on the selected device, and returns the generated text. +For a quickstart guide, refer to the :doc:`GenAI API Guide <../../learn-openvino/llm_inference_guide/genai-guide>`. + +To see GenAI in action, check the Jupyter notebooks: +`LLM-powered Chatbot `__ and +`LLM Instruction-following pipeline `__. + +The OpenVINO GenAI flavor is available for installation via PyPI and Archive distributions: + +PyPI Installation +############################### + +To install the GenAI flavor of OpenVINO via PyPI, follow the standard :doc:`installation steps `, +but use the *openvino-genai* package instead of *openvino*: + +.. code-block:: python + + python -m pip install openvino-genai + +Archive Installation +############################### + +The OpenVINO GenAI archive package includes the OpenVINO™ Runtime and :doc:`Tokenizers <../../learn-openvino/llm_inference_guide/ov-tokenizers>`. +To install the GenAI flavor of OpenVINO from an archive file, follow the standard installation steps for your system +but instead of using the vanilla package file, download the one with OpenVINO GenAI: + +Linux +++++++++++++++++++++++++++ + +.. tab-set:: + + .. tab-item:: x86_64 + :sync: x86-64 + + .. tab-set:: + + .. tab-item:: Ubuntu 24.04 + :sync: ubuntu-24 + + .. code-block:: sh + + curl -L https://storage.openvinotoolkit.org/repositories/openvino_genai/packages/2024.2/linux/openvino_genai_ubuntu24_2024.2.0.0_x86_64.tar.gz --output openvino_genai_2024.2.0.0.tgz + tar -xf openvino_genai_2024.2.0.0.tgz + + .. tab-item:: Ubuntu 22.04 + :sync: ubuntu-22 + + .. code-block:: sh + + curl -L https://storage.openvinotoolkit.org/repositories/openvino_genai/packages/2024.2/linux/openvino_genai_ubuntu22_2024.2.0.0_x86_64.tar.gz --output openvino_genai_2024.2.0.0.tgz + tar -xf openvino_genai_2024.2.0.0.tgz + + .. tab-item:: Ubuntu 20.04 + :sync: ubuntu-20 + + .. code-block:: sh + + curl -L https://storage.openvinotoolkit.org/repositories/openvino_genai/packages/2024.2/linux/openvino_genai_ubuntu20_2024.2.0.0_x86_64.tar.gz --output openvino_genai_2024.2.0.0.tgz + tar -xf openvino_genai_2024.2.0.0.tgz + + + .. tab-item:: ARM 64-bit + :sync: arm-64 + + .. code-block:: sh + + curl -L https://storage.openvinotoolkit.org/repositories/openvino_genai/packages/2024.2/linux/openvino_genai_ubuntu20_2024.2.0.0_arm64.tar.gz -O openvino_genai_2024.2.0.0.tgz + tar -xf openvino_genai_2024.2.0.0.tgz + + +Windows +++++++++++++++++++++++++++ + +.. code-block:: sh + + cd /Downloads + curl -L https://storage.openvinotoolkit.org/repositories/openvino_genai/packages/2024.2/windows/openvino_genai_windows_2024.2.0.0_x86_64.zip --output openvino_genai_2024.2.0.0.zip + +macOS +++++++++++++++++++++++++++ + +.. tab-set:: + + .. tab-item:: x86, 64-bit + :sync: x86-64 + + .. code-block:: sh + + curl -L https://storage.openvinotoolkit.org/repositories/openvino_genai/packages/2024.2/macos/openvino_genai_macos_12_6_2024.2.0.0_x86_64.tar.gz --output openvino_genai_2024.2.0.0.tgz + tar -xf openvino_genai_2024.2.0.0.tgz + + .. tab-item:: ARM, 64-bit + :sync: arm-64 + + .. code-block:: sh + + curl -L https://storage.openvinotoolkit.org/repositories/openvino_genai/packages/2024.2/macos/openvino_genai_macos_11_0_2024.2.0.0_arm64.tar.gz --output openvino_genai_2024.2.0.0.tgz + tar -xf openvino_genai_2024.2.0.0.tgz + + +Here are the full guides: +:doc:`Linux `, +:doc:`Windows `, and +:doc:`macOS `. + + + diff --git a/docs/articles_en/get-started/install-openvino/install-openvino-linux.rst b/docs/articles_en/get-started/install-openvino/install-openvino-linux.rst index 38561aaebae758..36df7f39ff856a 100644 --- a/docs/articles_en/get-started/install-openvino/install-openvino-linux.rst +++ b/docs/articles_en/get-started/install-openvino/install-openvino-linux.rst @@ -26,7 +26,6 @@ Install OpenVINO™ Runtime on Linux Use Docker Use npm - If you want to install OpenVINO™ Runtime on Linux, you have the following options: * :doc:`Install OpenVINO using an Archive File ` diff --git a/docs/articles_en/get-started/install-openvino/install-openvino-pip.rst b/docs/articles_en/get-started/install-openvino/install-openvino-pip.rst index ed3c1b12428ed5..191d7563f6def8 100644 --- a/docs/articles_en/get-started/install-openvino/install-openvino-pip.rst +++ b/docs/articles_en/get-started/install-openvino/install-openvino-pip.rst @@ -100,13 +100,12 @@ Use the following command: Step 4. Install the Package +++++++++++++++++++++++++++ -Use the following command: +Use the following command to install OpenVINO Base or OpenVINO GenAI package: -.. code-block:: sh +.. code-block:: python python -m pip install openvino - Step 5. Verify that the Package Is Installed ++++++++++++++++++++++++++++++++++++++++++++ diff --git a/docs/articles_en/get-started/install-openvino/install-openvino-vcpkg.rst b/docs/articles_en/get-started/install-openvino/install-openvino-vcpkg.rst index 3074bd8135e4e7..828b5205169fea 100644 --- a/docs/articles_en/get-started/install-openvino/install-openvino-vcpkg.rst +++ b/docs/articles_en/get-started/install-openvino/install-openvino-vcpkg.rst @@ -102,6 +102,12 @@ Now that you've installed OpenVINO Runtime, you can try the following things: * Learn more about :doc:`OpenVINO Workflow <../../../openvino-workflow>`. * To prepare your models for working with OpenVINO, see :doc:`Model Preparation <../../../openvino-workflow/model-preparation>`. * See pre-trained deep learning models in our :doc:`Open Model Zoo <../../../documentation/legacy-features/model-zoo>`. + + .. important:: + + Due to the deprecation of Open Model Zoo, models in the OpenVINO IR format are now + published on `Hugging Face `__. + * Learn more about :doc:`Inference with OpenVINO Runtime <../../../openvino-workflow/running-inference>`. * See sample applications in :doc:`OpenVINO toolkit Samples Overview <../../../learn-openvino/openvino-samples>`. * Check out the OpenVINO `product home page `__ . diff --git a/docs/articles_en/get-started/install-openvino/install-openvino-yocto.rst b/docs/articles_en/get-started/install-openvino/install-openvino-yocto.rst new file mode 100644 index 00000000000000..0ff1b95c8eb212 --- /dev/null +++ b/docs/articles_en/get-started/install-openvino/install-openvino-yocto.rst @@ -0,0 +1,113 @@ +Create a Yocto Image with OpenVINO™ +=================================== + +.. meta:: + :description: Learn how to create a Yocto image with OpenVINO™ toolkit on your host system. + +.. note:: + + Note that the YOCTO distribution is community-supported and has not been fully tested. + You will need to set up and configure your host machine to be compatible with BitBake. For + instruction on how to do that, follow the + `Yocto Project official documentation `__ . + + +Step 1: Set up the environment +############################## + +1. Clone the repositories. + + .. code-block:: sh + + git clone https://git.yoctoproject.org/git/poky + git clone https://git.yoctoproject.org/meta-intel + git clone https://git.openembedded.org/meta-openembedded + git clone https://github.com/kraj/meta-clang.git + +2. Set up the OpenEmbedded build environment. + + .. code-block:: sh + + source poky/oe-init-build-env + +3. Add BitBake layers. + + .. code-block:: sh + + bitbake-layers add-layer ../meta-intel + bitbake-layers add-layer ../meta-openembedded/meta-oe + bitbake-layers add-layer ../meta-openembedded/meta-python + bitbake-layers add-layer ../meta-clang + +4. Verify if the layers have been added (optional). + + .. code-block:: sh + + bitbake-layers show-layers + +5. Set up BitBake configurations. + + Include extra configuration in the `conf/local.conf` file in your build directory as required. + + .. code-block:: sh + + # Build with SSE4.2, AVX2 etc. extensions + MACHINE = "intel-skylake-64" + # Enable clDNN GPU plugin when needed. + # This requires meta-clang and meta-oe layers to be included in bblayers.conf + # and is not enabled by default. + PACKAGECONFIG:append:pn-openvino-inference-engine = " opencl" + # Enable building OpenVINO Python API. + # This requires meta-python layer to be included in bblayers.conf. + PACKAGECONFIG:append:pn-openvino-inference-engine = " python3" + # This adds OpenVINO related libraries in the target image. + CORE_IMAGE_EXTRA_INSTALL:append = " openvino-inference-engine" + # This adds OpenVINO samples in the target image. + CORE_IMAGE_EXTRA_INSTALL:append = " openvino-inference-engine-samples" + # Include OpenVINO Python API package in the target image. + CORE_IMAGE_EXTRA_INSTALL:append = " openvino-inference-engine-python3" + +Step 2: Build a Yocto Image with OpenVINO Packages +################################################## + +Run BitBake to build your image with OpenVINO packages. For example, to build the minimal image, +run the following command: + +.. code-block:: sh + + bitbake core-image-minimal + +.. note:: + For validation/testing/reviewing purposes, you may consider using the ``nohup`` command and + ensure that your vpn/ssh connection remains uninterrupted. + +Step 3: Verify the Yocto Image +############################## + +Verify that OpenVINO packages have been built successfully. Run the following command: + +.. code-block:: sh + + oe-pkgdata-util list-pkgs | grep openvino + +If the image build is successful, it will return the list of packages as below: + +.. code-block:: sh + + openvino-inference-engine + openvino-inference-engine-dbg + openvino-inference-engine-dev + openvino-inference-engine-python3 + openvino-inference-engine-samples + openvino-inference-engine-src + +Additional Resources +#################### + +- :doc:`Troubleshooting Guide <../troubleshooting-install-config>` +- `Official Yocto Project documentation `__ +- `BitBake Tool `__ +- `Poky `__ +- `Meta-intel `__ +- `Meta-openembedded `__ +- `Meta-clang `__ \ No newline at end of file diff --git a/docs/articles_en/get-started/install-openvino/install-openvino-yum.rst b/docs/articles_en/get-started/install-openvino/install-openvino-yum.rst index a5559e937e50d6..cbb25eac817f97 100644 --- a/docs/articles_en/get-started/install-openvino/install-openvino-yum.rst +++ b/docs/articles_en/get-started/install-openvino/install-openvino-yum.rst @@ -226,6 +226,12 @@ You can also try the following things: * Learn more about :doc:`OpenVINO Workflow <../../../openvino-workflow>`. * To prepare your models for working with OpenVINO, see :doc:`Model Preparation <../../../openvino-workflow/model-preparation>`. * See pre-trained deep learning models in our :doc:`Open Model Zoo <../../../documentation/legacy-features/model-zoo>`. + + .. important:: + + Due to the deprecation of Open Model Zoo, models in the OpenVINO IR format are now + published on `Hugging Face `__. + * Learn more about :doc:`Inference with OpenVINO Runtime <../../../openvino-workflow/running-inference>`. * See sample applications in :doc:`OpenVINO toolkit Samples Overview <../../../learn-openvino/openvino-samples>`. * Take a glance at the OpenVINO `product home page `__ . diff --git a/docs/articles_en/get-started/install-openvino/install-openvino-zypper.rst b/docs/articles_en/get-started/install-openvino/install-openvino-zypper.rst index 20166da049fee7..ee0b2633299c66 100644 --- a/docs/articles_en/get-started/install-openvino/install-openvino-zypper.rst +++ b/docs/articles_en/get-started/install-openvino/install-openvino-zypper.rst @@ -170,6 +170,12 @@ You can also try the following things: * Learn more about :doc:`OpenVINO Workflow <../../../openvino-workflow>`. * To prepare your models for working with OpenVINO, see :doc:`Model Preparation <../../../openvino-workflow/model-preparation>`. * See pre-trained deep learning models in our :doc:`Open Model Zoo <../../../documentation/legacy-features/model-zoo>`. + + .. important:: + + Due to the deprecation of Open Model Zoo, models in the OpenVINO IR format are now + published on `Hugging Face `__. + * Learn more about :doc:`Inference with OpenVINO Runtime <../../../openvino-workflow/running-inference>`. * See sample applications in :doc:`OpenVINO toolkit Samples Overview <../../../learn-openvino/openvino-samples>`. * Take a glance at the OpenVINO `product home page `__ . diff --git a/docs/articles_en/get-started/troubleshooting-install-config.rst b/docs/articles_en/get-started/troubleshooting-install-config.rst index 5b7bee827ad9e1..1e518609f57bd6 100644 --- a/docs/articles_en/get-started/troubleshooting-install-config.rst +++ b/docs/articles_en/get-started/troubleshooting-install-config.rst @@ -1,5 +1,3 @@ -.. {#openvino_docs_get_started_guide_troubleshooting} - Troubleshooting Guide for OpenVINO™ Installation & Configuration ================================================================ @@ -142,4 +140,18 @@ Troubleshooting Guide for OpenVINO™ Installation & Configuration For specific issues, see Errors with Installing via PIP for Users in China and Proxy issues with installing OpenVINO on Linux from Docker questions above. +.. dropdown:: a Yocto Image error when adding the "meta-intel" layer + + When using the ``bitbake-layers add-layer meta-intel`` command, the following error might + occur: + + .. code-block:: sh + + NOTE: Starting bitbake server... + ERROR: The following required tools (as specified by HOSTTOOLS) appear to be unavailable in PATH, please install them in order to proceed: chrpath diffstat pzstd zstd + + To resolve the issue, install the ``chrpath diffstat zstd`` tools: + + .. code-block:: sh + sudo apt-get install chrpath diffstat zstd diff --git a/docs/articles_en/learn-openvino.rst b/docs/articles_en/learn-openvino.rst index e955c1e6f8abbf..19e96db09bd543 100644 --- a/docs/articles_en/learn-openvino.rst +++ b/docs/articles_en/learn-openvino.rst @@ -6,7 +6,7 @@ Learn OpenVINO .. meta:: :description: Explore learning materials, including interactive - Python learn-openvino/interactive-tutorials-python and sample console applications that explain + Python tutorials and sample console applications that explain how to use OpenVINO features. @@ -26,7 +26,7 @@ accommodating different learning needs, which means you should find it useful if as well as an experienced user. | :doc:`Tutorials ` -| A collection of interactive Python learn-openvino/interactive-tutorials-python. It introduces you to the OpenVINO™ toolkit explaining how to use the Python API and tools for optimized deep learning inference. The learn-openvino/interactive-tutorials-python are available in Jupyter notebooks and can be run in your browser. No installation required. +| A collection of interactive Python tutorials. It introduces you to the OpenVINO™ toolkit explaining how to use the Python API and tools for optimized deep learning inference. The tutorials are available in Jupyter notebooks and can be run in your browser. No installation required. | :doc:`OpenVINO Samples ` | The OpenVINO samples (Python and C++) are simple console applications that show how to use specific OpenVINO API features. They can assist you in executing tasks such as loading a model, running inference, querying particular device capabilities, etc. diff --git a/docs/articles_en/learn-openvino/interactive-tutorials-python/notebooks-installation.rst b/docs/articles_en/learn-openvino/interactive-tutorials-python/notebooks-installation.rst index bda27362fbfe99..8a5fe91395912a 100644 --- a/docs/articles_en/learn-openvino/interactive-tutorials-python/notebooks-installation.rst +++ b/docs/articles_en/learn-openvino/interactive-tutorials-python/notebooks-installation.rst @@ -32,15 +32,17 @@ The table below lists the supported operating systems and Python versions. | | (64-bit | | | ) `__ | +=====================================+================================+ -| Ubuntu 18.04 LTS | 3.8, 3.9, 3.10. 3.11 | +| Ubuntu 20.04 LTS, 64-bit | 3.8, 3.9, 3.10. 3.11 | +-------------------------------------+--------------------------------+ -| Ubuntu 20.04 LTS | 3.8, 3.9, 3.10, 3.11 | +| Ubuntu 22.04 LTS, 64-bit | 3.8, 3.9, 3.10, 3.11 | +-------------------------------------+--------------------------------+ | Red Hat Enterprise Linux 8 | 3.8, 3.9, 3.10, 3.11 | +-------------------------------------+--------------------------------+ -| macOS 12.6.x versions | 3.8, 3.9, 3.10, 3.11 | +| CentOS 7, 64 bit | 3.8, 3.9, 3.10, 3.11 | +-------------------------------------+--------------------------------+ -| Windows 10 Pro, Enterprise | 3.8, 3.9, 3.10, 3.11 | +| macOS 10.15.x versions or higher | 3.8, 3.9, 3.10, 3.11 | ++-------------------------------------+--------------------------------+ +| Windows 10, 64-bit Pro, Enterprise | 3.8, 3.9, 3.10, 3.11 | | or Education editions | | +-------------------------------------+--------------------------------+ | Windows Server 2016 or higher | 3.8, 3.9, 3.10, 3.11 | @@ -64,6 +66,7 @@ Installing prerequisites Run the installer by double clicking it. Follow the installation steps to set up the software. While installing, make sure you check the box to *add Python to system PATH*. + Also, it is recommended to use the installer option to disable the PATH length limit. .. note:: @@ -81,6 +84,12 @@ Installing prerequisites Run the installer by double clicking it. Follow the installation steps to set up the software. + 4. (Optional) Install FFMPEG + + Download FFMPEG binary from `here `__ + + Set FFMPEG's path (e.g., ``C:\ffmpeg\bin``) to the PATH environmental variable on Windows. + .. tab-item:: Linux :sync: linux @@ -96,7 +105,7 @@ Installing prerequisites sudo apt-get update sudo apt-get upgrade - sudo apt-get install python3-venv build-essential python3-dev git-all + sudo apt-get install python3-venv build-essential python3-dev git-all libgl1-mesa-dev ffmpeg For an Intel Integrated Graphics Card, you can install the `Intel Graphics Compute Runtime `__ to enable inference on this device. The command for Ubuntu 20.04 is: @@ -133,7 +142,8 @@ Installing prerequisites .. code-block:: sh /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" - After you install it, follow the instructions from the Homebrew installation to set it up. + + After you install it, follow the instructions from the Homebrew installation to set it up. 3. **Install Python and dependencies** @@ -142,6 +152,8 @@ Installing prerequisites brew install python@3.9 brew install protobuf + # optional but recommended + brew install ffmpeg Run each step below in a terminal. @@ -250,6 +262,56 @@ Installing prerequisites CMD /tmp/scripts/run + .. tab-item:: Amazon SageMaker + :sync: amazon-sagemaker + + .. note:: + + An `AWS `__ + account and access to + `Amazon SageMaker Studio `__ + are required. + + 1. **Log into your Amazon SageMaker Studio Environment and** ``Add user``. + + |amazon-studio-1| + + 2. **Choose desired user profile name** + + |amazon-studio-2| + + 3. **Choose Jupyter Lab version 3.0** + + |amazon-studio-3| + + 4. **Choose the remaining default setting and click "Submit" to add a user.** + 5. **Launch the Amazon SageMaker Studio environment.** + + Click "Open Studio" to start the environment: + + |amazon-studio-4| + + .. note:: + + You are using an ``ml.t3.medium`` instance, which is for free for + 250 hours per month for the first 2 months on Studio notebook. + + 6. **Wait for a couple of minutes for your environment to load.** + + You should be able to see the following screen: + + |amazon-studio-5| + + 7. **Select a SageMaker image.** + + Choose ``Data Science 3.0`` in "Select a SageMaker image" drop-down under + "Notebooks and compute resources". + + Then, click **+** on "Image Terminal" to start a terminal session: + + |amazon-studio-6| + + Installing notebooks ++++++++++++++++++++ @@ -294,6 +356,19 @@ Installing notebooks pip install -r requirements.txt + + .. important:: + + In case of problems with accessing HuggingFace in PRC, set-up the networking + environment before you launch the notebooks: + + .. code-block:: + + pip install -U huggingface_hub + set HF_ENDPOINT = https://hf-mirror.com + + For more information, visit `HF-Mirror HuggingFace `__. + .. tab-item:: Linux :sync: linux @@ -333,6 +408,18 @@ Installing notebooks pip install -r requirements.txt + .. important:: + + In case of problems with accessing HuggingFace in PRC, set-up the networking + environment before you launch the notebooks: + + .. code-block:: + + pip install -U huggingface_hub + set HF_ENDPOINT = https://hf-mirror.com + + For more information, visit `HF-Mirror HuggingFace `__. + .. tab-item:: macOS :sync: macos @@ -475,6 +562,69 @@ Installing notebooks While running the container on Windows and macOS, only CPU devices can be used. To access the iGPU, install the notebooks locally, following the instructions above. + .. tab-item:: Amazon SageMaker + :sync: amazon-sagemaker + + + **Use the terminal and follow the steps below.** + + |amazon-studio-7| + + + 1. **Install few system dependencies.** + + .. code-block:: + + apt update + apt install build-essential -y + apt install libpython3.9-dev -y + apt install libgl1-mesa-glx -y + + 2. **Setup OpenVINO conda environment.** + + .. code-block:: + + conda create --name openvino_env python=3.9 + conda activate openvino_env + conda install ipykernel + set PATH="/anaconda/envs/openvino_env/bin;%PATH%" + + 3. **Setup OpenVINO Notebooks.** + + .. code-block:: + + git clone https://github.com/openvinotoolkit/openvino_notebooks.git + cd openvino_notebooks + # Install OpenVINO and OpenVINO notebook Requirements + python -m pip install --upgrade pip + pip install -r requirements.txt + + 4. **Run the Notebooks** + + * To run the notebooks, click the top level "openvino_notebooks" folder + and navigate to your example: + + |amazon-studio-8| + + * Choose "Image" - ``Data Science 3.0``, + "Kernel" - ``Python [conda env:openvino_env],`` + "Instance type"- your desired compute instance. + + |amazon-studio-9| + + |amazon-studio-10| + + |amazon-studio-11| + + .. note:: + + Make sure you use the ``Python [conda env:openvino_env]`` + environment (not ``Python 3``). + + * Next, run the cells of the notebook. You may try other notebooks to + explore OpenVINO features and examples. + + Run the Notebooks ################# @@ -614,6 +764,27 @@ Additional Resources .. |ml-studio-2| image:: https://user-images.githubusercontent.com/15709723/117582205-b6f4d580-b0b5-11eb-9b83-eb2004ad9b19.png -.. |docker-terminal-1| image:: https://user-images.githubusercontent.com/15709723/127793994-355e4d29-d131-432d-a12a-b08ca6131223.png +.. |amazon-studio-1| image:: https://user-images.githubusercontent.com/4837253/199801883-7bb64ad2-bb7f-4477-ace1-25111d4fd43c.png + +.. |amazon-studio-2| image:: https://user-images.githubusercontent.com/4837253/199802173-8d65c851-604b-4b92-bafa-cae86b17d1ec.png +.. |amazon-studio-3| image:: https://user-images.githubusercontent.com/4837253/199802353-14c17233-3dae-4649-bbfe-59b8a598450c.png + +.. |amazon-studio-4| image:: https://user-images.githubusercontent.com/4837253/199802726-97c85732-ff25-4cdd-ad6e-d491b4ed122b.png + +.. |amazon-studio-5| image:: https://user-images.githubusercontent.com/15709723/199784252-c8581c73-342a-4c70-9207-5543d7b87346.png + +.. |amazon-studio-6| image:: https://user-images.githubusercontent.com/4837253/199805717-5d102d27-e92e-4426-8d14-0484fd5ba24c.png + +.. |amazon-studio-7| image:: https://user-images.githubusercontent.com/4837253/199807022-3cc5dd9e-f9f0-445d-be5e-d429dc1b752c.png + +.. |amazon-studio-8| image:: https://user-images.githubusercontent.com/4837253/199810405-0f6748e1-d5f5-469e-8305-a96724dfffba.png + +.. |amazon-studio-9| image:: https://user-images.githubusercontent.com/4837253/199812540-c52ea429-9d53-4bdb-aec1-a0b8616c6fcc.png + +.. |amazon-studio-10| image:: https://user-images.githubusercontent.com/4837253/199812587-20c3e360-3a31-4032-b17a-8b242d6ccc26.png + +.. |amazon-studio-11| image:: https://user-images.githubusercontent.com/4837253/199812713-32074aa7-8190-43c8-815c-231542c7b286.png + +.. |docker-terminal-1| image:: https://user-images.githubusercontent.com/15709723/127793994-355e4d29-d131-432d-a12a-b08ca6131223.png diff --git a/docs/articles_en/learn-openvino/llm_inference_guide.rst b/docs/articles_en/learn-openvino/llm_inference_guide.rst index b26c3adc74da22..6db776a3c1f5fb 100644 --- a/docs/articles_en/learn-openvino/llm_inference_guide.rst +++ b/docs/articles_en/learn-openvino/llm_inference_guide.rst @@ -13,8 +13,9 @@ Large Language Model Inference Guide :maxdepth: 1 :hidden: - LLM Inference with Optimum Intel - LLM Inference with OpenVINO API + Run LLMs with Optimum Intel + Run LLMs with OpenVINO GenAI Flavor + Run LLMs with Base OpenVINO OpenVINO Tokenizers Large Language Models (LLMs) like GPT are transformative deep learning networks capable of a @@ -25,26 +26,44 @@ conversion to advanced use cases. The advantages of using OpenVINO for LLM deployment: -* **OpenVINO offers optimized LLM inference**; provides a full C/C++ API, leading to faster operation than Python-based runtimes; includes a Python API for rapid development, with the option for further optimization in C++. - -* **Compatible with diverse hardware**, supports CPUs, GPUs, and neural accelerators across ARM and x86/x64 architectures, integrated Intel® Processor Graphics, discrete Intel® Arc™ A-Series Graphics, and discrete Intel® Data Center GPU Flex Series; features automated optimization to maximize performance on target hardware. - -* **Requires fewer dependencies** than frameworks like Hugging Face and PyTorch, resulting in a smaller binary size and reduced memory footprint, making deployments easier and updates more manageable. - -* **Provides compression and precision management techniques** such as 8-bit and 4-bit weight compression, including embedding layers, and storage format reduction. This includes fp16 precision for non-compressed models and int8/int4 for compressed models, like GPTQ models from Hugging Face. - -* **Supports a wide range of deep learning models and architectures** including text, image, and audio generative models like Llama 2, MPT, OPT, Stable Diffusion, Stable Diffusion XL. This enables the development of multimodal applications, allowing for write-once, deploy-anywhere capabilities. - -* **Enhances inference capabilities**: fused inference primitives such as Scaled Dot Product Attention, Rotary Positional Embedding, Group Query Attention, and Mixture of Experts. It also offers advanced features like in-place KV-cache, dynamic quantization, KV-cache quantization and encapsulation, dynamic beam size configuration, and speculative sampling. - -* **Provides stateful model optimization**: models from the Hugging Face Transformers are converted into a stateful form, optimizing inference performance and memory usage in long-running text generation tasks by managing past KV-cache tensors more efficiently internally. This feature is automatically activated for many supported models, while unsupported ones remain stateless. Learn more about the :doc:`Stateful models and State API <../openvino-workflow/running-inference/stateful-models>`. - -OpenVINO offers two main paths for Generative AI use cases: +* **OpenVINO offers optimized LLM inference**: + provides a full C/C++ API, leading to faster operation than Python-based runtimes; includes a + Python API for rapid development, with the option for further optimization in C++. +* **Compatible with diverse hardware**: + supports CPUs, GPUs, and neural accelerators across ARM and x86/x64 architectures, integrated + Intel® Processor Graphics, discrete Intel® Arc™ A-Series Graphics, and discrete Intel® Data + Center GPU Flex Series; features automated optimization to maximize performance on target + hardware. +* **Requires fewer dependencies**: + than frameworks like Hugging Face and PyTorch, resulting in a smaller binary size and reduced + memory footprint, making deployments easier and updates more manageable. +* **Provides compression and precision management techniques**: + such as 8-bit and 4-bit weight compression, including embedding layers, and storage format + reduction. This includes fp16 precision for non-compressed models and int8/int4 for compressed + models, like GPTQ models from `Hugging Face `__. +* **Supports a wide range of deep learning models and architectures**: + including text, image, and audio generative models like Llama 2, MPT, OPT, Stable Diffusion, + Stable Diffusion XL. This enables the development of multimodal applications, allowing for + write-once, deploy-anywhere capabilities. +* **Enhances inference capabilities**: + fused inference primitives such as Scaled Dot Product Attention, Rotary Positional Embedding, + Group Query Attention, and Mixture of Experts. It also offers advanced features like in-place + KV-cache, dynamic quantization, KV-cache quantization and encapsulation, dynamic beam size + configuration, and speculative sampling. +* **Provides stateful model optimization**: + models from the Hugging Face Transformers are converted into a stateful form, optimizing + inference performance and memory usage in long-running text generation tasks by managing past + KV-cache tensors more efficiently internally. This feature is automatically activated for many + supported models, while unsupported ones remain stateless. Learn more about the + :doc:`Stateful models and State API <../openvino-workflow/running-inference/stateful-models>`. + +OpenVINO offers three main paths for Generative AI use cases: * **Hugging Face**: use OpenVINO as a backend for Hugging Face frameworks (transformers, diffusers) through the `Optimum Intel `__ extension. -* **Native OpenVINO**: use OpenVINO native APIs (Python and C++) with +* **OpenVINO GenAI Flavor**: use OpenVINO GenAI APIs (Python and C++). +* **Base OpenVINO**: use OpenVINO native APIs (Python and C++) with `custom pipeline code `__. In both cases, the OpenVINO runtime is used for inference, and OpenVINO tools are used for @@ -54,23 +73,23 @@ The Hugging Face API is easy to learn, provides a simple interface and hides the model initialization and text generation for a better developer experience. However, it has more dependencies, less customization, and cannot be ported to C/C++. -The Native OpenVINO API requires fewer dependencies, minimizing the application footprint, and -enables the use of generative models in C++ applications. However, it requires explicit -implementation of the text generation loop, tokenization functions, and scheduler functions used -in a typical LLM pipeline. +The OpenVINO GenAI Flavor reduces the complexity of LLMs implementation by +automatically managing essential tasks like the text generation loop, tokenization, +and scheduling. The Native OpenVINO API provides a more hands-on experience, +requiring manual setup of these functions. Both methods are designed to minimize dependencies +and the overall application footprint and enable the use of generative models in C++ applications. It is recommended to start with Hugging Face frameworks to experiment with different models and -scenarios. Then the model can be used with OpenVINO native APIs if it needs to be optimized +scenarios. Then the model can be used with OpenVINO APIs if it needs to be optimized further. Optimum Intel provides interfaces that enable model optimization (weight compression) using `Neural Network Compression Framework (NNCF) `__, and export models to the OpenVINO model format for use in native API applications. -* To proceed with **Hugging Face API**, read - :doc:`LLM Inference with Hugging Face and Optimum Intel <./llm_inference_guide/llm-inference-hf>` - guide. -* To proceed with **Native OpenVINO API**, go to - :doc:`LLM Inference with Native OpenVINO <./llm_inference_guide/llm-inference-native-ov>` - page. +Proceed to run LLMs with: + +* :doc:`Hugging Face and Optimum Intel <./llm_inference_guide/llm-inference-hf>` +* :doc:`OpenVINO GenAI Flavor <./llm_inference_guide/genai-guide>` +* :doc:`Native OpenVINO API <./llm_inference_guide/llm-inference-native-ov>` The table below summarizes the differences between Hugging Face and the native OpenVINO API approaches. @@ -119,4 +138,5 @@ approaches. - Best suited for high-performance, resource-optimized production environments * - Model Serving - Paid service, based on CPU/GPU usage with Hugging Face - - Free code solution, run script for own server; costs may incur for cloud services like AWS but generally cheaper than Hugging Face rates + - Free code solution, run script for own server; costs may incur for cloud services + like AWS but generally cheaper than Hugging Face rates diff --git a/docs/articles_en/learn-openvino/llm_inference_guide/genai-guide.rst b/docs/articles_en/learn-openvino/llm_inference_guide/genai-guide.rst new file mode 100644 index 00000000000000..821f16062c271b --- /dev/null +++ b/docs/articles_en/learn-openvino/llm_inference_guide/genai-guide.rst @@ -0,0 +1,288 @@ +Run LLMs with OpenVINO GenAI Flavor +===================================== + +.. meta:: + :description: Learn how to use the OpenVINO GenAI flavor to execute LLM models. + +This guide will show you how to integrate the OpenVINO GenAI flavor into your application, covering +loading a model and passing the input context to receive generated text. Note that the vanilla flavor of OpenVINO +will not work with these instructions, make sure to +:doc:`install OpenVINO GenAI <../../get-started/install-openvino/install-openvino-genai>`. + +.. note:: + + The examples use the CPU as the target device, however, the GPU is also supported. + Note that for the LLM pipeline, the GPU is used only for inference, while token selection, tokenization, and + detokenization remain on the CPU, for efficiency. Tokenizers are represented as a separate model and also run + on the CPU. + +1. Export an LLM model via Hugging Face Optimum-Intel. A chat-tuned TinyLlama model is used in this example: + + .. code-block:: python + + optimum-cli export openvino --model "TinyLlama/TinyLlama-1.1B-Chat-v1.0" --weight-format fp16 --trust-remote-code "TinyLlama-1.1B-Chat-v1.0" + + *Optional*. Optimize the model: + + The model is an optimized OpenVINO IR with FP16 precision. For enhanced LLM performance, + it is recommended to use lower precision for model weights, such as INT4, and to compress weights + using NNCF during model export directly: + + .. code-block:: python + + optimum-cli export openvino --model "TinyLlama/TinyLlama-1.1B-Chat-v1.0" --weight-format int4 --trust-remote-code + +2. Perform generation using the new GenAI API: + + .. tab-set:: + + .. tab-item:: Python + :sync: py + + .. code-block:: python + + import openvino_genai as ov_genai + pipe = ov_genai.LLMPipeline(model_path, "CPU") + print(pipe.generate("The Sun is yellow because")) + + .. tab-item:: C++ + :sync: cpp + + .. code-block:: cpp + + #include "openvino/genai/llm_pipeline.hpp" + #include + + int main(int argc, char* argv[]) { + std::string model_path = argv[1]; + ov::genai::LLMPipeline pipe(model_path, "CPU");//target device is CPU + std::cout << pipe.generate("The Sun is yellow because"); //input context + +The `LLMPipeline` is the main object used for decoding. You can construct it directly from the +folder with the converted model. It will automatically load the main model, tokenizer, detokenizer, +and the default generation configuration. + +Once the model is exported from Hugging Face Optimum-Intel, it already contains all the information +necessary for execution, including the tokenizer/detokenizer and the generation config, ensuring that +its results match those generated by Hugging Face. + +Streaming the Output +########################### + +For more interactive UIs during generation, streaming of model output tokens is supported. See the example +below, where a lambda function outputs words to the console immediately upon generation: + +.. tab-set:: + + .. tab-item:: C++ + + .. code-block:: cpp + + #include "openvino/genai/llm_pipeline.hpp" + #include + + int main(int argc, char* argv[]) { + std::string model_path = argv[1]; + ov::genai::LLMPipeline pipe(model_path, "CPU"); + + auto streamer = [](std::string word) { std::cout << word << std::flush; }; + std::cout << pipe.generate("The Sun is yellow because", streamer); + } + +You can also create your custom streamer for more sophisticated processing: + +.. tab-set:: + + .. tab-item:: C++ + + .. code-block:: cpp + + #include + + class CustomStreamer: publict StreamerBase { + public: + void put(int64_t token) {/* decode tokens and do process them*/}; + + void end() {/* decode tokens and do process them*/}; + }; + + int main(int argc, char* argv[]) { + CustomStreamer custom_streamer; + + std::string model_path = argv[1]; + ov::LLMPipeline pipe(model_path, "CPU"); + cout << pipe.generate("The Sun is yellow because", custom_streamer); + } + +Optimizing the Chat Scenario +################################ + +For chat scenarios where inputs and outputs represent a conversation, maintaining KVCache across inputs +may prove beneficial. The chat-specific methods **start_chat** and **finish_chat** are used to +mark a conversation session, as you can see in these simple examples: + +.. tab-set:: + + .. tab-item:: Python + :sync: py + + .. code-block:: python + + import openvino_genai as ov_genai + pipe = ov_genai.LLMPipeline(model_path) + + config = {'num_groups': 3, 'group_size': 5, 'diversity_penalty': 1.1} + pipe.set_generation_cofnig(config) + + pipe.start_chat() + while True: +     print('question:') +     prompt = input() + if prompt == 'Stop!': +         break +     print(pipe(prompt)) + pipe.finish_chat() + + + .. tab-item:: C++ + :sync: cpp + + .. code-block:: cpp + + int main(int argc, char* argv[]) { + std::string prompt; + + std::string model_path = argv[1]; + ov::LLMPipeline pipe(model_path, "CPU"); + + pipe.start_chat(); + for (size_t i = 0; i < questions.size(); i++) { + std::cout << "question:\n"; + std::getline(std::cin, prompt); + + std::cout << pipe(prompt) << std::endl>>; + } + pipe.finish_chat(); + } + +Optimizing Generation with Grouped Beam Search +####################################################### + +Leverage grouped beam search decoding and configure generation_config for better text generation +quality and efficient batch processing in GenAI applications. + +Use grouped beam search decoding: + +.. tab-set:: + + .. tab-item:: C++ + + .. code-block:: cpp + + int main(int argc, char* argv[]) { + std::string model_path = argv[1]; + ov::LLMPipeline pipe(model_path, "CPU"); + ov::GenerationConfig config = pipe.get_generation_config(); + config.max_new_tokens = 256; + config.num_groups = 3; + config.group_size = 5; + config.diversity_penalty = 1.0f; + + cout << pipe.generate("The Sun is yellow because", config); + } + +Specify generation_config to use grouped beam search: + +.. tab-set:: + + .. tab-item:: C++ + + .. code-block:: cpp + + int main(int argc, char* argv[]) { + std::string prompt; + + std::string model_path = argv[1]; + ov::LLMPipeline pipe(model_path, "CPU"); + + ov::GenerationConfig config = pipe.get_generation_config(); + config.max_new_tokens = 256; + config.num_groups = 3; + config.group_size = 5; + config.diversity_penalty = 1.0f; + + auto streamer = [](std::string word) { std::cout << word << std::flush; }; + + pipe.start_chat(); + for (size_t i = 0; i < questions.size(); i++) { + + std::cout << "question:\n"; + cout << prompt << endl; + + auto answer = pipe(prompt, config, streamer); + // no need to print answer, streamer will do that + } + pipe.finish_chat(); + } + +Comparing with Hugging Face Results +####################################### + +Compare and analyze results with those generated by Hugging Face models. + +.. tab-set:: + + .. tab-item:: Python + + .. code-block:: python + + from transformers import AutoTokenizer, AutoModelForCausalLM + + tokenizer = AutoTokenizer.from_pretrained("TinyLlama/TinyLlama-1.1B-Chat-v1.0") + model = AutoModelForCausalLM.from_pretrained("TinyLlama/TinyLlama-1.1B-Chat-v1.0") + + max_new_tokens = 32 + prompt = 'table is made of' + + encoded_prompt = tokenizer.encode(prompt, return_tensors='pt', add_special_tokens=False) + hf_encoded_output = model.generate(encoded_prompt, max_new_tokens=max_new_tokens, do_sample=False) + hf_output = tokenizer.decode(hf_encoded_output[0, encoded_prompt.shape[1]:]) + print(f'hf_output: {hf_output}') + + import sys + sys.path.append('build-Debug/') + import py_generate_pipeline as genai # set more friendly module name + + pipe = genai.LLMPipeline('text_generation/causal_lm/TinyLlama-1.1B-Chat-v1.0/pytorch/dldt/FP16/') + ov_output = pipe(prompt, max_new_tokens=max_new_tokens) + print(f'ov_output: {ov_output}') + + assert hf_output == ov_output + +GenAI API +####################################### + +OpenVINO GenAI Flavor includes the following API: + +* generation_config - defines a configuration class for text generation, enabling customization of the generation process such as the maximum length of the generated text, whether to ignore end-of-sentence tokens, and the specifics of the decoding strategy (greedy, beam search, or multinomial sampling). + +* llm_pipeline - provides classes and utilities for text generation, including a pipeline for processing inputs, generating text, and managing outputs with configurable options. + +* streamer_base - an abstract base class for creating streamers. + +* tokenizer - the tokenizer class for text encoding and decoding. + +* visibility - controls the visibility of the GenAI library. + +Learn more about API in the `GenAI repository `__. + +Additional Resources +#################### + +* `OpenVINO GenAI Repo `__ +* `OpenVINO GenAI Samples `__ +* `OpenVINO Tokenizers `__ +* `Neural Network Compression Framework `__ + + + diff --git a/docs/articles_en/learn-openvino/llm_inference_guide/llm-inference-hf.rst b/docs/articles_en/learn-openvino/llm_inference_guide/llm-inference-hf.rst index aa5ad1066b5d6c..4ee10dd9b10637 100644 --- a/docs/articles_en/learn-openvino/llm_inference_guide/llm-inference-hf.rst +++ b/docs/articles_en/learn-openvino/llm_inference_guide/llm-inference-hf.rst @@ -1,16 +1,18 @@ .. {#llm_inference} -Inference with Hugging Face and Optimum Intel +Run LLMs with Hugging Face and Optimum Intel ===================================================== -The steps below show how to load and infer LLMs from Hugging Face using Optimum Intel. -They also show how to convert models into OpenVINO IR format so they can be optimized -by NNCF and used with other OpenVINO tools. +The steps below show how to load and infer LLMs from +`Hugging Face `__ using Optimum Intel. They also show how to +convert models into OpenVINO IR format so they can be optimized by NNCF and used with +other OpenVINO tools. Prerequisites ############################################################ -* Create a Python environment by following the instructions on the :doc:`Install OpenVINO PIP <../../get-started/install-openvino>` page. +* Create a Python environment by following the instructions on the + :doc:`Install OpenVINO PIP <../../get-started/install-openvino>` page. * Install the necessary dependencies for Optimum Intel: .. code-block:: console @@ -20,7 +22,8 @@ Prerequisites Loading a Hugging Face Model to Optimum Intel ############################################################ -To start using OpenVINO as a backend for Hugging Face, change the original Hugging Face code in two places: +To start using OpenVINO as a backend for Hugging Face, change the original Hugging Face +code in two places: .. code-block:: diff @@ -225,10 +228,15 @@ OpenVINO runtime provides a set of optimizations for more efficient LLM inferenc includes **Dynamic quantization** of activations of 4/8-bit quantized MatMuls and **KV-cache quantization**. -* **Dynamic quantization** enables quantization of activations of MatMul operations that have 4 or 8-bit quantized weights (see :doc:`LLM Weight Compression <../../openvino-workflow/model-optimization-guide/weight-compression>`). - It improves inference latency and throughput of LLMs, though it may cause insignificant deviation in generation accuracy. Quantization is performed in a - group-wise manner, with configurable group size. It means that values in a group share quantization parameters. Larger group sizes lead to faster inference but lower accuracy. Recommended group size values are ``32``, ``64``, or ``128``. To enable Dynamic quantization, use the corresponding - inference property as follows: +* **Dynamic quantization** enables quantization of activations of MatMul operations + that have 4 or 8-bit quantized weights (see + :doc:`LLM Weight Compression <../../openvino-workflow/model-optimization-guide/weight-compression>`). + It improves inference latency and throughput of LLMs, though it may cause + insignificant deviation in generation accuracy. Quantization is performed in a group-wise + manner, with configurable group size. It means that values in a group share quantization + parameters. Larger group sizes lead to faster inference but lower accuracy. Recommended + group size values are ``32``, ``64``, or ``128``. To enable Dynamic quantization, use + the corresponding inference property as follows: .. code-block:: python @@ -238,9 +246,12 @@ includes **Dynamic quantization** of activations of 4/8-bit quantized MatMuls an ov_config={"DYNAMIC_QUANTIZATION_GROUP_SIZE": "32", "PERFORMANCE_HINT": "LATENCY"} ) -* **KV-cache quantization** allows lowering the precision of Key and Value cache in LLMs. This helps reduce memory consumption during inference, improving latency and throughput. KV-cache can be quantized into the following precisions: - ``u8``, ``bf16``, ``f16``. If ``u8`` is used, KV-cache quantization is also applied in a group-wise manner. Thus, it can use ``DYNAMIC_QUANTIZATION_GROUP_SIZE`` value if defined. - Otherwise, the group size ``32`` is used by default. KV-cache quantization can be enabled as follows: +* **KV-cache quantization** allows lowering the precision of Key and Value cache in LLMs. + This helps reduce memory consumption during inference, improving latency and throughput. + KV-cache can be quantized into the following precisions: ``u8``, ``bf16``, ``f16``. + If ``u8`` is used, KV-cache quantization is also applied in a group-wise manner. Thus, + it can use ``DYNAMIC_QUANTIZATION_GROUP_SIZE`` value if defined. Otherwise, the group + size ``32`` is used by default. KV-cache quantization can be enabled as follows: .. code-block:: python diff --git a/docs/articles_en/learn-openvino/llm_inference_guide/llm-inference-native-ov.rst b/docs/articles_en/learn-openvino/llm_inference_guide/llm-inference-native-ov.rst index 33b1310fe65a42..c1923e543e3b64 100644 --- a/docs/articles_en/learn-openvino/llm_inference_guide/llm-inference-native-ov.rst +++ b/docs/articles_en/learn-openvino/llm_inference_guide/llm-inference-native-ov.rst @@ -1,31 +1,43 @@ .. {#llm_inference_native_ov} -Inference with Native OpenVINO +Run LLMs with Base OpenVINO =============================== -To run Generative AI models using native OpenVINO APIs you need to follow regular **Convert -> Optimize -> Deploy** path with a few simplifications. +To run Generative AI models using native OpenVINO APIs you need to follow regular +**Convert -> Optimize -> Deploy** path with a few simplifications. -To convert a model from Hugging Face, you can use Optimum-Intel export feature that allows you to export model in the OpenVINO format without invoking conversion API and tools directly. -In this case, the conversion process is a bit more simplified. You can still use a regular conversion path if the model comes from outside of Hugging Face ecosystem, i.e., in source framework format (PyTorch, etc.) +To convert a model from `Hugging Face `__, you can use +Optimum-Intel export feature that allows you to export model in the OpenVINO format without +invoking conversion API and tools directly. In this case, the conversion process is a bit +more simplified. You can still use a regular conversion path if the model comes from +outside of Hugging Face ecosystem, i.e., in source framework format (PyTorch, etc.) -Model optimization can be performed within Hugging Face or directly using NNCF as described in :doc:`Weight Compression <../../openvino-workflow/model-optimization-guide/weight-compression>`. +Model optimization can be performed within Hugging Face or directly using NNCF as described in +:doc:`Weight Compression <../../openvino-workflow/model-optimization-guide/weight-compression>`. .. note:: - It is recommended to use models in 4-bit precision, as maintaining the model in its original precision may result in significantly decreased performance. + It is recommended to use models in 4-bit precision, as maintaining the model in its + original precision may result in significantly decreased performance. -Inference code that uses native API cannot benefit from Hugging Face pipelines. You need to write your custom code or take it from the available examples. Below are some examples of popular Generative AI scenarios: +Inference code that uses native API cannot benefit from Hugging Face pipelines. +You need to write your custom code or take it from the available examples. Below are +some examples of popular Generative AI scenarios: -* In case of LLMs for text generation, you need to handle tokenization, inference and token selection loop, and de-tokenization. If token selection involves beam search, it also needs to be written. -* For image generation models, you need to make a pipeline that includes several model inferences: inference for source (e.g., text) encoder models, inference loop for diffusion process and inference for the decoding part. Scheduler code is also required. +* In case of LLMs for text generation, you need to handle tokenization, inference and + token selection loop, and de-tokenization. If token selection involves beam search, + it also needs to be written. +* For image generation models, you need to make a pipeline that includes several model + inferences: inference for source (e.g., text) encoder models, inference loop for + diffusion process and inference for the decoding part. Scheduler code is also required. To write such pipelines, you can follow the examples provided as part of OpenVINO: -* `Text generation C++ samples that support most popular models like LLaMA 2 `__ * `OpenVINO Latent Consistency Model C++ image generation pipeline `__ * `OpenVINO Stable Diffusion (with LoRA) C++ image generation pipeline `__ -To perform inference, models must be first converted to OpenVINO IR format using Hugging Face Optimum-Intel API. +To perform inference, models must be first converted to OpenVINO IR format using +Hugging Face Optimum-Intel API. An inference pipeline for a text generation LLM is set up in the following stages: @@ -174,11 +186,9 @@ This step is essential for interpreting the model's output. Additional Resources #################### -* `Text generation C++ samples that support most popular models like LLaMA 2 `__ * `OpenVINO GenAI Repo `__ * `OpenVINO Tokenizers `__ * `Neural Network Compression Framework `__ * :doc:`Stateful Models Low-Level Details <../../openvino-workflow/running-inference/stateful-models>` * :doc:`Working with Textual Data <../../openvino-workflow/running-inference/string-tensors>` - diff --git a/docs/articles_en/learn-openvino/llm_inference_guide/ov-tokenizers.rst b/docs/articles_en/learn-openvino/llm_inference_guide/ov-tokenizers.rst index 571743701ce01a..d670c9940b707c 100644 --- a/docs/articles_en/learn-openvino/llm_inference_guide/ov-tokenizers.rst +++ b/docs/articles_en/learn-openvino/llm_inference_guide/ov-tokenizers.rst @@ -338,7 +338,7 @@ Additional Resources * `OpenVINO Tokenizers repo `__ * `OpenVINO Tokenizers Notebook `__ -* `Text generation C++ samples that support most popular models like LLaMA 2 `__ +* `Text generation C++ samples that support most popular models like LLaMA 2 `__ * `OpenVINO GenAI Repo `__ diff --git a/docs/articles_en/openvino-workflow/deployment-locally.rst b/docs/articles_en/openvino-workflow/deployment-locally.rst index bc431d1233108f..53eb9a8829eb2f 100644 --- a/docs/articles_en/openvino-workflow/deployment-locally.rst +++ b/docs/articles_en/openvino-workflow/deployment-locally.rst @@ -6,8 +6,10 @@ Deploy Locally :maxdepth: 1 :hidden: - Local Distribution Libraries - Optimize Binaries Size + Local Distribution Libraries <./deployment-locally/local-distribution-libraries> + Optimize Binaries Size <./deployment-locally/optimial-binary-size-conditional-compilation> + Integrate OpenVINO with Ubuntu Snap <./deployment-locally/integrate-openvino-with-ubuntu-snap> + .. meta:: :description: There are several ways of deploying OpenVINO™ application once diff --git a/docs/articles_en/openvino-workflow/deployment-locally/integrate-openvino-with-ubuntu-snap.rst b/docs/articles_en/openvino-workflow/deployment-locally/integrate-openvino-with-ubuntu-snap.rst new file mode 100644 index 00000000000000..c47fe7f17d84ed --- /dev/null +++ b/docs/articles_en/openvino-workflow/deployment-locally/integrate-openvino-with-ubuntu-snap.rst @@ -0,0 +1,216 @@ +OpenVINO Integration with Ubuntu Snap +============================================= + +A snap is a way to containerize applications and embed them on Linux devices. Currently, OpenVINO +supports this form of deployment for Ubuntu. Building a snap package involves several steps, +including setting up your development environment, creating the necessary configuration files, and +`using the snapcraft tool to build the Snap `__. +This article will show you how to integrate OpenVINO toolkit with your application snap: + +* `Method 1: User Application Snap based on OpenVINO Sources <#method-1-user-application-snap-based-on-openvino-sources>`__ +* `Method 2: Separate OpenVINO and User Application Snaps <#method-2-separate-openvino-and-user-application-snaps>`__ +* `Method 3: User Application Snap based on OpenVINO Debian Packages (Recommended) <#method-3-recommended-user-application-snap-based-on-openvino-debian-packages>`__ + + + +Method 1: User Application Snap based on OpenVINO Sources +######################################################### + +OpenVINO libraries can be built using the CMake plugin (https://snapcraft.io/docs/cmake-plugin). +To build and install it to the application snap image, you need to configure the new part in +the application snapcraft.yaml: + +.. code-block:: sh + + openvino-build: + plugin: cmake + source-type: git + source: https://github.com/openvino.git + source-branch: master + cmake-generator: Ninja + cmake-parameters: + - -DENABLE_SAMPLES:BOOL=OFF + - -DENABLE_TESTS:BOOL=OFF + build-environment: + - CMAKE_BUILD_PARALLEL_LEVEL: ${SNAPCRAFT_PARALLEL_BUILD_COUNT} + - CMAKE_BUILD_TYPE: Release + build-packages: + - build-essential + - ninja-build + - pkg-config + - gzip + + + +Method 2: Separate OpenVINO and User Application Snaps +###################################################### + +This approach means that OpenVINO libraries and user applications will be distributed as +separate snaps. It involves three steps: + +1. Configure the OpenVINO snapcraft.yaml. + + Add the part to build and install OpenVINO as described in the 1st Method: + + .. code-block:: sh + + openvino-build: + plugin: cmake + source-type: git + source: https://github.com/openvino.git + source-branch: master + cmake-generator: Ninja + cmake-parameters: + - -DENABLE_SAMPLES:BOOL=OFF + - -DENABLE_TESTS:BOOL=OFF + build-environment: + - CMAKE_BUILD_PARALLEL_LEVEL: ${SNAPCRAFT_PARALLEL_BUILD_COUNT} + - CMAKE_BUILD_TYPE: Release + build-packages: + - build-essential + - ninja-build + - pkg-config + - gzip + + Define the slots provided by the OpenVINO Snap. Slots are the interfaces your Snap + exposes for other Snaps to connect to: + + .. code-block:: sh + + slots: + openvino-libs: + interface: content + content: openvino-libs + read: + - $SNAP/usr/local/li + openvino-3rdparty-libs: + interface: content + content: openvino-extra-libs + read: + - $SNAP/usr/local/runtime/3rdparty/tbb/lib + +2. Configure the application's snapcraft.yaml: + + Edit your snapcraft.yaml file to include OpenVINO plugs. + + .. code-block:: sh + + plugs: + openvino-libs: + interface: content + content: openvino-libs + target: $SNAP/openvino-libs + default-provider: openvino-libs-test + + openvino-3rdparty-libs: + interface: content + content: openvino-extra-libs + target: $SNAP/openvino-extra-libs + default-provider: openvino-libs-test + + Add OpenVINO snap to build-snaps: + + .. code-block:: sh + + parts: + app-build: + build-snaps: + - openvino-libs-test + + Set the OpenVINO environment in the build part: + + .. code-block:: sh + + parts: + app-build: + build-environment: + - OpenVINO_DIR: /snap/openvino-libs/current/usr/local/runtime/cmake + - LD_LIBRARY_PATH: $LD_LIBRARY_PATH:/snap/openvino-libs/current/usr/local/runtime/3rdparty/tbb/lib + + + Set the OpenVINO environment in the apps section: + + .. code-block:: sh + + apps: + app: + command: usr/local/app + environment: + LD_LIBRARY_PATH: $LD_LIBRARY_PATH:$SNAP/openvino-libs:$SNAP/openvino-extra-libs + +3. Install snaps and Connect plugs. Snaps can be connected automatically only if they are + published by the same user, otherwise you need to manually connect Application plugs with + OpenVINO slots after installation: + + .. code-block:: sh + + snap connect app:openvino-libs openvino-libs:openvino-libs + snap connect app:openvino-3rdparty-libs openvino-libs:openvino-3rdparty-libs + + +Method 3 (Recommended): User Application Snap based on OpenVINO Debian Packages +############################################################################### + +OpenVINO toolkit is also distributed via the +`APT repository `__, +which can be used in the snaps. Third-party apt repositories can be added to the snap's +snapcraft.yaml (`see the snapcraft guide `__). + +1. Download the `GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB `__: + + .. code-block:: sh + + wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB + +2. To determine a key-id from a given key file with gpg, type the following: + + .. code-block:: sh + + gpg --show-keys ./GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB + + pub rsa2048 2019-07-26 [SC] [expired: 2023-07-26] + E1BA4ECEFB0656C61BF9794936B9569B3F1A1BC7 + uid KEY-PIDT-PGP-20190726 + + pub rsa2048 2020-05-18 [SC] [expires: 2024-05-18] + 6113D31362A0D280FC025AAB640736427872A220 + uid CN=Intel(R) Software Development Products (PREPROD USE ONLY) + + pub rsa2048 2023-08-21 [SC] [expires: 2027-08-21] + E9BF0AFC46D6E8B7DA5882F1BAC6F0C353D04109 + uid CN=Intel(R) Software Development Products + +3. Export GPG key to asc file: + + .. code-block:: sh + + gpg --armor --export E9BF0AFC46D6E8B7DA5882F1BAC6F0C353D04109./GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB > 53D04109.acs + + where the name of the .asc file is `the last 8 symbols `__ + +4. Save this key in ``/snap/keys/folder``. Snapcraft will install the corresponding key. + +5. Then, the OpenVINO apt repositoriy can be added to the snap's snapcraft.yaml by using the + top-level package repositories keyword with a deb-type repository: + + .. code-block:: sh + + package-repositories: + - type: apt + components: [main] + suites: [ubuntu20] + key-id: E9BF0AFC46D6E8B7DA5882F1BAC6F0C353D04109 + url: https://apt.repos.intel.com/openvino/2024 + +6. Add OpenVINO dep packages to build-packages and stage-packages dependencies: + + .. code-block:: sh + + parts: + app-build: + build-packages: + - openvino-libraries-dev + stage-packages: + - openvino-libraries-2024.1.0 + +7. Build User Application diff --git a/docs/articles_en/openvino-workflow/deployment-locally/optimial-binary-size-conditional-compilation.rst b/docs/articles_en/openvino-workflow/deployment-locally/optimial-binary-size-conditional-compilation.rst index ed7604117a1a14..93adbf17b044f7 100644 --- a/docs/articles_en/openvino-workflow/deployment-locally/optimial-binary-size-conditional-compilation.rst +++ b/docs/articles_en/openvino-workflow/deployment-locally/optimial-binary-size-conditional-compilation.rst @@ -241,7 +241,7 @@ Build OpenVINO with conditional compilation enabled: cd %OPENVINO_HOME% md build_cc cd build_cc - cmake -G Ninja -Wno-dev -DCMAKE_BUILD_TYPE=Debug -DENABLE_CPPLINT=OFF -DCMAKE_VERBOSE_MAKEFILE=ON -DCMAKE_COMPILE_WARNING_AS_ERROR=OFF -DENABLE_FASTER_BUILD=ON -DENABLE_SANITIZER=OFF -DTHREADING=TBB -DBUILD_SHARED_LIBS=OFF -DENABLE_PROFILING_ITT=ON -DSELECTIVE_BUILD=COLLECT -DENABLE_INTEL_GPU=OFF -DENABLE_MULTI=OFF -DENABLE_AUTO=OFF -DENABLE_AUTO_BATCH=OFF -DENABLE_HETERO=OFF -DENABLE_TEMPLATE=OFF -DENABLE_OV_ONNX_FRONTEND=OFF -DENABLE_OV_PADDLE_FRONTEND=OFF -DENABLE_OV_PYTORCH_FRONTEND=OFF -DENABLE_OV_TF_FRONTEND=OFF -DCMAKE_INSTALL_PREFIX=install .. + cmake -G Ninja -Wno-dev -DCMAKE_BUILD_TYPE=Debug -DENABLE_CPPLINT=OFF -DCMAKE_VERBOSE_MAKEFILE=ON -DCMAKE_COMPILE_WARNING_AS_ERROR=OFF -DENABLE_FASTER_BUILD=ON -DENABLE_SANITIZER=OFF -DTHREADING=TBB -DBUILD_SHARED_LIBS=OFF -DENABLE_PROFILING_ITT=ON -DSELECTIVE_BUILD=COLLECT -DENABLE_INTEL_GPU=OFF -DENABLE_MULTI=OFF -DENABLE_AUTO=OFF -DENABLE_AUTO_BATCH=OFF -DENABLE_HETERO=OFF -DENABLE_TEMPLATE=OFF -DENABLE_OV_ONNX_FRONTEND=OFF -DENABLE_OV_PADDLE_FRONTEND=OFF -DENABLE_OV_PYTORCH_FRONTEND=OFF -DENABLE_OV_JAX_FRONTEND=OFF -DENABLE_OV_TF_FRONTEND=OFF -DCMAKE_INSTALL_PREFIX=install .. cmake --build . --config Debug @@ -274,7 +274,7 @@ Generate final optimal binaries size of OpenVINO package md build cd build - cmake -G "Visual Studio 16 2019" -A x64 -DENABLE_CPPLINT=OFF -DCMAKE_VERBOSE_MAKEFILE=ON -DCMAKE_COMPILE_WARNING_AS_ERROR=OFF -DCMAKE_BUILD_TYPE=Release -DENABLE_FASTER_BUILD=ON -DENABLE_PROFILING_ITT=OFF -DSELECTIVE_BUILD=ON -DENABLE_INTEL_GPU=OFF -DENABLE_MULTI=OFF -DENABLE_AUTO=OFF -DENABLE_AUTO_BATCH=OFF -DENABLE_HETERO=OFF -DENABLE_TEMPLATE=OFF -DENABLE_OV_ONNX_FRONTEND=OFF -DENABLE_OV_PADDLE_FRONTEND=OFF -DENABLE_OV_PYTORCH_FRONTEND=OFF -DENABLE_OV_TF_FRONTEND=OFF -DSELECTIVE_BUILD_STAT=%OPENVINO_HOME%\cc_data\*.csv -DBUILD_SHARED_LIBS=OFF -DENABLE_LTO=ON -DENABLE_ONEDNN_FOR_GPU=OFF -DENABLE_OV_TF_LITE_FRONTEND=OFF -DENABLE_PROFILING_FIRST_INFERENCE=OFF .. + cmake -G "Visual Studio 16 2019" -A x64 -DENABLE_CPPLINT=OFF -DCMAKE_VERBOSE_MAKEFILE=ON -DCMAKE_COMPILE_WARNING_AS_ERROR=OFF -DCMAKE_BUILD_TYPE=Release -DENABLE_FASTER_BUILD=ON -DENABLE_PROFILING_ITT=OFF -DSELECTIVE_BUILD=ON -DENABLE_INTEL_GPU=OFF -DENABLE_MULTI=OFF -DENABLE_AUTO=OFF -DENABLE_AUTO_BATCH=OFF -DENABLE_HETERO=OFF -DENABLE_TEMPLATE=OFF -DENABLE_OV_ONNX_FRONTEND=OFF -DENABLE_OV_PADDLE_FRONTEND=OFF -DENABLE_OV_PYTORCH_FRONTEND=OFF -DENABLE_OV_JAX_FRONTEND=OFF -DENABLE_OV_TF_FRONTEND=OFF -DSELECTIVE_BUILD_STAT=%OPENVINO_HOME%\cc_data\*.csv -DBUILD_SHARED_LIBS=OFF -DENABLE_LTO=ON -DENABLE_ONEDNN_FOR_GPU=OFF -DENABLE_OV_TF_LITE_FRONTEND=OFF -DENABLE_PROFILING_FIRST_INFERENCE=OFF .. cmake --build . --config Release diff --git a/docs/articles_en/openvino-workflow/model-optimization-guide/weight-compression.rst b/docs/articles_en/openvino-workflow/model-optimization-guide/weight-compression.rst index 7ce2893107e626..a393a0925cba3c 100644 --- a/docs/articles_en/openvino-workflow/model-optimization-guide/weight-compression.rst +++ b/docs/articles_en/openvino-workflow/model-optimization-guide/weight-compression.rst @@ -13,27 +13,32 @@ memory footprint, a crucial factor for Large Language Models (LLMs). Unlike full model quantization, where weights and activations are quantized, weight compression in `Neural Network Compression Framework (NNCF) `__ -only targets the model's weights. This approach -allows the activations to remain as floating-point numbers, preserving most -of the model's accuracy while improving its speed and reducing -its size. +only targets the model's weights. This approach allows the activations to remain as +floating-point numbers, preserving most of the model's accuracy while improving its +speed and reducing its size. The reduction in size is especially noticeable with larger models, for instance the 7 billion parameter Llama 2 model can be reduced -from about 25GB to 4GB using 4-bit weight compression. With smaller models (i.e. less than 1B parameters), -weight compression may result in more accuracy reduction than with larger models. +from about 25GB to 4GB using 4-bit weight compression. With smaller models (i.e. less +than 1B parameters), weight compression may result in more accuracy reduction than +with larger models. LLMs and other models that require extensive memory to store the weights during inference can benefit from weight compression as it: -* enables inference of exceptionally large models that cannot be accommodated in the device memory; +* enables inference of exceptionally large models that cannot be accommodated in the + device memory; -* reduces storage and memory overhead, making models more lightweight and less resource intensive for deployment; +* reduces storage and memory overhead, making models more lightweight and less resource + intensive for deployment; -* improves inference speed by reducing the latency of memory access when computing the operations with weights, for example, Linear layers. The weights are smaller and thus faster to load from memory; +* improves inference speed by reducing the latency of memory access when computing the + operations with weights, for example, Linear layers. The weights are smaller and thus + faster to load from memory; -* unlike quantization, does not require sample data to calibrate the range of activation values. +* unlike quantization, does not require sample data to calibrate the range of + activation values. Currently, `NNCF `__ provides weight quantization to 8 and 4-bit integer data types as a compression @@ -50,7 +55,8 @@ Transformer-based models. Models with 8-bit compressed weights are performant on vast majority of supported CPU and GPU platforms. -The code snippet below shows how to do 8-bit quantization of the model weights represented in OpenVINO IR using NNCF: +The code snippet below shows how to do 8-bit quantization of the model weights represented +in OpenVINO IR using NNCF: .. tab-set:: @@ -70,7 +76,8 @@ where INT4 is considered as the primary precision and INT8 is the backup one. It usually results in a smaller model size and lower inference latency, although the accuracy degradation could be higher, depending on the model. -The code snippet below shows how to do 4-bit quantization of the model weights represented in OpenVINO IR using NNCF: +The code snippet below shows how to do 4-bit quantization of the model weights represented +in OpenVINO IR using NNCF: .. tab-set:: @@ -108,13 +115,16 @@ memory reduction, speed gain, and accuracy loss. -The INT4 method has several parameters that can provide different performance-accuracy trade-offs after optimization: +The INT4 method has several parameters that can provide different performance-accuracy +trade-offs after optimization: * ``mode`` - there are two optimization modes: symmetric and asymmetric. **Symmetric Compression** - ``INT4_SYM`` - INT4 Symmetric mode involves quantizing weights to an unsigned 4-bit integer symmetrically with a fixed zero point of 8. This mode is faster than the INT8, making it ideal for situations where **speed and size reduction are prioritized over accuracy**. + INT4 Symmetric mode involves quantizing weights to an unsigned 4-bit integer + symmetrically with a fixed zero point of 8. This mode is faster than the INT8, making + it ideal for situations where **speed and size reduction are prioritized over accuracy**. .. code-block:: python @@ -125,7 +135,10 @@ The INT4 method has several parameters that can provide different performance-ac **Asymmetric Compression** - ``INT4_ASYM`` - INT4 Asymmetric mode also uses an unsigned 4-bit integer but quantizes weights asymmetrically with a non-fixed zero point. This mode slightly compromises speed in favor of better accuracy compared to the symmetric mode. This mode is useful when **minimal accuracy loss is crucial**, but a faster performance than INT8 is still desired. + INT4 Asymmetric mode also uses an unsigned 4-bit integer but quantizes weights + asymmetrically with a non-fixed zero point. This mode slightly compromises speed in + favor of better accuracy compared to the symmetric mode. This mode is useful when + **minimal accuracy loss is crucial**, but a faster performance than INT8 is still desired. .. code-block:: python @@ -134,19 +147,31 @@ The INT4 method has several parameters that can provide different performance-ac compressed_model = compress_weights(model, mode=CompressWeightsMode.INT4_ASYM) -* ``group_size`` controls the size of the group of weights that share the same quantization parameters. Shared quantization parameters help to speed up the calculation of activation values as they are dequantized and quantized between layers. However, they can reduce accuracy. The following group sizes are recommended: ``128``, ``64``, ``32`` (``128`` is default value). +* ``group_size`` controls the size of the group of weights that share the same + quantization parameters. Shared quantization parameters help to speed up the + calculation of activation values as they are dequantized and quantized between + layers. However, they can reduce accuracy. The following group sizes are + recommended: ``128``, ``64``, ``32`` (``128`` is default value). - `Smaller Group Size`: Leads to a more accurate model but increases the model's footprint and reduces inference speed. + `Smaller Group Size`: Leads to a more accurate model but increases the model's + footprint and reduces inference speed. - `Larger Group Size`: Results in faster inference and a smaller model, but might compromise accuracy. + `Larger Group Size`: Results in faster inference and a smaller model, but might + compromise accuracy. -* ``ratio`` controls the ratio between INT4 and INT8 compressed layers in the model. Ratio is a decimal between 0 and 1. For example, 0.8 means that 80% of layers will be compressed to INT4, while the rest will be compressed to INT8 precision. The default value for ratio is 1. +* ``ratio`` controls the ratio between INT4 and INT8 compressed layers in the model. + Ratio is a decimal between 0 and 1. For example, 0.8 means that 80% of layers will be + compressed to INT4, while the rest will be compressed to INT8 precision. The default + value for ratio is 1. - `Higher Ratio (more INT4)`: Reduces the model size and increase inference speed but might lead to higher accuracy degradation. + `Higher Ratio (more INT4)`: Reduces the model size and increase inference speed but + might lead to higher accuracy degradation. - `Lower Ratio (more INT8)`: Maintains better accuracy but results in a larger model size and potentially slower inference. + `Lower Ratio (more INT8)`: Maintains better accuracy but results in a larger model size + and potentially slower inference. - In this example, 90% of the model's layers are quantized to INT4 asymmetrically with a group size of 64: + In this example, 90% of the model's layers are quantized to INT4 asymmetrically with + a group size of 64: .. code-block:: python @@ -160,25 +185,43 @@ The INT4 method has several parameters that can provide different performance-ac ratio=0.9, ) -* ``dataset`` - calibration dataset for data-aware weight compression. It is required for some compression options, for example, some types ``sensitivity_metric`` can use data for precision selection. +* ``dataset`` - calibration dataset for data-aware weight compression. It is required + for some compression options, for example, some types ``sensitivity_metric`` can use + data for precision selection. -* ``sensitivity_metric`` - controls the metric to estimate the sensitivity of compressing layers in the bit-width selection algorithm. Some of the metrics require dataset to be provided. The following types are supported: +* ``sensitivity_metric`` - controls the metric to estimate the sensitivity of compressing + layers in the bit-width selection algorithm. Some of the metrics require dataset to be + provided. The following types are supported: - * ``nncf.SensitivityMetric.WEIGHT_QUANTIZATION_ERROR`` - data-free metric computed as the inverted 8-bit quantization noise. Weights with highest value of this metric can be accurately quantized channel-wise to 8-bit. The idea is to leave these weights in 8 bit, and quantize the rest of layers to 4-bit group-wise. Since group-wise is more accurate than per-channel, accuracy should not degrade. + * ``nncf.SensitivityMetric.WEIGHT_QUANTIZATION_ERROR`` - data-free metric computed as + the inverted 8-bit quantization noise. Weights with highest value of this metric can + be accurately quantized channel-wise to 8-bit. The idea is to leave these weights in + 8 bit, and quantize the rest of layers to 4-bit group-wise. Since group-wise is more + accurate than per-channel, accuracy should not degrade. - * ``nncf.SensitivityMetric.HESSIAN_INPUT_ACTIVATION`` - requires dataset. The average Hessian trace of weights with respect to the layer-wise quantization error multiplied by L2 norm of 8-bit quantization noise. + * ``nncf.SensitivityMetric.HESSIAN_INPUT_ACTIVATION`` - requires dataset. The average + Hessian trace of weights with respect to the layer-wise quantization error multiplied + by L2 norm of 8-bit quantization noise. - * ``nncf.SensitivityMetric.MEAN_ACTIVATION_VARIANCE`` - requires dataset. The mean variance of the layers' inputs multiplied by inverted 8-bit quantization noise. + * ``nncf.SensitivityMetric.MEAN_ACTIVATION_VARIANCE`` - requires dataset. The mean + variance of the layers' inputs multiplied by inverted 8-bit quantization noise. - * ``nncf.SensitivityMetric.MAX_ACTIVATION_VARIANCE`` - requires dataset. The maximum variance of the layers' inputs multiplied by inverted 8-bit quantization noise. + * ``nncf.SensitivityMetric.MAX_ACTIVATION_VARIANCE`` - requires dataset. The maximum + variance of the layers' inputs multiplied by inverted 8-bit quantization noise. - * ``nncf.SensitivityMetric.MEAN_ACTIVATION_MAGNITUDE`` - requires dataset. The mean magnitude of the layers' inputs multiplied by inverted 8-bit quantization noise. + * ``nncf.SensitivityMetric.MEAN_ACTIVATION_MAGNITUDE`` - requires dataset. The mean + magnitude of the layers' inputs multiplied by inverted 8-bit quantization noise. -* ``all_layers`` - boolean parameter that enables INT4 weight quantization of all Fully-Connected and Embedding layers, including the first and last layers in the model. +* ``all_layers`` - boolean parameter that enables INT4 weight quantization of all + Fully-Connected and Embedding layers, including the first and last layers in the model. -* ``awq`` - boolean parameter that enables the AWQ method for more accurate INT4 weight quantization. Especially helpful when the weights of all the layers are quantized to 4 bits. The method can sometimes result in reduced accuracy when used with Dynamic Quantization of activations. Requires dataset. +* ``awq`` - boolean parameter that enables the AWQ method for more accurate INT4 weight + quantization. Especially helpful when the weights of all the layers are quantized to + 4 bits. The method can sometimes result in reduced accuracy when used with + Dynamic Quantization of activations. Requires dataset. -For data-aware weight compression refer to the following `example `__. +For data-aware weight compression refer to the following +`example `__. The example below shows data-free 4-bit weight quantization applied on top of OpenVINO IR. Before trying the example, make sure Optimum Intel @@ -191,11 +234,11 @@ is installed in your environment by running the following command: The first example loads a pre-trained Hugging Face model using the Optimum Intel API, compresses it to INT4 using NNCF, and then executes inference with a text phrase. -If the model comes from Hugging Face and is supported by Optimum, it can -be easier to use the Optimum Intel API to perform weight compression. The compression -type is specified when the model is loaded using the ``load_in_8bit=True`` or ``load_in_4bit=True`` parameter. -The second example uses the Weight Compression API from Optimum Intel instead of NNCF -to compress the model to INT8. +If the model comes from `Hugging Face `__ and is supported +by Optimum, it may be easier to use the Optimum Intel API to perform weight compression. +The compression type is specified when the model is loaded using the ``load_in_8bit=True`` +or ``load_in_4bit=True`` parameter. The second example uses the Weight Compression API +from Optimum Intel instead of NNCF to compress the model to INT8. .. tab-set:: @@ -291,14 +334,17 @@ Make sure to install GPTQ dependencies by running the following command: results = pipe(phrase) print(results) -An `example of a model `__ that has been optimized using GPTQ. +An `example of a model `__ +that has been optimized using GPTQ. Compression Metrics Examples ######################################## -The table below shows examples of text-generation Language Models with different optimization settings in a data-free setup, where no dataset is used at the optimization step. -The Perplexity metric is a measurement of response accuracy, where a higher complexity score indicates a lower accuracy. -It is measured on the `Lambada OpenAI dataset `__. +The table below shows examples of text-generation Language Models with different +optimization settings in a data-free setup, where no dataset is used at the optimization step. +The Perplexity metric is a measurement of response accuracy, where a higher complexity +score indicates a lower accuracy. It is measured on the +`Lambada OpenAI dataset `__. .. list-table:: :widths: 40 55 25 25 @@ -370,7 +416,8 @@ It is measured on the `Lambada OpenAI dataset `__. +The following table shows accuracy metric in a data-aware 4-bit weight quantization +setup measured on the `Wikitext dataset `__. .. list-table:: :widths: 40 55 25 25 @@ -406,21 +453,28 @@ The following table shows accuracy metric in a data-aware 4-bit weight quantizat - 2.6 -\*Perplexity metric in both tables was measured without the Dynamic Quantization feature enabled in the OpenVINO runtime. +\*Perplexity metric in both tables was measured without the Dynamic Quantization feature +enabled in the OpenVINO runtime. Auto-tuning of Weight Compression Parameters ############################################ -To find the optimal weight compression parameters for a particular model, refer to the `example `__ , where weight compression parameters are being searched from the subset of values. To speed up the search, a self-designed -validation pipeline called `WhoWhatBench `__ is used. -The pipeline can quickly evaluate the changes in the accuracy of the optimized model compared to the baseline. +To find the optimal weight compression parameters for a particular model, refer to the +`example `__ , +where weight compression parameters are being searched from the subset of values. +To speed up the search, a self-designed validation pipeline called +`WhoWhatBench `__ +is used. The pipeline can quickly evaluate the changes in the accuracy of the optimized +model compared to the baseline. Additional Resources #################### - `Data-aware Weight Compression Example `__ - `Tune Weight Compression Parameters Example `__ -- `OpenVINO GenAI Repo `__: Repository containing example pipelines that implement image and text generation tasks. It also provides a tool to benchmark LLMs. +- `OpenVINO GenAI Repo `__ + : Repository containing example pipelines that implement image and text generation + tasks. It also provides a tool to benchmark LLMs. - `WhoWhatBench `__ - `NNCF GitHub `__ - :doc:`Post-training Quantization ` diff --git a/docs/articles_en/openvino-workflow/model-optimization.rst b/docs/articles_en/openvino-workflow/model-optimization.rst index 2cf08990d6b1d6..6b56ee121b47ae 100644 --- a/docs/articles_en/openvino-workflow/model-optimization.rst +++ b/docs/articles_en/openvino-workflow/model-optimization.rst @@ -13,23 +13,48 @@ Model Optimization Guide model-optimization-guide/weight-compression -Model optimization is an optional offline step of improving the final model performance and reducing the model size by applying special optimization methods, such as 8-bit quantization, pruning, etc. OpenVINO offers three optimization paths implemented in `Neural Network Compression Framework (NNCF) `__: +Model optimization is an optional offline step of improving the final model performance +and reducing the model size by applying special optimization methods, such as 8-bit +quantization, pruning, etc. OpenVINO offers three optimization paths implemented in +`Neural Network Compression Framework (NNCF) `__: -- :doc:`Post-training Quantization ` is designed to optimize the inference of deep learning models by applying the post-training 8-bit integer quantization that does not require model retraining or fine-tuning. +- :doc:`Post-training Quantization ` + is designed to optimize the inference of deep learning models by applying the + post-training 8-bit integer quantization that does not require model retraining + or fine-tuning. + +- :doc:`Training-time Optimization ` + , a suite of advanced methods for training-time model optimization within the DL + framework, such as PyTorch and TensorFlow 2.x. It supports methods like + Quantization-aware Training, Structured and Unstructured Pruning, etc. -- :doc:`Training-time Optimization `, a suite of advanced methods for training-time model optimization within the DL framework, such as PyTorch and TensorFlow 2.x. It supports methods like Quantization-aware Training, Structured and Unstructured Pruning, etc. +- :doc:`Weight Compression `, + an easy-to-use method for Large Language Models footprint reduction and inference acceleration. -- :doc:`Weight Compression `, an easy-to-use method for Large Language Models footprint reduction and inference acceleration. +.. note:: -.. note:: OpenVINO also supports optimized models (for example, quantized) from source frameworks such as PyTorch, TensorFlow, and ONNX (in Q/DQ; Quantize/DeQuantize format). No special steps are required in this case and optimized models can be converted to the OpenVINO Intermediate Representation format (IR) right away. + OpenVINO also supports optimized models (for example, quantized) from source + frameworks such as PyTorch, TensorFlow, and ONNX (in Q/DQ; Quantize/DeQuantize format). + No special steps are required in this case and optimized models can be converted to + the OpenVINO Intermediate Representation format (IR) right away. -Post-training Quantization is the fastest way to optimize an arbitrary DL model and should be applied first, but it is limited in terms of achievable accuracy-performance trade-off. The recommended approach to obtain OpenVINO quantized model is to convert a model from original framework to ``ov.Model`` and ensure that the model works correctly in OpenVINO, for example, by calculating the model metrics. Then, ``ov.Model`` can be used as input for the ``nncf.quantize()`` method to get the quantized model or as input for the ``nncf.compress_weights()`` method to compress weights of Large Language Models (see the diagram below). +Post-training Quantization is the fastest way to optimize an arbitrary DL model and should +be applied first, but it is limited in terms of achievable accuracy-performance trade-off. +The recommended approach to obtain OpenVINO quantized model is to convert +`a model from original framework `__ to ``ov.Model`` +and ensure that the model works correctly in OpenVINO, for example, by calculating the +model metrics. Then, ``ov.Model`` can be used as input for the ``nncf.quantize()`` +method to get the quantized model or as input for the ``nncf.compress_weights()`` +method to compress weights of Large Language Models (see the diagram below). -In case of unsatisfactory accuracy or performance after Post-training Quantization, Training-time Optimization can be used as an option. +In case of unsatisfactory accuracy or performance after Post-training Quantization, +Training-time Optimization can be used as an option. .. image:: ../assets/images/DEVELOPMENT_FLOW_V3_crunch.svg -Once the model is optimized using the aforementioned methods, it can be used for inference using the regular OpenVINO inference workflow. No changes to the inference code are required. +Once the model is optimized using the aforementioned methods, it can be used for +inference using the regular OpenVINO inference workflow. No changes to the inference +code are required. .. image:: ../assets/images/WHAT_TO_USE.svg diff --git a/docs/articles_en/openvino-workflow/model-preparation/conversion-parameters.rst b/docs/articles_en/openvino-workflow/model-preparation/conversion-parameters.rst index bb1626559f6987..6a7023fc16afc3 100644 --- a/docs/articles_en/openvino-workflow/model-preparation/conversion-parameters.rst +++ b/docs/articles_en/openvino-workflow/model-preparation/conversion-parameters.rst @@ -7,9 +7,15 @@ Conversion Parameters .. meta:: :description: Model Conversion API provides several parameters to adjust model conversion. -This document describes all available parameters for ``openvino.convert_model``, ``ovc``, and ``openvino.save_model`` without focusing on a particular framework model format. Use this information for your reference as a common description of the conversion API capabilities in general. Part of the options can be not relevant to some specific frameworks. Use :doc:`Supported Model Formats <../model-preparation>` page for more dedicated framework-dependent tutorials. +This document describes all available parameters for ``openvino.convert_model``, ``ovc``, +and ``openvino.save_model`` without focusing on a particular framework model format. +Use this information for your reference as a common description of the conversion API +capabilities in general. Part of the options can be not relevant to some specific +frameworks. Use :doc:`Supported Model Formats <../model-preparation>` page for more +dedicated framework-dependent tutorials. -In most cases when it is required to convert a model the following simple syntax can be used: +You can obtain a model from `Hugging Face `__. When you +need to convert it, in most cases, you can use the following simple syntax: .. tab-set:: @@ -36,15 +42,34 @@ In most cases when it is required to convert a model the following simple syntax ovc path_to_your_model -Providing just a path to the model or model object as ``openvino.convert_model`` argument is frequently enough to make a successful conversion. However, depending on the model topology and original deep learning framework, additional parameters may be required, which are described below. - -- ``example_input`` parameter available in Python ``openvino.convert_model`` only is intended to trace the model to obtain its graph representation. This parameter is crucial for converting PyTorch models and may sometimes be required for TensorFlow models. For more details, refer to the :doc:`PyTorch Model Conversion ` or :doc:`TensorFlow Model Conversion `. - -- ``input`` parameter to set or override shapes for model inputs. It configures dynamic and static dimensions in model inputs depending on your inference requirements. For more information on this parameter, refer to the :doc:`Setting Input Shapes ` guide. - -- ``output`` parameter to select one or multiple outputs from the original model. This is useful when the model has outputs that are not required for inference in a deployment scenario. By specifying only necessary outputs, you can create a more compact model that infers faster. - -- ``compress_to_fp16`` parameter that is provided by ``ovc`` CLI tool and ``openvino.save_model`` Python function, gives controls over the compression of model weights to FP16 format when saving OpenVINO model to IR. This option is enabled by default which means all produced IRs are saved using FP16 data type for weights which saves up to 2x storage space for the model file and in most cases doesn't sacrifice model accuracy. In case it does affect accuracy, the compression can be disabled by setting this flag to ``False``: +Providing just a path to the model or model object as ``openvino.convert_model`` argument +is frequently enough to make a successful conversion. However, depending on the model +topology and original deep learning framework, additional parameters may be required, +which are described below. + +- ``example_input`` parameter available in Python ``openvino.convert_model`` only is + intended to trace the model to obtain its graph representation. This parameter is crucial + for converting PyTorch models and may sometimes be required for TensorFlow models. + For more details, refer to the :doc:`PyTorch Model Conversion ` + or :doc:`TensorFlow Model Conversion `. + +- ``input`` parameter to set or override shapes for model inputs. It configures dynamic + and static dimensions in model inputs depending on your inference requirements. For more + information on this parameter, refer to the :doc:`Setting Input Shapes ` + guide. + +- ``output`` parameter to select one or multiple outputs from the original model. + This is useful when the model has outputs that are not required for inference in a + deployment scenario. By specifying only necessary outputs, you can create a more + compact model that infers faster. + +- ``compress_to_fp16`` parameter that is provided by ``ovc`` CLI tool and + ``openvino.save_model`` Python function, gives controls over the compression of + model weights to FP16 format when saving OpenVINO model to IR. This option is enabled + by default which means all produced IRs are saved using FP16 data type for weights + which saves up to 2x storage space for the model file and in most cases doesn't + sacrifice model accuracy. In case it does affect accuracy, the compression can be + disabled by setting this flag to ``False``: .. tab-set:: @@ -75,23 +100,59 @@ For details on how plugins handle compressed ``FP16`` models, see Refer to the :doc:`Post-training optimization <../model-optimization-guide/quantizing-models-post-training>` guide for more information about that. -- ``extension`` parameter which makes possible conversion of the models consisting of operations that are not supported by OpenVINO out-of-the-box. It requires implementing of an OpenVINO extension first, please refer to :doc:`Frontend Extensions <../../documentation/openvino-extensibility/frontend-extensions>` guide. - -- ``share_weigths`` parameter with default value ``True`` allows reusing memory with original weights. For models loaded in Python and then passed to ``openvino.convert_model``, that means that OpenVINO model will share the same areas in program memory where the original weights are located. For models loaded from files by ``openvino.convert_model``, file memory mapping is used to avoid extra memory allocation. When enabled, the original model cannot be modified (Python object cannot be deallocated and original model file cannot be deleted) for the whole lifetime of OpenVINO model. Even model inference by original framework can lead to model modification. If it is not desired, set ``share_weights=False`` when calling ``openvino.convert_model``. - -.. note:: ``ovc`` does not have ``share_weights`` option and always uses sharing to reduce conversion time and consume less amount of memory during the conversion. +- ``extension`` parameter which makes possible conversion of the models consisting of + operations that are not supported by OpenVINO out-of-the-box. It requires implementing of + an OpenVINO extension first, please refer to + :doc:`Frontend Extensions <../../documentation/openvino-extensibility/frontend-extensions>` + guide. + +- ``share_weigths`` parameter with default value ``True`` allows reusing memory with + original weights. For models loaded in Python and then passed to ``openvino.convert_model``, + that means that OpenVINO model will share the same areas in program memory where the + original weights are located. For models loaded from files by ``openvino.convert_model``, + file memory mapping is used to avoid extra memory allocation. When enabled, the + original model cannot be modified (Python object cannot be deallocated and original + model file cannot be deleted) for the whole lifetime of OpenVINO model. Even model + inference by original framework can lead to model modification. If it is not desired, + set ``share_weights=False`` when calling ``openvino.convert_model``. + + .. note:: + + ``ovc`` does not have ``share_weights`` option and always uses sharing to reduce + conversion time and consume less amount of memory during the conversion. + +- ``output_model`` parameter in ``ovc`` and ``openvino.save_model`` specifies name for + output ``.xml`` file with the resulting OpenVINO IR. The accompanying ``.bin`` file + name will be generated automatically by replacing ``.xml`` extension with ``.bin`` + extension. The value of ``output_model`` must end with ``.xml`` extension. For ``ovc`` + command line tool, ``output_model`` can also contain a name of a directory. In this case, + the resulting OpenVINO IR files will be put into that directory with a base name of + ``.xml`` and ``.bin`` files matching the original model base name passed to ``ovc`` as a + parameter. For example, when calling ``ovc your_model.onnx --output_model directory_name``, + files ``directory_name/your_model.xml`` and ``directory_name/your_model.bin`` will be + created. If ``output_model`` is not used, then the current directory is used as + a destination directory. + + .. note:: + + ``openvino.save_model`` does not support a directory for ``output_model`` + parameter value because ``openvino.save_model`` gets OpenVINO model object + represented in a memory and there is no original model file name available for + output file name generation. For the same reason, ``output_model`` is a mandatory + parameter for ``openvino.save_model``. + +- ``verbose`` parameter activates extra diagnostics printed to the standard output. + Use for debugging purposes in case there is an issue with the conversion and to collect + information for better bug reporting to OpenVINO team. -- ``output_model`` parameter in ``ovc`` and ``openvino.save_model`` specifies name for output ``.xml`` file with the resulting OpenVINO IR. The accompanying ``.bin`` file name will be generated automatically by replacing ``.xml`` extension with ``.bin`` extension. The value of ``output_model`` must end with ``.xml`` extension. For ``ovc`` command line tool, ``output_model`` can also contain a name of a directory. In this case, the resulting OpenVINO IR files will be put into that directory with a base name of ``.xml`` and ``.bin`` files matching the original model base name passed to ``ovc`` as a parameter. For example, when calling ``ovc your_model.onnx --output_model directory_name``, files ``directory_name/your_model.xml`` and ``directory_name/your_model.bin`` will be created. If ``output_model`` is not used, then the current directory is used as a destination directory. - -.. note:: ``openvino.save_model`` does not support a directory for ``output_model`` parameter value because ``openvino.save_model`` gets OpenVINO model object represented in a memory and there is no original model file name available for output file name generation. For the same reason, ``output_model`` is a mandatory parameter for ``openvino.save_model``. - -- ``verbose`` parameter activates extra diagnostics printed to the standard output. Use for debugging purposes in case there is an issue with the conversion and to collect information for better bug reporting to OpenVINO team. +.. note:: -.. note:: Weights sharing does not equally work for all the supported model formats. The value of this flag is considered as a hint for the conversion API, and actual sharing is used only if it is implemented and possible for a particular model representation. + Weights sharing does not equally work for all the supported model formats. The value + of this flag is considered as a hint for the conversion API, and actual sharing is + used only if it is implemented and possible for a particular model representation. -You can always run ``ovc -h`` or ``ovc --help`` to recall all the supported parameters for ``ovc``. +You can always run ``ovc -h`` or ``ovc --help`` to recall all the supported +parameters for ``ovc``. Use ``ovc --version`` to check the version of OpenVINO package installed. - - diff --git a/docs/articles_en/openvino-workflow/model-preparation/convert-model-onnx.rst b/docs/articles_en/openvino-workflow/model-preparation/convert-model-onnx.rst index b52552ad55730e..9c903388a5c3bd 100644 --- a/docs/articles_en/openvino-workflow/model-preparation/convert-model-onnx.rst +++ b/docs/articles_en/openvino-workflow/model-preparation/convert-model-onnx.rst @@ -11,18 +11,30 @@ Converting an ONNX Model Introduction to ONNX #################### -`ONNX `__ is a representation format for deep learning models that enables AI developers to easily transfer models between different frameworks. +`ONNX `__ is a representation format for deep learning models +that enables AI developers to easily transfer models between different frameworks. -.. note:: An ONNX model file can be loaded by ``openvino.Core.read_model`` or ``openvino.Core.compile_model`` methods by OpenVINO runtime API without the need to prepare an OpenVINO IR first. Refer to the :doc:`inference example <../running-inference/integrate-openvino-with-your-application>` for more details. Using ``openvino.convert_model`` is still recommended if the model load latency is important for the inference application. +.. note:: + + An ONNX model file can be loaded by ``openvino.Core.read_model`` or + ``openvino.Core.compile_model`` methods by OpenVINO runtime API without the need to + prepare an OpenVINO IR first. Refer to the + :doc:`inference example <../running-inference/integrate-openvino-with-your-application>` + for more details. Using ``openvino.convert_model`` is still recommended if the model + load latency is important for the inference application. Converting an ONNX Model ######################## -This page provides instructions on model conversion from the ONNX format to the OpenVINO IR format. +This page provides instructions on model conversion from the ONNX format to the +OpenVINO IR format. -For model conversion, you need an ONNX model either directly downloaded from a public repository or converted from any framework that supports exporting to the ONNX format. +For model conversion, you need an ONNX model either directly downloaded from +an online database, for example `Hugging Face `__ , or +converted from any framework that supports exporting to the ONNX format. -To convert an ONNX model, run model conversion with the path to the input model ``.onnx`` file: +To convert an ONNX model, run model conversion with the path to the input +model ``.onnx`` file: .. tab-set:: diff --git a/docs/articles_en/openvino-workflow/model-preparation/convert-model-paddle.rst b/docs/articles_en/openvino-workflow/model-preparation/convert-model-paddle.rst index c70724b4047f95..97c968bc4a7692 100644 --- a/docs/articles_en/openvino-workflow/model-preparation/convert-model-paddle.rst +++ b/docs/articles_en/openvino-workflow/model-preparation/convert-model-paddle.rst @@ -8,16 +8,31 @@ Converting a PaddlePaddle Model :description: Learn how to convert a model from the PaddlePaddle format to the OpenVINO Model. -This page provides general instructions on how to convert a model from the PaddlePaddle format to the OpenVINO IR format using OpenVINO model conversion API. The instructions are different depending on the PaddlePaddle model format. +This page provides general instructions on how to convert a model from the PaddlePaddle +format to the OpenVINO IR format using OpenVINO model conversion API. The instructions +are different depending on the PaddlePaddle model format. -.. note:: PaddlePaddle model serialized in a file can be loaded by ``openvino.Core.read_model`` or ``openvino.Core.compile_model`` methods by OpenVINO runtime API without preparing OpenVINO IR first. Refer to the :doc:`inference example <../running-inference/integrate-openvino-with-your-application>` for more details. Using ``openvino.convert_model`` is still recommended if model load latency matters for the inference application. +.. note:: + + PaddlePaddle model serialized in a file can be loaded by ``openvino.Core.read_model`` + or ``openvino.Core.compile_model`` methods by OpenVINO runtime API without preparing + OpenVINO IR first. Refer to the + :doc:`inference example <../running-inference/integrate-openvino-with-your-application>` + for more details. Using ``openvino.convert_model`` is still recommended if model load + latency matters for the inference application. Converting PaddlePaddle Model Files ################################### -PaddlePaddle inference model includes ``.pdmodel`` (storing model structure) and ``.pdiparams`` (storing model weight). For details on how to export a PaddlePaddle inference model, refer to the `Exporting PaddlePaddle Inference Model `__ Chinese guide. +PaddlePaddle inference model includes ``.pdmodel`` (storing model structure) and +``.pdiparams`` (storing model weight). For details on how to export a PaddlePaddle +inference model, refer to the +`Exporting PaddlePaddle Inference Model `__ +Chinese guide. -To convert a PaddlePaddle model, use the ``ovc`` or ``openvino.convert_model`` and specify the path to the input ``.pdmodel`` model file: +You can download a PaddlePaddle model from `Hugging Face `__. +To convert the model, use the ``ovc`` or ``openvino.convert_model`` +and specify the path to the input ``.pdmodel`` model file: .. tab-set:: diff --git a/docs/articles_en/openvino-workflow/model-preparation/convert-model-pytorch.rst b/docs/articles_en/openvino-workflow/model-preparation/convert-model-pytorch.rst index 7289c6cbe020c0..6ac806daf0cda0 100644 --- a/docs/articles_en/openvino-workflow/model-preparation/convert-model-pytorch.rst +++ b/docs/articles_en/openvino-workflow/model-preparation/convert-model-pytorch.rst @@ -7,7 +7,9 @@ Converting a PyTorch Model PyTorch format to the OpenVINO Model. -To convert a PyTorch model, use the ``openvino.convert_model`` function. +You can download a PyTorch model from `PyTorch Hub `__ or +`Hugging Face `__. +To convert the model, use the ``openvino.convert_model`` function. Here is the simplest example of PyTorch model conversion using a model from ``torchvision``: diff --git a/docs/articles_en/openvino-workflow/model-preparation/convert-model-tensorflow-lite.rst b/docs/articles_en/openvino-workflow/model-preparation/convert-model-tensorflow-lite.rst index ecf08f9ff55575..93738f48f1d39f 100644 --- a/docs/articles_en/openvino-workflow/model-preparation/convert-model-tensorflow-lite.rst +++ b/docs/articles_en/openvino-workflow/model-preparation/convert-model-tensorflow-lite.rst @@ -9,7 +9,10 @@ Converting a TensorFlow Lite Model TensorFlow Lite format to the OpenVINO Model. -To convert an TensorFlow Lite model, run model conversion with the path to the ``.tflite`` model file: +You can download a TensorFlow Lite model from +`Kaggle `__ +or `Hugging Face `__. +To convert the model, run model conversion with the path to the ``.tflite`` model file: .. tab-set:: @@ -28,16 +31,27 @@ To convert an TensorFlow Lite model, run model conversion with the path to the ` ovc your_model_file.tflite -.. note:: TensorFlow Lite model file can be loaded by ``openvino.Core.read_model`` or ``openvino.Core.compile_model`` methods by OpenVINO runtime API without preparing OpenVINO IR first. Refer to the :doc:`inference example <../running-inference/integrate-openvino-with-your-application>` for more details. Using ``openvino.convert_model`` is still recommended if model load latency matters for the inference application. +.. note:: + + TensorFlow Lite model file can be loaded by ``openvino.Core.read_model`` or + ``openvino.Core.compile_model`` methods by OpenVINO runtime API without preparing + OpenVINO IR first. Refer to the + :doc:`inference example <../running-inference/integrate-openvino-with-your-application>` + for more details. Using ``openvino.convert_model`` is still recommended if model + load latency matters for the inference application. Supported TensorFlow Lite Layers ################################### -For the list of supported standard layers, refer to the :doc:`Supported Operations <../../about-openvino/compatibility-and-support/supported-operations-framework-frontend>` page. +For the list of supported standard layers, refer to the +:doc:`Supported Operations <../../about-openvino/compatibility-and-support/supported-operations-framework-frontend>` +page. Supported TensorFlow Lite Models ################################### -More than eighty percent of public TensorFlow Lite models are supported from open sources `TensorFlow Hub `__ and `MediaPipe `__. +More than eighty percent of public TensorFlow Lite models are supported from open +sources `Kaggle `__ +and `MediaPipe `__. Unsupported models usually have custom TensorFlow Lite operations. diff --git a/docs/articles_en/openvino-workflow/model-preparation/convert-model-tensorflow.rst b/docs/articles_en/openvino-workflow/model-preparation/convert-model-tensorflow.rst index 84aa153cfb82be..1db451045e702d 100644 --- a/docs/articles_en/openvino-workflow/model-preparation/convert-model-tensorflow.rst +++ b/docs/articles_en/openvino-workflow/model-preparation/convert-model-tensorflow.rst @@ -8,13 +8,39 @@ Converting a TensorFlow Model :description: Learn how to convert a model from a TensorFlow format to the OpenVINO Model. -This page provides general instructions on how to run model conversion from a TensorFlow format to the OpenVINO IR format. The instructions are different depending on whether your model was created with TensorFlow v1.X or TensorFlow v2.X. +This page provides general instructions on how to run model conversion from a TensorFlow +format to the OpenVINO IR format. The instructions are different depending on whether +your model was created with TensorFlow v1.X or TensorFlow v2.X. -.. note:: TensorFlow models can be loaded by ``openvino.Core.read_model`` or ``openvino.Core.compile_model`` methods by OpenVINO runtime API without preparing OpenVINO IR first. Refer to the :doc:`inference example <../running-inference/integrate-openvino-with-your-application>` for more details. Using ``openvino.convert_model`` is still recommended if model load latency matters for the inference application. +TensorFlow models can be obtained from +`Kaggle `__ +or `Hugging Face `__. -.. note:: ``openvino.convert_model`` uses sharing of model weights by default. That means that OpenVINO model will share the same areas in program memory where the original weights are located, for this reason the original model cannot be modified (Python object cannot be deallocated and original model file cannot be deleted) for the whole lifetime of OpenVINO model. Model inference for TensorFlow models can lead to model modification, so original TF model should not be inferred during the lifetime of OpenVINO model. If it is not desired, set ``share_weights=False`` when calling ``openvino.convert_model``. +.. note:: + + TensorFlow models can be loaded by ``openvino.Core.read_model`` or + ``openvino.Core.compile_model`` methods by OpenVINO runtime API without preparing + OpenVINO IR first. Refer to the + :doc:`inference example <../running-inference/integrate-openvino-with-your-application>` + for more details. Using ``openvino.convert_model`` is still recommended if model load + latency matters for the inference application. + +.. note:: + + ``openvino.convert_model`` uses sharing of model weights by default. That means that + OpenVINO model will share the same areas in program memory where the original weights + are located, for this reason the original model cannot be modified (Python object + cannot be deallocated and original model file cannot be deleted) for the whole + lifetime of OpenVINO model. Model inference for TensorFlow models can lead to model + modification, so original TF model should not be inferred during the lifetime of + OpenVINO model. If it is not desired, set ``share_weights=False`` when calling + ``openvino.convert_model``. + +.. note:: -.. note:: Examples below that convert TensorFlow models from a file, do not require any version of TensorFlow to be installed on the system, except in cases when the ``tensorflow`` module is imported explicitly. + Examples below that convert TensorFlow models from a file, do not require any version + of TensorFlow to be installed on the system, except in cases when the ``tensorflow`` + module is imported explicitly. Converting TensorFlow 2 Models ############################## @@ -25,7 +51,8 @@ Below are the instructions on how to convert each of them. SavedModel Format +++++++++++++++++ -A model in the SavedModel format consists of a directory with a ``saved_model.pb`` file and two subfolders: ``variables`` and ``assets`` inside. +A model in the SavedModel format consists of a directory with a ``saved_model.pb`` +file and two subfolders: ``variables`` and ``assets`` inside. To convert a model, run conversion with the directory as the model argument: .. tab-set:: @@ -59,8 +86,9 @@ SavedModel format. Here is an example of how to do it: model = tf.keras.models.load_model('model.h5') tf.saved_model.save(model,'model') -Converting a Keras H5 model with a custom layer to the SavedModel format requires special considerations. -For example, the model with a custom layer ``CustomLayer`` from ``custom_layer.py`` is converted as follows: +Converting a Keras H5 model with a custom layer to the SavedModel format requires special +considerations. For example, the model with a custom layer ``CustomLayer`` from +``custom_layer.py`` is converted as follows: .. code-block:: py :force: @@ -82,7 +110,8 @@ Converting TensorFlow 1 Models Converting Frozen Model Format +++++++++++++++++++++++++++++++ -To convert a TensorFlow model, run model conversion with the path to the input model ``*.pb*`` file: +To convert a TensorFlow model, run model conversion with the path to the input +model ``*.pb*`` file: .. tab-set:: @@ -107,9 +136,12 @@ Converting Non-Frozen Model Formats There are three ways to store non-frozen TensorFlow models. -1. **SavedModel format**. In this case, a model consists of a special directory with a ``.pb`` file -and several subfolders: ``variables``, ``assets``, and ``assets.extra``. For more information about the SavedModel directory, refer to the `README `__ file in the TensorFlow repository. -To convert such TensorFlow model, run the conversion similarly to other model formats and pass a path to the directory as a model argument: +1. **SavedModel format**. In this case, a model consists of a special directory with a + ``.pb`` file and several subfolders: ``variables``, ``assets``, and ``assets.extra``. + For more information about the SavedModel directory, refer to the + `README `__ + file in the TensorFlow repository. To convert such TensorFlow model, run the conversion + similarly to other model formats and pass a path to the directory as a model argument: .. tab-set:: @@ -128,9 +160,12 @@ To convert such TensorFlow model, run the conversion similarly to other model fo ovc path_to_saved_model_dir -2. **Checkpoint**. In this case, a model consists of two files: ``inference_graph.pb`` (or ``inference_graph.pbtxt``) and ``checkpoint_file.ckpt``. -If you do not have an inference graph file, refer to the `Freezing Custom Models in Python <#Freezing-Custom-Models-in-Python>`__ section. -To convert the model with the inference graph in ``.pb`` format, provide paths to both files as an argument for ``ovc`` or ``openvino.convert_model``: +2. **Checkpoint**. In this case, a model consists of two files: ``inference_graph.pb`` + (or ``inference_graph.pbtxt``) and ``checkpoint_file.ckpt``. + If you do not have an inference graph file, refer to the + `Freezing Custom Models in Python <#Freezing-Custom-Models-in-Python>`__ section. + To convert the model with the inference graph in ``.pb`` format, provide paths to both + files as an argument for ``ovc`` or ``openvino.convert_model``: .. tab-set:: @@ -149,11 +184,16 @@ To convert the model with the inference graph in ``.pb`` format, provide paths t ovc path_to_inference_graph.pb path_to_checkpoint_file.ckpt -To convert the model with the inference graph in the ``.pbtxt`` format, specify the path to ``.pbtxt`` file instead of the ``.pb`` file. The conversion API automatically detects the format of the provided file, there is no need to specify the model file format explicitly when calling ``ovc`` or ``openvino.convert_model`` in all examples in this document. +To convert the model with the inference graph in the ``.pbtxt`` format, specify the path +to ``.pbtxt`` file instead of the ``.pb`` file. The conversion API automatically detects +the format of the provided file, there is no need to specify the model file format +explicitly when calling ``ovc`` or ``openvino.convert_model`` in all examples in this document. -3. **MetaGraph**. In this case, a model consists of three or four files stored in the same directory: ``model_name.meta``, ``model_name.index``, -``model_name.data-00000-of-00001`` (the numbers may vary), and ``checkpoint`` (optional). -To convert such a TensorFlow model, run the conversion providing a path to `.meta` file as an argument: +3. **MetaGraph**. In this case, a model consists of three or four files stored in the same + directory: ``model_name.meta``, ``model_name.index``, + ``model_name.data-00000-of-00001`` (the numbers may vary), and ``checkpoint`` (optional). + To convert such a TensorFlow model, run the conversion providing a path to ``.meta`` + file as an argument: .. tab-set:: @@ -176,9 +216,11 @@ To convert such a TensorFlow model, run the conversion providing a path to `.met Freezing Custom Models in Python ++++++++++++++++++++++++++++++++ -When a model is defined in Python code, you must create an inference graph file. Graphs are usually built in a form -that allows model training. That means all trainable parameters are represented as variables in the graph. -To be able to use such a graph with the model conversion API, it should be frozen first before passing to the ``openvino.convert_model`` function: +When a model is defined in Python code, you must create an inference graph file. Graphs are +usually built in a form that allows model training. That means all trainable parameters are +represented as variables in the graph. To be able to use such a graph with the model +conversion API, it should be frozen first before passing to the +``openvino.convert_model`` function: .. code-block:: py :force: @@ -192,22 +234,28 @@ To be able to use such a graph with the model conversion API, it should be froze Where: -* ``sess`` is the instance of the TensorFlow Session object where the network topology is defined. -* ``["name_of_the_output_node"]`` is the list of output node names in the graph; ``frozen`` graph will include only those nodes from the original ``sess.graph_def`` that are directly or indirectly used to compute given output nodes. The ``'name_of_the_output_node'`` is an example of a possible output node name. You should derive the names based on your own graph. +* ``sess`` is the instance of the TensorFlow Session object where the network topology + is defined. +* ``["name_of_the_output_node"]`` is the list of output node names in the graph; + ``frozen`` graph will include only those nodes from the original ``sess.graph_def`` + that are directly or indirectly used to compute given output nodes. The + ``'name_of_the_output_node'`` is an example of a possible output node name. + You should derive the names based on your own graph. Converting TensorFlow Models from Memory Using Python API ############################################################ Model conversion API supports passing TensorFlow/TensorFlow2 models directly from memory. -* ``Trackable``. The object returned by ``hub.load()`` can be converted to ``ov.Model`` with ``convert_model()``. +* ``Trackable``. The object returned by ``hub.load()`` can be converted to + ``ov.Model`` with ``convert_model()``. .. code-block:: py :force: import tensorflow_hub as hub import openvino as ov - + model = hub.load("https://tfhub.dev/google/movenet/singlepose/lightning/4") ov_model = ov.convert_model(model) @@ -234,7 +282,10 @@ Model conversion API supports passing TensorFlow/TensorFlow2 models directly fro model = tf.keras.applications.ResNet50(weights="imagenet") ov_model = ov.convert_model(model) -* ``tf.keras.layers.Layer``. The ``ov.Model`` converted from ``tf.keras.layers.Layer`` does not contain original input and output names. So it is recommended to convert the model to ``tf.keras.Model`` before conversion or use ``hub.load()`` for TensorFlow Hub models. +* ``tf.keras.layers.Layer``. The ``ov.Model`` converted from ``tf.keras.layers.Layer`` + does not contain original input and output names. So it is recommended to convert the + model to ``tf.keras.Model`` before conversion or use ``hub.load()`` for + TensorFlow Hub models. .. code-block:: py :force: @@ -265,7 +316,12 @@ Model conversion API supports passing TensorFlow/TensorFlow2 models directly fro model = MyModule(name="simple_module") ov_model = ov.convert_model(model, input=[-1]) -.. note:: There is a known bug in ``openvino.convert_model`` on using ``tf.Variable`` nodes in the model graph. The results of the conversion of such models are unpredictable. It is recommended to save a model with ``tf.Variable`` into TensorFlow Saved Model format and load it with ``openvino.convert_model``. +.. note:: + + There is a known bug in ``openvino.convert_model`` on using ``tf.Variable`` nodes in + the model graph. The results of the conversion of such models are unpredictable. It + is recommended to save a model with ``tf.Variable`` into TensorFlow Saved Model format + and load it with ``openvino.convert_model``. * ``tf.compat.v1.Graph`` @@ -328,7 +384,9 @@ Model conversion API supports passing TensorFlow/TensorFlow2 models directly fro Supported TensorFlow and TensorFlow 2 Keras Layers ################################################## -For the list of supported standard layers, refer to the :doc:`Supported Operations <../../about-openvino/compatibility-and-support/supported-operations-framework-frontend>` page. +For the list of supported standard layers, refer to the +:doc:`Supported Operations <../../about-openvino/compatibility-and-support/supported-operations-framework-frontend>` +page. Summary ####### @@ -339,5 +397,3 @@ In this document, you learned: * Which TensorFlow models are supported. * How to freeze a TensorFlow model. - - diff --git a/docs/articles_en/openvino-workflow/model-preparation/convert-model-to-ir.rst b/docs/articles_en/openvino-workflow/model-preparation/convert-model-to-ir.rst index ac008631e49fff..be67f581173309 100644 --- a/docs/articles_en/openvino-workflow/model-preparation/convert-model-to-ir.rst +++ b/docs/articles_en/openvino-workflow/model-preparation/convert-model-to-ir.rst @@ -18,7 +18,8 @@ Convert to OpenVINO IR Convert from PaddlePaddle -:doc:`IR (Intermediate Representation) <../../documentation/openvino-ir-format>` is OpenVINO own format consisting of ``.xml`` and ``.bin`` files. +:doc:`IR (Intermediate Representation) <../../documentation/openvino-ir-format>` is +OpenVINO own format consisting of ``.xml`` and ``.bin`` files. Convert the model into OpenVINO IR for `better performance <#ir-conversion-benefits>`__. Convert Models @@ -616,4 +617,5 @@ Additional Resources #################### * :doc:`Transition guide from the legacy to new conversion API <../../documentation/legacy-features/transition-legacy-conversion-api>` +* `Download models from Hugging Face `__. diff --git a/docs/articles_en/openvino-workflow/model-preparation/setting-input-shapes.rst b/docs/articles_en/openvino-workflow/model-preparation/setting-input-shapes.rst index 7ab05fed2ec56a..9e17fdaad9999c 100644 --- a/docs/articles_en/openvino-workflow/model-preparation/setting-input-shapes.rst +++ b/docs/articles_en/openvino-workflow/model-preparation/setting-input-shapes.rst @@ -20,7 +20,8 @@ performance and memory consumption. For more information on changing input shapes in runtime, refer to the :doc:`Changing input shapes <../running-inference/changing-input-shape>` guide. To learn more about dynamic shapes in runtime, refer to the -:doc:`Dynamic Shapes <../running-inference/dynamic-shapes>` guide. +:doc:`Dynamic Shapes <../running-inference/dynamic-shapes>` guide. To download models, +you can visit `Hugging Face `__. The OpenVINO Runtime API may present certain limitations in inferring models with undefined dimensions on some hardware. See the :doc:`Feature support matrix <../../about-openvino/compatibility-and-support/supported-devices>` @@ -49,9 +50,13 @@ and specify the input shape of ``[2,300,300,3]``: ovc MobileNet.pb --input [2,300,300,3] -If a model has multiple inputs, the input shape should be specified in ``input`` parameter as a list. In ``ovc``, this is a command separate list, and in ``openvino.convert_model`` this is a Python list or tuple with number of elements matching the number of inputs in the model. Use input names from the original model to define the mapping between inputs and shapes specified. -The following example demonstrates the conversion of the ONNX OCR model with a pair of inputs ``data`` and ``seq_len`` -and specifies shapes ``[3,150,200,1]`` and ``[3]`` for them respectively: +If a model has multiple inputs, the input shape should be specified in ``input`` parameter +as a list. In ``ovc``, this is a command separate list, and in ``openvino.convert_model`` +this is a Python list or tuple with number of elements matching the number of inputs in +the model. Use input names from the original model to define the mapping between inputs +and shapes specified. The following example demonstrates the conversion of the ONNX OCR +model with a pair of inputs ``data`` and ``seq_len`` and specifies shapes ``[3,150,200,1]`` +and ``[3]`` for them respectively: .. tab-set:: @@ -71,7 +76,9 @@ and specifies shapes ``[3,150,200,1]`` and ``[3]`` for them respectively: ovc ocr.onnx --input data[3,150,200,1],seq_len[3] -If the order of inputs is defined in the input model and the order is known for the user, names could be omitted. In this case, it is important to specify shapes in the same order of input model inputs: +If the order of inputs is defined in the input model and the order is known for the user, +names could be omitted. In this case, it is important to specify shapes in the +same order of input model inputs: .. tab-set:: @@ -91,12 +98,19 @@ If the order of inputs is defined in the input model and the order is known for ovc ocr.onnx --input [3,150,200,1],[3] -Whether the model has a specified order of inputs depends on the original framework. Usually, it is convenient to set shapes without specifying the names of the parameters in the case of PyTorch model conversion because a PyTorch model is considered as a callable that usually accepts positional parameters. On the other hand, names of inputs are convenient when converting models from model files, because naming of inputs is a good practice for many frameworks that serialize models to files. +Whether the model has a specified order of inputs depends on the original framework. +Usually, it is convenient to set shapes without specifying the names of the parameters +in the case of PyTorch model conversion because a PyTorch model is considered as +a callable that usually accepts positional parameters. On the other hand, names of inputs +are convenient when converting models from model files, because naming of inputs is +a good practice for many frameworks that serialize models to files. -The ``input`` parameter allows overriding original input shapes if it is supported by the model topology. -Shapes with dynamic dimensions in the original model can be replaced with static shapes for the converted model, and vice versa. -The dynamic dimension can be marked in model conversion API parameter as ``-1`` or ``?`` when using ``ovc``. -For example, launch model conversion for the ONNX OCR model and specify dynamic batch dimension for inputs: +The ``input`` parameter allows overriding original input shapes if it is supported by +the model topology. Shapes with dynamic dimensions in the original model can be replaced +with static shapes for the converted model, and vice versa. The dynamic dimension can be +marked in model conversion API parameter as ``-1`` or ``?`` when using ``ovc``. +For example, launch model conversion for the ONNX OCR model and specify dynamic batch +dimension for inputs: .. tab-set:: @@ -116,9 +130,13 @@ For example, launch model conversion for the ONNX OCR model and specify dynamic ovc ocr.onnx --input "data[?,150,200,1],seq_len[?]" -To optimize memory consumption for models with undefined dimensions in run-time, model conversion API provides the capability to define boundaries of dimensions. -The boundaries of undefined dimension can be specified with ellipsis in the command line or with ``openvino.Dimension`` class in Python. -For example, launch model conversion for the ONNX OCR model and specify a boundary for the batch dimension 1..3, which means that the input tensor will have batch dimension minimum 1 and maximum 3 in inference: +To optimize memory consumption for models with undefined dimensions in run-time, +model conversion API provides the capability to define boundaries of dimensions. +The boundaries of undefined dimension can be specified with ellipsis in the command +line or with ``openvino.Dimension`` class in Python. +For example, launch model conversion for the ONNX OCR model and specify a boundary for +the batch dimension 1..3, which means that the input tensor will have batch dimension +minimum 1 and maximum 3 in inference: .. tab-set:: @@ -139,5 +157,9 @@ For example, launch model conversion for the ONNX OCR model and specify a bounda ovc ocr.onnx --input data[1..3,150,200,1],seq_len[1..3] -In practice, not every model is designed in a way that allows change of input shapes. An attempt to change the shape for such models may lead to an exception during model conversion, later in model inference, or even to wrong results of inference without explicit exception raised. A knowledge about model topology is required to set shapes appropriately. +In practice, not every model is designed in a way that allows change of input shapes. +An attempt to change the shape for such models may lead to an exception during model +conversion, later in model inference, or even to wrong results of inference without +explicit exception raised. A knowledge about model topology is required to set +shapes appropriately. diff --git a/docs/articles_en/openvino-workflow/running-inference.rst b/docs/articles_en/openvino-workflow/running-inference.rst index 3ccd9f3ff7cc2e..95c6bc66e902b8 100644 --- a/docs/articles_en/openvino-workflow/running-inference.rst +++ b/docs/articles_en/openvino-workflow/running-inference.rst @@ -39,9 +39,9 @@ OpenVINO IR provides by far the best first-inference latency scores. For more detailed information on how to convert, read, and compile supported model formats see the :doc:`Model Preparation article `. - Note that TensorFlow models can be run using the + Note that PyTorch models can be run using the :doc:`torch.compile feature `, as well as the standard ways of - :doc:`converting TensorFlow ` + :doc:`converting Pytorch ` or running its inference. OpenVINO Runtime uses a plugin architecture. Its plugins are software components that contain complete implementation for inference on a particular Intel® hardware device: CPU, GPU, etc. Each plugin implements the unified API and provides additional hardware-specific APIs for configuring devices or API interoperability between OpenVINO Runtime and underlying plugin backend. diff --git a/docs/articles_en/openvino-workflow/running-inference/inference-devices-and-modes.rst b/docs/articles_en/openvino-workflow/running-inference/inference-devices-and-modes.rst index 19328ec7a9c1f9..c11696ce07bc18 100644 --- a/docs/articles_en/openvino-workflow/running-inference/inference-devices-and-modes.rst +++ b/docs/articles_en/openvino-workflow/running-inference/inference-devices-and-modes.rst @@ -83,4 +83,3 @@ Accordingly, the code that loops over all available devices of the "GPU" type on :language: cpp :fragment: [part3] - diff --git a/docs/articles_en/openvino-workflow/running-inference/inference-devices-and-modes/auto-device-selection.rst b/docs/articles_en/openvino-workflow/running-inference/inference-devices-and-modes/auto-device-selection.rst index d087e369ff117d..69bdf26eb0cf79 100644 --- a/docs/articles_en/openvino-workflow/running-inference/inference-devices-and-modes/auto-device-selection.rst +++ b/docs/articles_en/openvino-workflow/running-inference/inference-devices-and-modes/auto-device-selection.rst @@ -61,15 +61,14 @@ How AUTO Works ############## To put it simply, when loading the model to the first device on the list fails, AUTO will try to load it to the next device in line, until one of them succeeds. -What is important, **AUTO starts inference with the CPU of the system by default**, as it provides very low latency and can start inference with no additional delays. +What is important, **AUTO starts inference with the CPU of the system by default unless there is model cached for the best suited device**, as it provides very low latency and can start inference with no additional delays. While the CPU is performing inference, AUTO continues to load the model to the device best suited for the purpose and transfers the task to it when ready. This way, the devices which are much slower in compiling models, GPU being the best example, do not impact inference at its initial stages. For example, if you use a CPU and a GPU, the first-inference latency of AUTO will be better than that of using GPU alone. Note that if you choose to exclude CPU from the priority list or disable the initial CPU acceleration feature via ``ov::intel_auto::enable_startup_fallback``, it will be -unable to support the initial model compilation stage. The models with dynamic -input/output or :doc:`stateful operations <../stateful-models>` +unable to support the initial model compilation stage. The models with :doc:`stateful operations <../stateful-models>` will be loaded to the CPU if it is in the candidate list. Otherwise, these models will follow the normal flow and be loaded to the device based on priority. @@ -510,8 +509,9 @@ For more information, refer to the :doc:`Benchmark Tool <../../../learn-openvino Additional Resources #################### -- :doc:`Debugging AUTO ` -- :doc:`Running on Multiple Devices Simultaneously ` -- :doc:`Inference Devices and Modes <../inference-devices-and-modes>` +* `Automatic Device Selection with OpenVINO™ Notebook `__ +* :doc:`Debugging AUTO ` +* :doc:`Running on Multiple Devices Simultaneously ` +* :doc:`Inference Devices and Modes <../inference-devices-and-modes>` diff --git a/docs/articles_en/openvino-workflow/running-inference/inference-devices-and-modes/cpu-device.rst b/docs/articles_en/openvino-workflow/running-inference/inference-devices-and-modes/cpu-device.rst index d95f97959f5b2a..e2f87fed9d3c60 100644 --- a/docs/articles_en/openvino-workflow/running-inference/inference-devices-and-modes/cpu-device.rst +++ b/docs/articles_en/openvino-workflow/running-inference/inference-devices-and-modes/cpu-device.rst @@ -7,7 +7,7 @@ CPU Device :maxdepth: 1 :hidden: - cpu-device/performance-hint-and-threads-scheduling + cpu-device/performance-hint-and-thread-scheduling .. meta:: :description: The CPU plugin in the Intel® Distribution of OpenVINO™ toolkit @@ -234,6 +234,8 @@ This can be achieved by specifying ``MULTI:CPU,GPU.0`` as a target device in cas For more details, see the :doc:`Multi-device execution ` article. +.. _multi_stream_execution: + Multi-stream Execution +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ @@ -242,7 +244,7 @@ property is set for CPU plugin, then multiple streams are created for the model. host thread, which means that incoming infer requests can be processed simultaneously. Each stream is pinned to its own group of physical cores with respect to NUMA nodes physical memory usage to minimize overhead on data transfer between NUMA nodes. -For more details, see the :doc:`optimization guide <../optimize-inference>` and :doc:`threads scheduling introduction `. +For more details, see the :doc:`optimization guide <../optimize-inference>` and :doc:`thread scheduling introduction `. .. note:: @@ -390,7 +392,7 @@ Multi-Threading Optimization CPU inference will infer an input or multiple inputs in parallel on multiple logical processors. -For more details, see the :doc:`threads scheduling introduction `. +For more details, see the :doc:`thread scheduling introduction `. Denormals Optimization diff --git a/docs/articles_en/openvino-workflow/running-inference/inference-devices-and-modes/cpu-device/performance-hint-and-threads-scheduling.rst b/docs/articles_en/openvino-workflow/running-inference/inference-devices-and-modes/cpu-device/performance-hint-and-thread-scheduling.rst similarity index 72% rename from docs/articles_en/openvino-workflow/running-inference/inference-devices-and-modes/cpu-device/performance-hint-and-threads-scheduling.rst rename to docs/articles_en/openvino-workflow/running-inference/inference-devices-and-modes/cpu-device/performance-hint-and-thread-scheduling.rst index 3087bcf2d95783..dc158fe9352042 100644 --- a/docs/articles_en/openvino-workflow/running-inference/inference-devices-and-modes/cpu-device/performance-hint-and-threads-scheduling.rst +++ b/docs/articles_en/openvino-workflow/running-inference/inference-devices-and-modes/cpu-device/performance-hint-and-thread-scheduling.rst @@ -1,50 +1,51 @@ - -Performance Hints and Threads Scheduling +Performance Hints and Thread Scheduling ======================================== .. meta:: - :description: The Threads Scheduling of CPU plugin in OpenVINO™ Runtime + :description: Thread Scheduling of the CPU plugin in OpenVINO™ Runtime detects CPU architecture and sets low-level properties based on performance hints automatically. -While all supported devices in OpenVINO offer low-level performance settings, it is advisable -not to use these settings widely unless targeting specific platforms and models. The recommended -approach is to configure performance in OpenVINO Runtime using the high-level performance hints -property ``ov::hint::performance_mode``. Performance hints ensure optimal portability and -scalability of applications across various platforms and models. - -To simplify the configuration of hardware devices, OpenVINO offers two performance hints: the -latency hint ``ov::hint::PerformanceMode::LATENCY`` and the throughput hint -``ov::hint::PerformanceMode::THROUGHPUT``. +To simplify the configuration of hardware devices, it is recommended to use the +ov::hint::PerformanceMode::LATENCY and ov::hint::PerformanceMode::THROUGHPUT +:doc:`high-level performance hints <../../optimize-inference/high-level-performance-hints>`. Both performance hints ensure optimal portability +and scalability of applications across various platforms and models. - ``ov::inference_num_threads`` limits the number of logical processors used for CPU inference. If the number set by the user is greater than the number of logical processors on the platform, the multi-threading scheduler only uses the platform number for CPU inference. - ``ov::num_streams`` limits the number of infer requests that can be run in parallel. If the number set by the user is greater than the number of inference threads, multi-threading - scheduler only uses the number of inference threads to ensure that there is at least one thread per stream. -- ``ov::hint::scheduling_core_type`` specifies the type of CPU cores for CPU inference when the user runs - inference on a hybird platform that includes both Performance-cores (P-cores) and Efficient-cores (E-cores). - If the user platform only has one type of CPU core, this property has no effect, and CPU inference always uses this unique core type. + scheduler only uses the number of inference threads to ensure that there is at least + one thread per stream. +- ``ov::hint::scheduling_core_type`` specifies the type of CPU cores for CPU inference when + the user runs inference on a hybird platform that includes both Performance-cores (P-cores) + and Efficient-cores (E-cores). If the user platform only has one type of CPU core, this + property has no effect, and CPU inference always uses this unique core type. - ``ov::hint::enable_hyper_threading`` limits the use of one or two logical processors per CPU core when the platform has CPU hyperthreading enabled. - If there is only one logical processor per CPU core, such as Efficient-cores, this property has no effect, and CPU inference uses all logical processors. + If there is only one logical processor per CPU core, such as Efficient-cores, this + property has no effect, and CPU inference uses all logical processors. - ``ov::hint::enable_cpu_pinning`` enables CPU pinning during CPU inference. - If the user enables this property but the inference scenario does not support it, this property will be disabled during model compilation. + If the user enables this property but the inference scenario does not support it, this + property will be disabled during model compilation. -For additional details on the above configurations, refer to `Multi-stream Execution `__. +For additional details on the above configurations, refer to +:ref:`Multi-stream Execution `. Latency Hint -################################### +##################### In this scenario, the default setting of ``ov::hint::scheduling_core_type`` is determined by the model precision and the ratio of P-cores and E-cores. .. note:: - P-cores is short for Performance-cores and E-cores stands for Efficient-cores. These types of cores are available starting with the 12th Gen Intel® Core™ processors. + P-cores is short for Performance-cores and E-cores stands for Efficient-cores. These + types of cores are available starting with the 12th Gen Intel® Core™ processors. + +.. _core_type_latency: -.. _Core Type Table of Latency Hint: +----------------------------+---------------------+---------------------+ | | INT8 Model | FP32 Model | +============================+=====================+=====================+ @@ -57,7 +58,8 @@ the model precision and the ratio of P-cores and E-cores. .. note:: - Both P-cores and E-cores may be used for any configuration starting with 14th Gen Intel® Core™ processors on Windows. + Both P-cores and E-cores may be used for any configuration starting with 14th Gen Intel® + Core™ processors on Windows. Then the default settings for low-level performance properties on Windows and Linux are as follows: @@ -68,7 +70,7 @@ Then the default settings for low-level performance properties on Windows and Li +--------------------------------------+------------------------------------------------------------------------+--------------------------------------------------------------------+ | ``ov::inference_num_threads`` | is equal to the number of P-cores or P-cores+E-cores on one socket | is equal to the number of P-cores or P-cores+E-cores on one socket | +--------------------------------------+------------------------------------------------------------------------+--------------------------------------------------------------------+ -| ``ov::hint::scheduling_core_type`` | `Core Type Table of Latency Hint`_ | `Core Type Table of Latency Hint`_ | +| ``ov::hint::scheduling_core_type`` | :ref:`Core Type Table of Latency Hint ` | :ref:`Core Type Table of Latency Hint ` | +--------------------------------------+------------------------------------------------------------------------+--------------------------------------------------------------------+ | ``ov::hint::enable_hyper_threading`` | No | No | +--------------------------------------+------------------------------------------------------------------------+--------------------------------------------------------------------+ @@ -77,13 +79,20 @@ Then the default settings for low-level performance properties on Windows and Li .. note:: - - ``ov::hint::scheduling_core_type`` may be adjusted for a particular inferred model on a specific platform based on internal heuristics to guarantee optimal performance. - - Both P-cores and E-cores are used for the Latency Hint on Intel® Core™ Ultra Processors on Windows, except in the case of large language models. - - In case hyper-threading is enabled, two logical processors share the hardware resources of one CPU core. OpenVINO does not expect to use both logical processors in one stream for a single infer request. So ``ov::hint::enable_hyper_threading`` is set to ``No`` in this scenario. - - ``ov::hint::enable_cpu_pinning`` is disabled by default on Windows and macOS, and enabled on Linux. Such default settings are aligned with typical workloads running in the corresponding environments to guarantee better out-of-the-box (OOB) performance. + - ``ov::hint::scheduling_core_type`` may be adjusted for a particular inferred model on a + specific platform based on internal heuristics to guarantee optimal performance. + - Both P-cores and E-cores are used for the Latency Hint on Intel® Core™ Ultra Processors + on Windows, except in the case of large language models. + - In case hyper-threading is enabled, two logical processors share the hardware resources + of one CPU core. OpenVINO does not expect to use both logical processors in one stream + for a single infer request. So ``ov::hint::enable_hyper_threading`` is set to + ``No`` in this scenario. + - ``ov::hint::enable_cpu_pinning`` is disabled by default on Windows and macOS, and + enabled on Linux. Such default settings are aligned with typical workloads running + in the corresponding environments to guarantee better out-of-the-box (OOB) performance. Throughput Hint -###################################### +##################### In this scenario, thread scheduling first evaluates the memory pressure of the model being inferred on the current platform, and determines the number of threads per stream, as shown below. @@ -99,7 +108,8 @@ inferred on the current platform, and determines the number of threads per strea +-----------------+-----------------------+ Then the value of ``ov::num_streams`` is calculated by dividing ``ov::inference_num_threads`` -by the number of threads per stream. The default settings for low-level performance properties on Windows and Linux are as follows: +by the number of threads per stream. The default settings for low-level performance +properties on Windows and Linux are as follows: +--------------------------------------+-------------------------------+-------------------------------+ | Property | Windows | Linux | @@ -117,13 +127,15 @@ by the number of threads per stream. The default settings for low-level performa .. note:: - - By default, different core types are not mixed within a single stream in this scenario. The cores from different NUMA nodes are not mixed within a single stream. + By default, different core types are not mixed within a single stream in this scenario. + The cores from different NUMA nodes are not mixed within a single stream. Multi-Threading Optimization -############################################## +############################ The following properties can be used to limit the available CPU resources for model inference. -If the platform or operating system supports this behavior, the OpenVINO Runtime will perform multi-threading scheduling based on the limited available CPU. +If the platform or operating system supports this behavior, the OpenVINO Runtime will +perform multi-threading scheduling based on the limited available CPU. - ``ov::inference_num_threads`` - ``ov::hint::scheduling_core_type`` @@ -148,11 +160,13 @@ If the platform or operating system supports this behavior, the OpenVINO Runtime .. note:: - ``ov::hint::scheduling_core_type`` and ``ov::hint::enable_hyper_threading`` only support Intel® x86-64 CPU on Linux and Windows in the current release. + ``ov::hint::scheduling_core_type`` and ``ov::hint::enable_hyper_threading`` only support + Intel® x86-64 CPU on Linux and Windows in the current release. In some use cases, OpenVINO Runtime will enable CPU thread pinning by default for better performance. Users can also turn this feature on or off using the property ``ov::hint::enable_cpu_pinning``. -Disabling thread pinning may be beneficial in complex applications where several workloads are executed in parallel. +Disabling thread pinning may be beneficial in complex applications where several workloads +are executed in parallel. .. tab-set:: @@ -172,4 +186,4 @@ Disabling thread pinning may be beneficial in complex applications where several For details on multi-stream execution check the -:doc:`optimization guide <../../optimize-inference/optimizing-throughput/advanced_throughput_options>`. \ No newline at end of file +:doc:`optimization guide <../../optimize-inference/optimizing-throughput/advanced_throughput_options>`. diff --git a/docs/articles_en/openvino-workflow/running-inference/inference-devices-and-modes/gpu-device.rst b/docs/articles_en/openvino-workflow/running-inference/inference-devices-and-modes/gpu-device.rst index 0e3506484b42ae..adad5dcf1e594d 100644 --- a/docs/articles_en/openvino-workflow/running-inference/inference-devices-and-modes/gpu-device.rst +++ b/docs/articles_en/openvino-workflow/running-inference/inference-devices-and-modes/gpu-device.rst @@ -476,6 +476,7 @@ Since OpenVINO relies on the OpenCL kernels for the GPU implementation, many gen Additional Resources ####################################### +* `Working with GPUs in OpenVINO™ Notebook `__ * :doc:`Inference Devices and Modes <../inference-devices-and-modes>`. * :doc:`Optimization guide <../optimize-inference>`. * `GPU plugin developer documentation `__ diff --git a/docs/articles_en/openvino-workflow/running-inference/inference-devices-and-modes/hetero-execution.rst b/docs/articles_en/openvino-workflow/running-inference/inference-devices-and-modes/hetero-execution.rst index 0ba7d26436ee9e..24f820755555b0 100644 --- a/docs/articles_en/openvino-workflow/running-inference/inference-devices-and-modes/hetero-execution.rst +++ b/docs/articles_en/openvino-workflow/running-inference/inference-devices-and-modes/hetero-execution.rst @@ -22,7 +22,7 @@ Execution via the heterogeneous mode can be divided into two independent steps: These two steps are not interconnected and affinities can be set in one of two ways, used separately or in combination (as described below): in the ``manual`` or the ``automatic`` mode. Defining and Configuring the Hetero Device -++++++++++++++++++++++++++++++++++++++++++ +########################################## Following the OpenVINO™ naming convention, the Hetero execution plugin is assigned the label of ``"HETERO".`` It may be defined with no additional parameters, resulting in defaults being used, or configured further with the following setup options: @@ -37,10 +37,10 @@ Following the OpenVINO™ naming convention, the Hetero execution plugin is assi Manual and Automatic modes for assigning affinities -+++++++++++++++++++++++++++++++++++++++++++++++++++ +################################################### The Manual Mode --------------------- ++++++++++++++++++++++ It assumes setting affinities explicitly for all operations in the model using `ov::Node::get_rt_info `__ with the ``"affinity"`` key. @@ -66,7 +66,7 @@ Randomly selecting operations and setting affinities may lead to decrease in mod The Automatic Mode --------------------- ++++++++++++++++++++++ It decides automatically which operation is assigned to which device according to the support from dedicated devices (``GPU``, ``CPU``, etc.) and query model step is called implicitly by Hetero device during model compilation. @@ -92,7 +92,7 @@ It does not take into account device peculiarities such as the inability to infe Using Manual and Automatic Modes in Combination ------------------------------------------------ ++++++++++++++++++++++++++++++++++++++++++++++++ In some cases you may need to consider manually adjusting affinities which were set automatically. It usually serves minimizing the number of total subgraphs to optimize memory transfers. To do it, you need to "fix" the automatically assigned affinities like so: @@ -121,7 +121,7 @@ Importantly, the automatic mode will not work if any operation in a model has it `ov::Core::query_model `__ does not depend on affinities set by a user. Instead, it queries for an operation support based on device capabilities. Configure fallback devices -++++++++++++++++++++++++++ +########################## If you want different devices in Hetero execution to have different device-specific configuration options, you can use the special helper property `ov::device::properties `__: @@ -146,7 +146,7 @@ If you want different devices in Hetero execution to have different device-speci In the example above, the ``GPU`` device is configured to enable profiling data and uses the default execution precision, while ``CPU`` has the configuration property to perform inference in ``fp32``. Handling of Difficult Topologies -++++++++++++++++++++++++++++++++ +################################ Some topologies are not friendly to heterogeneous execution on some devices, even to the point of being unable to execute. For example, models having activation operations that are not supported on the primary device are split by Hetero into multiple sets of subgraphs which leads to suboptimal execution. @@ -154,7 +154,7 @@ If transmitting data from one subgraph to another part of the model in the heter In such cases, you can define the heaviest part manually and set the affinity to avoid sending data back and forth many times during one inference. Analyzing Performance of Heterogeneous Execution -++++++++++++++++++++++++++++++++++++++++++++++++ +################################################ After enabling the ``OPENVINO_HETERO_VISUALIZE`` environment variable, you can dump GraphViz ``.dot`` files with annotations of operations per devices. @@ -186,7 +186,7 @@ Here is an example of the output for Googlenet v1 running on HDDL (device no lon Sample Usage -++++++++++++++++++++ +##################### OpenVINO™ sample programs can use the Heterogeneous execution used with the ``-d`` option: diff --git a/docs/articles_en/openvino-workflow/running-inference/inference-devices-and-modes/npu-device.rst b/docs/articles_en/openvino-workflow/running-inference/inference-devices-and-modes/npu-device.rst index ea39001b4f3fd4..7467328498b2c5 100644 --- a/docs/articles_en/openvino-workflow/running-inference/inference-devices-and-modes/npu-device.rst +++ b/docs/articles_en/openvino-workflow/running-inference/inference-devices-and-modes/npu-device.rst @@ -151,7 +151,8 @@ guaranteed. Additional Resources ############################# -* `Vision colorization Notebook `__ +* `Working with NPUs in OpenVINO™ Notebook `__ +* `Vision colorization Notebook <./../../../notebooks/vision-image-colorization-with-output.html>`__ * `Classification Benchmark C++ Demo `__ * `3D Human Pose Estimation Python Demo `__ * `Object Detection C++ Demo `__ diff --git a/docs/articles_en/openvino-workflow/running-inference/integrate-openvino-with-your-application.rst b/docs/articles_en/openvino-workflow/running-inference/integrate-openvino-with-your-application.rst index ce5e6fd20722a1..222c8760d0a880 100644 --- a/docs/articles_en/openvino-workflow/running-inference/integrate-openvino-with-your-application.rst +++ b/docs/articles_en/openvino-workflow/running-inference/integrate-openvino-with-your-application.rst @@ -432,7 +432,8 @@ To build your project using CMake with the default build tools currently availab Additional Resources #################### -* See the :doc:`OpenVINO Samples <../../learn-openvino/openvino-samples>` page or the `Open Model Zoo Demos `__ page for specific examples of how OpenVINO pipelines are implemented for applications like image classification, text prediction, and many others. +* See the :doc:`OpenVINO Samples <../../learn-openvino/openvino-samples>` page for specific examples of how OpenVINO pipelines are implemented for applications like image classification, text prediction, and many others. +* Models in the OpenVINO IR format on `Hugging Face `__. * :doc:`OpenVINO™ Runtime Preprocessing ` * :doc:`String Tensors ` * :doc:`Using Encrypted Models with OpenVINO <../../documentation/openvino-security/openvino-encrypted-models>` diff --git a/docs/articles_en/openvino-workflow/running-inference/string-tensors.rst b/docs/articles_en/openvino-workflow/running-inference/string-tensors.rst index 438a2a5d9ae259..22223649144d9f 100644 --- a/docs/articles_en/openvino-workflow/running-inference/string-tensors.rst +++ b/docs/articles_en/openvino-workflow/running-inference/string-tensors.rst @@ -10,11 +10,11 @@ String Tensors OpenVINO tensors can hold not only numerical data, like floating-point or integer numbers, but also textual information, represented as one or multiple strings. Such a tensor is called a string tensor and can be passed as input or retrieved as output of a text-processing model, such as -`tokenizers and detokenizers `__. +`tokenizers and detokenizers `__. While this section describes basic API to handle string tensors, more practical examples that leverage both string tensors and OpenVINO tokenizer can be found in -`GenAI Samples `__. +`GenAI Samples `__. Representation diff --git a/docs/dev/build_android.md b/docs/dev/build_android.md index 91cd94644a6e6e..8a42fc6f2be342 100644 --- a/docs/dev/build_android.md +++ b/docs/dev/build_android.md @@ -5,49 +5,119 @@ This article describes how to build OpenVINO for Android operating systems. ## Software requirements - [CMake](https://cmake.org/download/) 3.13 or higher -- Android NDK (this guide has been validated with r20 release) +- [SCons](https://scons.org/pages/download.html) 4.6.0 or higher +- [Android SDK Platform Tools](https://developer.android.com/tools/releases/platform-tools) (this guide has been validated with 30.0.0 release) +- [Android NDK](https://developer.android.com/ndk/downloads) (this guide has been validated with r26 release) ## How to build -1. Download and unpack [Android NDK](https://developer.android.com/ndk/downloads). Let's assume that `~/Downloads` is used as a working folder. +### Create a work directory ```sh - cd ~/Downloads - wget https://dl.google.com/android/repository/android-ndk-r20-linux-x86_64.zip - - unzip android-ndk-r20-linux-x86_64.zip - mv android-ndk-r20 android-ndk + mkdir openvino-android + export OPV_HOME_DIR=${PWD}/openvino-android ``` -2. Create a build folder: +### Download and unpack Android packages +* Download and unpack [Android NDK](https://developer.android.com/ndk/downloads) ```sh - mkdir build - ``` + wget https://dl.google.com/android/repository/android-ndk-r26d-linux.zip --directory-prefix $OPV_HOME_DIR -3. Change working directory to `build` and run `cmake` to create makefiles. Then run `make`. + unzip $OPV_HOME_DIR/android-ndk-r26d-linux.zip -d $OPV_HOME_DIR + mv $OPV_HOME_DIR/android-ndk-r26d $OPV_HOME_DIR/android-ndk + export ANDROID_NDK_PATH=$OPV_HOME_DIR/android-ndk + ``` +* Download and unpack [Android SDK Platform Tools](https://developer.android.com/tools/releases/platform-tools) ```sh - cd build + wget https://dl.google.com/android/repository/platform-tools-latest-linux.zip --directory-prefix $OPV_HOME_DIR - cmake .. \ - -DCMAKE_TOOLCHAIN_FILE=~/Downloads/android-ndk/build/cmake/android.toolchain.cmake \ - -DANDROID_ABI=x86_64 \ - -DANDROID_PLATFORM=21 \ - -DANDROID_STL=c++_shared - - make --jobs=$(nproc --all) + unzip $OPV_HOME_DIR/platform-tools-latest-linux.zip -d $OPV_HOME_DIR + export ANDROID_TOOLS_PATH=$OPV_HOME_DIR/platform-tools ``` +_For Windows and Mac operating systems, the downloading and unpacking steps are similar._ - * `ANDROID_ABI` specifies target architecture: +### Set the environment variables for building + ```sh + export CURRENT_ANDROID_ABI=`$ANDROID_TOOLS_PATH/adb shell getprop ro.product.cpu.abi` + export CURRENT_ANDROID_PLATFORM=30 + export CURRENT_ANDROID_STL=c++_shared + export CURRENT_CMAKE_TOOLCHAIN_FILE=$ANDROID_NDK_PATH/build/cmake/android.toolchain.cmake + ``` +* `ANDROID_ABI` specifies the target architecture: * `x86_64` for x64 build * `armeabi-v7a with NEON` for ARM with NEON support * `arm64-v8a` for ARM 64 bits - * `ANDROID_PLATFORM` - Android API version - * `ANDROID_STL` specifies that shared C++ runtime is used. Copy `~/Downloads/android-ndk/sources/cxx-stl/llvm-libc++/libs/x86_64/libc++_shared.so` from Android NDK along with built binaries +* `ANDROID_PLATFORM` specifies the Android API version. +* `ANDROID_STL` indicates that a shared C++ runtime is used. -4. To reduce the binaries size, use `strip` tool from NDK: +### Build and install OneTBB™ +To improve the parallelism performance of the OpenVINO™ library using OneTBB, it is required to separately build OneTBB for a specific version of the Android NDK: + ```sh + # Clone OneTBB™ repository + git clone --recursive https://github.com/oneapi-src/oneTBB $OPV_HOME_DIR/one-tbb + # Create build and install directory + mkdir $OPV_HOME_DIR/one-tbb-build $OPV_HOME_DIR/one-tbb-install + # Configure OneTBB™ CMake project + cmake -S $OPV_HOME_DIR/one-tbb \ + -B $OPV_HOME_DIR/one-tbb-build \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=$OPV_HOME_DIR/one-tbb-install \ + -DCMAKE_TOOLCHAIN_FILE=$CURRENT_CMAKE_TOOLCHAIN_FILE \ + -DANDROID_ABI=$CURRENT_ANDROID_ABI \ + -DANDROID_PLATFORM=$CURRENT_ANDROID_PLATFORM \ + -DANDROID_STL=$CURRENT_ANDROID_STL \ + -DTBB_TEST=OFF \ + -DCMAKE_SHARED_LINKER_FLAGS="-Wl,--undefined-version" + # Build OneTBB™ project + cmake --build $OPV_HOME_DIR/one-tbb-build --parallel + # Install OneTBB™ project + cmake --install $OPV_HOME_DIR/one-tbb-build + ``` -```bash -~/Downloads/android-ndk/toolchains/llvm/prebuilt/linux-x86_64/x86_64-linux-android/bin/strip openvino/bin/intel64/Release/lib/*.so -``` +### Build and install OpenVINO™ + ```sh + # Clone OpenVINO™ repository + git clone --recursive https://github.com/openvinotoolkit/openvino $OPV_HOME_DIR/openvino + # Create build and install directory + mkdir $OPV_HOME_DIR/openvino-build $OPV_HOME_DIR/openvino-install + # Configure OpenVINO™ CMake project + cmake -S $OPV_HOME_DIR/openvino \ + -B $OPV_HOME_DIR/openvino-build \ + -DCMAKE_INSTALL_PREFIX=$OPV_HOME_DIR/openvino-install \ + -DCMAKE_TOOLCHAIN_FILE=$CURRENT_CMAKE_TOOLCHAIN_FILE \ + -DANDROID_ABI=$CURRENT_ANDROID_ABI \ + -DANDROID_PLATFORM=$CURRENT_ANDROID_PLATFORM \ + -DANDROID_STL=$CURRENT_ANDROID_STL \ + -DTBB_DIR=$OPV_HOME_DIR/one-tbb-install/lib/cmake/TBB + # Build OpenVINO™ project + cmake --build $OPV_HOME_DIR/openvino-build --parallel + # Install OpenVINO™ project + cmake --install $OPV_HOME_DIR/openvino-build + ``` + +### Download the example model + We will use `mobelinet-v3-tf` example model + ```sh + mkdir $OPV_HOME_DIR/mobelinet-v3-tf + wget https://storage.openvinotoolkit.org/repositories/openvino_notebooks/models/mobelinet-v3-tf/FP32/v3-small_224_1.0_float.xml -P $OPV_HOME_DIR/mobelinet-v3-tf/ + wget https://storage.openvinotoolkit.org/repositories/openvino_notebooks/models/mobelinet-v3-tf/FP32/v3-small_224_1.0_float.bin -P $OPV_HOME_DIR/mobelinet-v3-tf/ + ``` + +### Use the ADB tool to run the example model and check the library +_This example is demonstrated for aarch64 architecture_ + ```sh + # Copy OpenVINO™ libraries to android device + $ANDROID_TOOLS_PATH/adb push --sync $OPV_HOME_DIR/openvino-install/runtime/lib/aarch64/* /data/local/tmp/ + # Copy OneTBB libraries to android device + $ANDROID_TOOLS_PATH/adb push --sync $OPV_HOME_DIR/one-tbb-install/lib/* /data/local/tmp/ + # Copy shared STL library to android device + $ANDROID_TOOLS_PATH/adb push --sync $ANDROID_NDK_PATH/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/lib/aarch64-linux-android/libc++_shared.so /data/local/tmp/ + # Copy example model files + $ANDROID_TOOLS_PATH/adb push --sync $OPV_HOME_DIR/mobelinet-v3-tf /data/local/tmp/ + # Copy OpenVINO™ benchmark_app tool to android device + $ANDROID_TOOLS_PATH/adb push --sync $OPV_HOME_DIR/openvino/bin/aarch64/Release/benchmark_app /data/local/tmp/ + # Run OpenVINO™ benchmark_app tool + $ANDROID_TOOLS_PATH/adb shell "LD_LIBRARY_PATH=/data/local/tmp ./data/local/tmp/benchmark_app -m /data/local/tmp/mobelinet-v3-tf/v3-small_224_1.0_float.xml -hint latency" + ``` ## See also @@ -55,4 +125,3 @@ This article describes how to build OpenVINO for Android operating systems. * [OpenVINO Developer Documentation](index.md) * [OpenVINO Get Started](./get_started.md) * [How to build OpenVINO](build.md) - diff --git a/docs/dev/build_linux.md b/docs/dev/build_linux.md index 814deaa3deec3c..ffaa78d75653c4 100644 --- a/docs/dev/build_linux.md +++ b/docs/dev/build_linux.md @@ -12,13 +12,7 @@ The software was validated on: - [CMake](https://cmake.org/download/) 3.13 or higher - GCC 7.5 or higher to build OpenVINO Runtime - Python 3.8 - 3.11 for OpenVINO Runtime Python API -- (Optional) Install Intel® Graphics Compute Runtime for OpenCL™ Driver package to enable inference on Intel integrated GPUs. Select a driver package from the table below depending on what version of Ubuntu you are installing on. - - | Ubuntu | Driver package | - | --- | ----------- | - | 22.04 | [23.13.26032.30](https://github.com/intel/compute-runtime/releases/tag/23.13.26032.30) | - | 20.04 | [22.24.23453](https://github.com/intel/compute-runtime/releases/tag/22.24.23453) | - | 18.04 | [21.38.21026](https://github.com/intel/compute-runtime/releases/tag/21.38.21026) | +- (Optional) Install Intel® Graphics Compute Runtime for OpenCL™ Driver package to enable inference on Intel integrated GPUs. ## How to build diff --git a/docs/dev/build_windows.md b/docs/dev/build_windows.md index eaf695f898ba18..2515000e5ec4f0 100644 --- a/docs/dev/build_windows.md +++ b/docs/dev/build_windows.md @@ -6,20 +6,20 @@ Supported configurations: - Windows 10 x86 64-bit or higher with Visual Studio 2019 or higher build for X64 architecture. - Windows on ARM (shortly WoA) to build for ARM64 architecture. OpenVINO was validated on [Windows DevKit 2023](https://developer.qualcomm.com/hardware/windows-on-snapdragon/windows-dev-kit-2023) -## Software requirements +## Software requirements - [CMake](https://cmake.org/download/) 3.13 or higher - Microsoft Visual Studio 2019 or higher, version 16.3 or later - > **NOTE**: Native Microsoft Visual Studio for WoA is available since 2022. + > **NOTE**: Native Microsoft Visual Studio for WoA has been available since version 3.11. - Python 3.8 - 3.11 for OpenVINO Runtime Python API - > **NOTE**: Python for ARM64 is available since [3.11](https://www.python.org/downloads/windows/) version. + > **NOTE**: Python for ARM64 is available since [3.11](https://www.python.org/downloads/windows/) version. - [Git for Windows*] - (Windows on ARM only) [LLVM for Windows on ARM (WoA)](https://github.com/llvm/llvm-project/releases/download/llvmorg-15.0.6/LLVM-15.0.6-woa64.exe) - > **NOTE**: After installation, make sure `clang-cl` compiler is available from `PATH`. + > **NOTE**: After installation, make sure `clang-cl` compiler is available from `PATH`. ## How to build -> **NOTE**: By default, the build enables the OpenVINO Runtime GPU plugin to infer models on your Intel® Processor Graphics. This requires you to download and install the Intel® Graphics Driver for Windows (26.20) [driver package](https://www.intel.com/content/www/us/en/download/19344/intel-graphics-windows-dch-drivers.html) before running the build. If you don't want to use the GPU plugin, use the `-DENABLE_INTEL_GPU=OFF` CMake build option and skip the installation of the Intel® Graphics Driver. +> **NOTE**: By default, the build enables the OpenVINO Runtime GPU plugin to infer models on your Intel® Processor Graphics. This requires you to download and install the [Intel® Graphics Driver for Windows](https://www.intel.com/content/www/us/en/download/19344/intel-graphics-windows-dch-drivers.html) before running the build. If you don't want to use the GPU plugin, use the `-DENABLE_INTEL_GPU=OFF` CMake build option and skip the installation of the Intel® Graphics Driver. 1. Clone submodules: ```sh @@ -37,12 +37,12 @@ Supported configurations: ```sh cmake -G "Visual Studio 17 2022" ``` - + > **HINT**: **Generating PDB Files and Debugging Your Build**
> If you intend to generate PDB files and debug your build, it is essential to set the CMake build type appropriately. > You should utilize one of the following CMake build type options:
>* `-DCMAKE_BUILD_TYPE=RelWithDebInfo`: This option generates PDB files with release information, making it suitable for debugging optimized builds.
- >* `-DCMAKE_BUILD_TYPE=Debug`: This option generates PDB files optimized for debugging, providing comprehensive debugging information. + >* `-DCMAKE_BUILD_TYPE=Debug`: This option generates PDB files optimized for debugging, providing comprehensive debugging information. 4. Build generated solution in Visual Studio or run `cmake --build . --config Release --verbose -j` to build from the command line. View the number of available processing units with `WMIC cpu get numberofLogicalProcessors`. Be aware that this process may take some time. @@ -62,7 +62,7 @@ Supported configurations: 3. After the build process finishes, export the newly built Python libraries to the user environment variables: ``` set PYTHONPATH=/bin//Release/python;%PYTHONPATH% - set OPENVINO_LIB_PATHS=/bin//Release;%OPENVINO_LIB_PATH% + set OPENVINO_LIB_PATHS=/bin//Release;/temp/tbb/bin ``` or install the wheel with pip: ``` diff --git a/docs/dev/cmake_options_for_custom_compilation.md b/docs/dev/cmake_options_for_custom_compilation.md index a5184dbc460979..d370285f99c8b4 100644 --- a/docs/dev/cmake_options_for_custom_compilation.md +++ b/docs/dev/cmake_options_for_custom_compilation.md @@ -42,6 +42,8 @@ This document provides description and default values for CMake options that can * `ON` is default. * `ENABLE_OV_PYTORCH_FRONTEND` enables [PyTorch] frontend plugin for OpenVINO Runtime: * `ON` is default. + * `ENABLE_OV_JAX_FRONTEND` enables [JAX] frontend plugin for OpenVINO Runtime: + * `ON` is default. * `ENABLE_OV_IR_FRONTEND` enables OpenVINO Intermediate Representation frontend plugin for OpenVINO Runtime: * `ON` is default. * `OPENVINO_EXTRA_MODULES` specifies path to add extra OpenVINO modules to the build. diff --git a/docs/dev/ov_dependencies_24_2.txt b/docs/dev/ov_dependencies_24_2.txt new file mode 100644 index 00000000000000..0f24303ec47d0c --- /dev/null +++ b/docs/dev/ov_dependencies_24_2.txt @@ -0,0 +1,374 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +#This file provides a comprehensive list of all dependencies of OpenVINO 2024.2 +#The file is part of the automation pipeline for posting OpenVINO IR models on the HuggingFace Hub, including OneBOM dependency checks. + + +absl-py +accuracy-checker +addict +alabaster +appnope +argparse +astor +astpretty +astroid +astunparse +atomicwrites +attrs +babel +backcall +backports-functools-lru-cache +bandit +beautifulsoup4 +black +Boost +Bootstrap +BrainMaGe +cached-property +cachetools +certifi +chardet +charset-normalizer +chosen-js +clang +click +CNPY +colorama +ComputeLibrary +coverage +cycler +cython +dataclasses +decorator +Deformable-ConvNets +defusedxml +Detectron2 +distlib +distro +docopt +docutils +Doxygen +Vovkos doxyrest +enum34 +eradicate +execnet +fastjsonschema +Ffmpeg +filelock +flake8 +flake8_builtins +flake8_coding +flake8_commas +flake8_pep3101 +flake8_quotes +flake8-annotations-complexity +flake8-broken-line +flake8-bugbear +flake8-class-attributes-order +flake8-comprehensions +flake8-debugger +flake8-docstrings +flake8-eradicate +flake8-executable +flake8-expression-complexity +flake8-plugin-utils +flake8-print +flake8-pytest-style +flake8-rst-docstrings +flake8-string-format +flake8-variables-names +flatbuffers +fonttools +freetype +funcsigs +functools32 +future +futures +gast +generator +gflags +gitdb +gitpython +google-auth +google-auth-oauthlib +google cpplint +google fonts +google-java-format +google-pasta +Google gtest-parallel +googletest +graphviz +grpcio +Gstreamer +h5py +hamcrest-core +Hyperopt +idna +idna +imagesize +importlib-metadata +importlib-resources +import-order +iniconfig +INTEL/CAFFE +Intel Integrated Performance Primitives - Intel IPP +intel/IntelSEAPI +intel/libva +ipython +isort +itt API +jedi +jinja2 +json +json-c +jsonschema +pboettch json-schema-validator +linjackson78 jstyleson +John Hauser Berkeley SoftFloat +jquery - jquery/jquery +Junit +keras-nightly +keras-preprocessing +kiwisolver +lazy-object-proxy +libjpeg-turbo +Libpng +Llohse libnpy +llvm +lxml +mando +facebookresearch/maskrcnn-benchmark +markdown +markupsafe +mathjax mathjax +matplotlib +mccabe +mistune +mongo-python-driver +more-itertools +mxnet +mypy +mypy-extensions +networkx +nibabel +Nithinn ncc +numpy +oauthlib +OneDNN +OneTBB +onnx +onnxruntime +Open Model Zoo +omz-tools +OpenCL +OpenCL headers +OpenCL ICD Loader +OpenCL-hpp headers +OpenCV +opencv-python +OpenJDK +OpenVINO OSS +openvino-telemetry +opt-einsum +packaging +paddlepaddle +PapaParse +pandas +parso +pathlib2 +pathspec +pbr +pep8-naming +pexpect +pickleshare +pillow +platformdirs +pluggy +pluggy +progress +prompt-toolkit +protobuf +ptyprocess +pugixml +py +pyasn1 +pyasn1-modules +pycodestyle +py-cpuinfo +py-leveldb +Pybind11 +pydata-sphinx-theme +pydocstyle +pyenchant +pyflakes +pygments +pylint +pymongo +pyparsing +pyrsistent +pytest +pytest-dependency +pytest-forked +pytest-html +pytest-metadata +pytest-metadata +pytest-timeout +pytest-xdist +Pytorch +Python +python-attrs +python-dateutil +python-fastjsonschema +python pandas +Python-pillow +pytz +pyyaml +radon +Rapidjson +requests +requests-oauthlib +restructuredtext-lint +retrying +Roboto Fonts +rsa +ruGPT3Medium +safestringlib +scikit-build +scipy +setuptools +six +smmap +snowballstemmer +soupsieve +sphinx +sphinxcontrib-applehelp +sphinxcontrib-devhelp +sphinxcontrib-htmlhelp +sphinxcontrib-jsmath +sphinxcontrib-qthelp +sphinxcontrib-serializinghtml +sphinx-copybutton +sphinx-inline-tabs +sphinx-panels +sphinx-sitemap +stevedore +tensorboard +tensorboard-data-server +tensorboard-plugin-wit +tensorflow +tensorflow +tensorflow-estimator +termcolor +test-generator +threading_building_blocks +foutaise/texttable/ +toml +tomli +Torch7 +tox +traitlets +Transformers +typed-ast +types-pkg_resources +typing +typing-extensions +urllib3 +Viewer +virtualenv +Torchvision +wcwidth +werkzeug +wheel +wrapt +xbyak +zipp +Zlib +backports +contextlib2 +dill +keras +singledispatch +Fluid +libusb (for MYRIAD) +libGNA (for GNA) +Benchmark +GitPython +huggingface/transformers +Hamcrest JavaHamcrest +Openvinotoolkit model-server +Openvinotoolkit nncf +pytorch/vision +Sphinx documentation builder +FlatBuffers +POCO C++ Libraries +Google Mock +Google Snappy +Microsoft onnxruntime +ML commons/inference +take-action +bitsandbytes +diffusers +einops +optimum-intel +tiktoken +timm +tokenizers +torch +optimum +psutil +vscode-extension-samples +fast_tokenizer +re2 +icu4c +sentencepiece +auto-gptq +transformers-stream-generator +eigen3 +gtk3-devel +gstreamer1-devel +gstreamer1-plugins-base-devel +ffmpeg-devel +libmfx-devel +cmake +git +python38-devel +python38-pip +gcc-c++ +gcc +python3-dev +python3-pip +ninja-build +libgtk-3-dev +libpng-dev +libjpeg-dev +libwebp-dev +libtiff5-dev +libopenexr-dev +libopenblas-dev +libx11-dev +libavutil-dev +libavcodec-dev +libavformat-dev +libswscale-dev +libavresample-dev +libtbb2 +libssl-dev +libva-dev +libmfx-dev +libgstreamer1.0-dev +libgstreamer-plugins-base1.0-dev +epel-release-latest-8.noarch.rpm +rpmfusion-free-release-8.noarch.rpm +pytest-repeat 0.9.3 +pip +make +httpx +ultralytics +llama.cpp +lixsmm + + + + + diff --git a/docs/dev/static_libaries.md b/docs/dev/static_libaries.md index 3570b4532d9157..0f694cda73f07d 100644 --- a/docs/dev/static_libaries.md +++ b/docs/dev/static_libaries.md @@ -47,6 +47,7 @@ cmake -DENABLE_INTEL_GPU=OFF \ -DENABLE_OV_TF_FRONTEND=OFF \ -DENABLE_OV_TF_LITE_FRONTEND=OFF \ -DENABLE_OV_PYTORCH_FRONTEND=OFF \ + -DENABLE_OV_JAX_FRONTEND=OFF \ -DENABLE_INTEL_CPU=ON \ -DENABLE_OV_IR_FRONTEND=ON ``` diff --git a/docs/nbdoc/consts.py b/docs/nbdoc/consts.py index 6545e590fd010e..f40a2a2ff9705a 100644 --- a/docs/nbdoc/consts.py +++ b/docs/nbdoc/consts.py @@ -6,7 +6,7 @@ repo_owner = "openvinotoolkit" repo_name = "openvino_notebooks" repo_branch = "tree/main" -artifacts_link = "http://repository.toolbox.iotg.sclab.intel.com/projects/ov-notebook/0.1.0-latest/20240515220822/dist/rst_files/" +artifacts_link = "http://repository.toolbox.iotg.sclab.intel.com/projects/ov-notebook/0.1.0-latest/20240619220807/dist/rst_files/" blacklisted_extensions = ['.xml', '.bin'] notebooks_repo = "https://github.com/openvinotoolkit/openvino_notebooks/blob/latest/" notebooks_binder = "https://mybinder.org/v2/gh/openvinotoolkit/openvino_notebooks/HEAD?filepath=" diff --git a/docs/notebooks/3D-pose-estimation-with-output.rst b/docs/notebooks/3D-pose-estimation-with-output.rst index fda23f07b57c4f..fc8fe233d35788 100644 --- a/docs/notebooks/3D-pose-estimation-with-output.rst +++ b/docs/notebooks/3D-pose-estimation-with-output.rst @@ -52,7 +52,7 @@ Prerequisites -**The ``pythreejs`` extension may not display properly when using a +**The** ``pythreejs`` **extension may not display properly when using a Jupyter Notebook release. Therefore, it is recommended to use Jupyter Lab instead.** @@ -67,84 +67,84 @@ Lab instead.** Collecting pythreejs Using cached pythreejs-2.4.2-py3-none-any.whl.metadata (5.4 kB) Collecting openvino-dev>=2024.0.0 - Using cached openvino_dev-2024.1.0-15008-py3-none-any.whl.metadata (16 kB) + Using cached openvino_dev-2024.2.0-15519-py3-none-any.whl.metadata (16 kB) Collecting opencv-python - Using cached opencv_python-4.9.0.80-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (20 kB) + Using cached opencv_python-4.10.0.84-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (20 kB) Collecting torch - Using cached https://download.pytorch.org/whl/cpu/torch-2.3.0%2Bcpu-cp38-cp38-linux_x86_64.whl (190.4 MB) + Using cached https://download.pytorch.org/whl/cpu/torch-2.3.1%2Bcpu-cp38-cp38-linux_x86_64.whl (190.4 MB) Collecting onnx - Using cached onnx-1.16.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (16 kB) - Requirement already satisfied: ipywidgets>=7.2.1 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from pythreejs) (8.1.2) + Using cached onnx-1.16.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (16 kB) + Requirement already satisfied: ipywidgets>=7.2.1 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-708/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from pythreejs) (8.1.3) Collecting ipydatawidgets>=1.1.1 (from pythreejs) Using cached ipydatawidgets-4.3.5-py2.py3-none-any.whl.metadata (1.4 kB) Collecting numpy (from pythreejs) Using cached numpy-1.24.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (5.6 kB) - Requirement already satisfied: traitlets in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from pythreejs) (5.14.3) - Requirement already satisfied: defusedxml>=0.7.1 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino-dev>=2024.0.0) (0.7.1) + Requirement already satisfied: traitlets in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-708/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from pythreejs) (5.14.3) + Requirement already satisfied: defusedxml>=0.7.1 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-708/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino-dev>=2024.0.0) (0.7.1) Collecting networkx<=3.1.0 (from openvino-dev>=2024.0.0) Using cached networkx-3.1-py3-none-any.whl.metadata (5.3 kB) Collecting openvino-telemetry>=2023.2.1 (from openvino-dev>=2024.0.0) Using cached openvino_telemetry-2024.1.0-py3-none-any.whl.metadata (2.3 kB) - Requirement already satisfied: packaging in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino-dev>=2024.0.0) (24.0) - Requirement already satisfied: pyyaml>=5.4.1 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino-dev>=2024.0.0) (6.0.1) - Requirement already satisfied: requests>=2.25.1 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino-dev>=2024.0.0) (2.31.0) - Collecting openvino==2024.1.0 (from openvino-dev>=2024.0.0) - Using cached openvino-2024.1.0-15008-cp38-cp38-manylinux2014_x86_64.whl.metadata (8.8 kB) + Requirement already satisfied: packaging in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-708/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino-dev>=2024.0.0) (24.1) + Requirement already satisfied: pyyaml>=5.4.1 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-708/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino-dev>=2024.0.0) (6.0.1) + Requirement already satisfied: requests>=2.25.1 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-708/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from openvino-dev>=2024.0.0) (2.32.0) + Collecting openvino==2024.2.0 (from openvino-dev>=2024.0.0) + Using cached openvino-2024.2.0-15519-cp38-cp38-manylinux2014_x86_64.whl.metadata (8.9 kB) Collecting filelock (from torch) - Using cached filelock-3.14.0-py3-none-any.whl.metadata (2.8 kB) - Requirement already satisfied: typing-extensions>=4.8.0 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch) (4.11.0) + Downloading filelock-3.15.3-py3-none-any.whl.metadata (2.9 kB) + Requirement already satisfied: typing-extensions>=4.8.0 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-708/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch) (4.12.2) Collecting sympy (from torch) - Using cached https://download.pytorch.org/whl/sympy-1.12-py3-none-any.whl (5.7 MB) - Requirement already satisfied: jinja2 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch) (3.1.4) + Using cached sympy-1.12.1-py3-none-any.whl.metadata (12 kB) + Requirement already satisfied: jinja2 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-708/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from torch) (3.1.4) Collecting fsspec (from torch) - Downloading fsspec-2024.5.0-py3-none-any.whl.metadata (11 kB) + Using cached fsspec-2024.6.0-py3-none-any.whl.metadata (11 kB) Collecting protobuf>=3.20.2 (from onnx) - Using cached protobuf-5.26.1-cp37-abi3-manylinux2014_x86_64.whl.metadata (592 bytes) + Using cached protobuf-5.27.1-cp38-abi3-manylinux2014_x86_64.whl.metadata (592 bytes) Collecting traittypes>=0.2.0 (from ipydatawidgets>=1.1.1->pythreejs) Using cached traittypes-0.2.1-py2.py3-none-any.whl.metadata (1.0 kB) - Requirement already satisfied: comm>=0.1.3 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipywidgets>=7.2.1->pythreejs) (0.2.2) - Requirement already satisfied: ipython>=6.1.0 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipywidgets>=7.2.1->pythreejs) (8.12.3) - Requirement already satisfied: widgetsnbextension~=4.0.10 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipywidgets>=7.2.1->pythreejs) (4.0.10) - Requirement already satisfied: jupyterlab-widgets~=3.0.10 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipywidgets>=7.2.1->pythreejs) (3.0.10) - Requirement already satisfied: charset-normalizer<4,>=2 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests>=2.25.1->openvino-dev>=2024.0.0) (3.3.2) - Requirement already satisfied: idna<4,>=2.5 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests>=2.25.1->openvino-dev>=2024.0.0) (3.7) - Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests>=2.25.1->openvino-dev>=2024.0.0) (2.2.1) - Requirement already satisfied: certifi>=2017.4.17 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests>=2.25.1->openvino-dev>=2024.0.0) (2024.2.2) - Requirement already satisfied: MarkupSafe>=2.0 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from jinja2->torch) (2.1.5) - Collecting mpmath>=0.19 (from sympy->torch) + Requirement already satisfied: comm>=0.1.3 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-708/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipywidgets>=7.2.1->pythreejs) (0.2.2) + Requirement already satisfied: ipython>=6.1.0 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-708/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipywidgets>=7.2.1->pythreejs) (8.12.3) + Requirement already satisfied: widgetsnbextension~=4.0.11 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-708/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipywidgets>=7.2.1->pythreejs) (4.0.11) + Requirement already satisfied: jupyterlab-widgets~=3.0.11 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-708/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipywidgets>=7.2.1->pythreejs) (3.0.11) + Requirement already satisfied: charset-normalizer<4,>=2 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-708/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests>=2.25.1->openvino-dev>=2024.0.0) (3.3.2) + Requirement already satisfied: idna<4,>=2.5 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-708/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests>=2.25.1->openvino-dev>=2024.0.0) (3.7) + Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-708/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests>=2.25.1->openvino-dev>=2024.0.0) (2.2.2) + Requirement already satisfied: certifi>=2017.4.17 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-708/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from requests>=2.25.1->openvino-dev>=2024.0.0) (2024.6.2) + Requirement already satisfied: MarkupSafe>=2.0 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-708/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from jinja2->torch) (2.1.5) + Collecting mpmath<1.4.0,>=1.1.0 (from sympy->torch) Using cached https://download.pytorch.org/whl/mpmath-1.3.0-py3-none-any.whl (536 kB) - Requirement already satisfied: backcall in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.2.0) - Requirement already satisfied: decorator in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (5.1.1) - Requirement already satisfied: jedi>=0.16 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.19.1) - Requirement already satisfied: matplotlib-inline in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.1.7) - Requirement already satisfied: pickleshare in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.7.5) - Requirement already satisfied: prompt-toolkit!=3.0.37,<3.1.0,>=3.0.30 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (3.0.43) - Requirement already satisfied: pygments>=2.4.0 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (2.18.0) - Requirement already satisfied: stack-data in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.6.3) - Requirement already satisfied: pexpect>4.3 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (4.9.0) - Requirement already satisfied: parso<0.9.0,>=0.8.3 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from jedi>=0.16->ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.8.4) - Requirement already satisfied: ptyprocess>=0.5 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from pexpect>4.3->ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.7.0) - Requirement already satisfied: wcwidth in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from prompt-toolkit!=3.0.37,<3.1.0,>=3.0.30->ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.2.13) - Requirement already satisfied: executing>=1.2.0 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from stack-data->ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (2.0.1) - Requirement already satisfied: asttokens>=2.1.0 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from stack-data->ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (2.4.1) - Requirement already satisfied: pure-eval in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from stack-data->ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.2.2) - Requirement already satisfied: six>=1.12.0 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from asttokens>=2.1.0->stack-data->ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (1.16.0) + Requirement already satisfied: backcall in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-708/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.2.0) + Requirement already satisfied: decorator in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-708/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (5.1.1) + Requirement already satisfied: jedi>=0.16 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-708/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.19.1) + Requirement already satisfied: matplotlib-inline in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-708/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.1.7) + Requirement already satisfied: pickleshare in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-708/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.7.5) + Requirement already satisfied: prompt-toolkit!=3.0.37,<3.1.0,>=3.0.30 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-708/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (3.0.47) + Requirement already satisfied: pygments>=2.4.0 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-708/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (2.18.0) + Requirement already satisfied: stack-data in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-708/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.6.3) + Requirement already satisfied: pexpect>4.3 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-708/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (4.9.0) + Requirement already satisfied: parso<0.9.0,>=0.8.3 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-708/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from jedi>=0.16->ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.8.4) + Requirement already satisfied: ptyprocess>=0.5 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-708/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from pexpect>4.3->ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.7.0) + Requirement already satisfied: wcwidth in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-708/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from prompt-toolkit!=3.0.37,<3.1.0,>=3.0.30->ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.2.13) + Requirement already satisfied: executing>=1.2.0 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-708/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from stack-data->ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (2.0.1) + Requirement already satisfied: asttokens>=2.1.0 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-708/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from stack-data->ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (2.4.1) + Requirement already satisfied: pure-eval in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-708/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from stack-data->ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (0.2.2) + Requirement already satisfied: six>=1.12.0 in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-708/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (from asttokens>=2.1.0->stack-data->ipython>=6.1.0->ipywidgets>=7.2.1->pythreejs) (1.16.0) Using cached pythreejs-2.4.2-py3-none-any.whl (3.4 MB) - Using cached openvino_dev-2024.1.0-15008-py3-none-any.whl (4.7 MB) - Using cached openvino-2024.1.0-15008-cp38-cp38-manylinux2014_x86_64.whl (38.7 MB) - Using cached opencv_python-4.9.0.80-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (62.2 MB) - Using cached onnx-1.16.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (15.9 MB) + Using cached openvino_dev-2024.2.0-15519-py3-none-any.whl (4.7 MB) + Using cached openvino-2024.2.0-15519-cp38-cp38-manylinux2014_x86_64.whl (38.7 MB) + Using cached opencv_python-4.10.0.84-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (62.5 MB) + Using cached onnx-1.16.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (15.9 MB) Using cached ipydatawidgets-4.3.5-py2.py3-none-any.whl (271 kB) Using cached networkx-3.1-py3-none-any.whl (2.1 MB) Using cached numpy-1.24.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (17.3 MB) Using cached openvino_telemetry-2024.1.0-py3-none-any.whl (23 kB) - Using cached protobuf-5.26.1-cp37-abi3-manylinux2014_x86_64.whl (302 kB) - Using cached filelock-3.14.0-py3-none-any.whl (12 kB) - Downloading fsspec-2024.5.0-py3-none-any.whl (316 kB) -  ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 316.1/316.1 kB 2.6 MB/s eta 0:00:00 + Using cached protobuf-5.27.1-cp38-abi3-manylinux2014_x86_64.whl (309 kB) + Downloading filelock-3.15.3-py3-none-any.whl (16 kB) + Using cached fsspec-2024.6.0-py3-none-any.whl (176 kB) + Using cached sympy-1.12.1-py3-none-any.whl (5.7 MB) Using cached traittypes-0.2.1-py2.py3-none-any.whl (8.6 kB) Installing collected packages: openvino-telemetry, mpmath, traittypes, sympy, protobuf, numpy, networkx, fsspec, filelock, torch, openvino, opencv-python, onnx, openvino-dev, ipydatawidgets, pythreejs - Successfully installed filelock-3.14.0 fsspec-2024.5.0 ipydatawidgets-4.3.5 mpmath-1.3.0 networkx-3.1 numpy-1.24.4 onnx-1.16.0 opencv-python-4.9.0.80 openvino-2024.1.0 openvino-dev-2024.1.0 openvino-telemetry-2024.1.0 protobuf-5.26.1 pythreejs-2.4.2 sympy-1.12 torch-2.3.0+cpu traittypes-0.2.1 + Successfully installed filelock-3.15.3 fsspec-2024.6.0 ipydatawidgets-4.3.5 mpmath-1.3.0 networkx-3.1 numpy-1.24.4 onnx-1.16.1 opencv-python-4.10.0.84 openvino-2024.2.0 openvino-dev-2024.2.0 openvino-telemetry-2024.1.0 protobuf-5.27.1 pythreejs-2.4.2 sympy-1.12.1 torch-2.3.1+cpu traittypes-0.2.1 Note: you may need to restart the kernel to use updated packages. @@ -158,28 +158,28 @@ Imports import collections import time from pathlib import Path - + import cv2 import ipywidgets as widgets import numpy as np from IPython.display import clear_output, display import openvino as ov - + # Fetch `notebook_utils` module import requests - + r = requests.get( url="https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/latest/utils/notebook_utils.py", ) with open("notebook_utils.py", "w") as f: f.write(r.text) - + r = requests.get( url="https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/latest/utils/engine3js.py", ) with open("engine3js.py", "w") as f: f.write(r.text) - + import notebook_utils as utils import engine3js as engine @@ -201,19 +201,19 @@ directory structure and downloads the selected model. # directory where model will be downloaded base_model_dir = "model" - + # model name as named in Open Model Zoo model_name = "human-pose-estimation-3d-0001" # selected precision (FP32, FP16) precision = "FP32" - + BASE_MODEL_NAME = f"{base_model_dir}/public/{model_name}/{model_name}" model_path = Path(BASE_MODEL_NAME).with_suffix(".pth") onnx_path = Path(BASE_MODEL_NAME).with_suffix(".onnx") - + ir_model_path = f"model/public/{model_name}/{precision}/{model_name}.xml" model_weights_path = f"model/public/{model_name}/{precision}/{model_name}.bin" - + if not model_path.exists(): download_command = f"omz_downloader " f"--name {model_name} " f"--output_dir {base_model_dir}" ! $download_command @@ -222,12 +222,12 @@ directory structure and downloads the selected model. .. parsed-literal:: ################|| Downloading human-pose-estimation-3d-0001 ||################ - + ========== Downloading model/public/human-pose-estimation-3d-0001/human-pose-estimation-3d-0001.tar.gz - - + + ========== Unpacking model/public/human-pose-estimation-3d-0001/human-pose-estimation-3d-0001.tar.gz - + Convert Model to OpenVINO IR format @@ -252,19 +252,20 @@ IR format. .. parsed-literal:: ========== Converting human-pose-estimation-3d-0001 to ONNX - Conversion to ONNX command: /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/.venv/bin/python -- /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/omz_tools/internal_scripts/pytorch_to_onnx.py --model-path=model/public/human-pose-estimation-3d-0001 --model-name=PoseEstimationWithMobileNet --model-param=is_convertible_by_mo=True --import-module=model --weights=model/public/human-pose-estimation-3d-0001/human-pose-estimation-3d-0001.pth --input-shape=1,3,256,448 --input-names=data --output-names=features,heatmaps,pafs --output-file=model/public/human-pose-estimation-3d-0001/human-pose-estimation-3d-0001.onnx - + Conversion to ONNX command: /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-708/.workspace/scm/ov-notebook/.venv/bin/python -- /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-708/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/omz_tools/internal_scripts/pytorch_to_onnx.py --model-path=model/public/human-pose-estimation-3d-0001 --model-name=PoseEstimationWithMobileNet --model-param=is_convertible_by_mo=True --import-module=model --weights=model/public/human-pose-estimation-3d-0001/human-pose-estimation-3d-0001.pth --input-shape=1,3,256,448 --input-names=data --output-names=features,heatmaps,pafs --output-file=model/public/human-pose-estimation-3d-0001/human-pose-estimation-3d-0001.onnx + ONNX check passed successfully. - + ========== Converting human-pose-estimation-3d-0001 to IR (FP32) - Conversion command: /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/.venv/bin/python -- /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/.venv/bin/mo --framework=onnx --output_dir=model/public/human-pose-estimation-3d-0001/FP32 --model_name=human-pose-estimation-3d-0001 --input=data '--mean_values=data[128.0,128.0,128.0]' '--scale_values=data[255.0,255.0,255.0]' --output=features,heatmaps,pafs --input_model=model/public/human-pose-estimation-3d-0001/human-pose-estimation-3d-0001.onnx '--layout=data(NCHW)' '--input_shape=[1, 3, 256, 448]' --compress_to_fp16=False - - [ INFO ] MO command line tool is considered as the legacy conversion API as of OpenVINO 2023.2 release. Please use OpenVINO Model Converter (OVC). OVC represents a lightweight alternative of MO and provides simplified model conversion API. + Conversion command: /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-708/.workspace/scm/ov-notebook/.venv/bin/python -- /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-708/.workspace/scm/ov-notebook/.venv/bin/mo --framework=onnx --output_dir=model/public/human-pose-estimation-3d-0001/FP32 --model_name=human-pose-estimation-3d-0001 --input=data '--mean_values=data[128.0,128.0,128.0]' '--scale_values=data[255.0,255.0,255.0]' --output=features,heatmaps,pafs --input_model=model/public/human-pose-estimation-3d-0001/human-pose-estimation-3d-0001.onnx '--layout=data(NCHW)' '--input_shape=[1, 3, 256, 448]' --compress_to_fp16=False + + [ INFO ] MO command line tool is considered as the legacy conversion API as of OpenVINO 2023.2 release. + In 2025.0 MO command line tool and openvino.tools.mo.convert_model() will be removed. Please use OpenVINO Model Converter (OVC) or openvino.convert_model(). OVC represents a lightweight alternative of MO and provides simplified model conversion API. Find more information about transition from MO to OVC at https://docs.openvino.ai/2023.2/openvino_docs_OV_Converter_UG_prepare_model_convert_model_MO_OVC_transition.html [ SUCCESS ] Generated IR version 11 model. - [ SUCCESS ] XML file: /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/notebooks/3D-pose-estimation-webcam/model/public/human-pose-estimation-3d-0001/FP32/human-pose-estimation-3d-0001.xml - [ SUCCESS ] BIN file: /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/notebooks/3D-pose-estimation-webcam/model/public/human-pose-estimation-3d-0001/FP32/human-pose-estimation-3d-0001.bin - + [ SUCCESS ] XML file: /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-708/.workspace/scm/ov-notebook/notebooks/3D-pose-estimation-webcam/model/public/human-pose-estimation-3d-0001/FP32/human-pose-estimation-3d-0001.xml + [ SUCCESS ] BIN file: /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-708/.workspace/scm/ov-notebook/notebooks/3D-pose-estimation-webcam/model/public/human-pose-estimation-3d-0001/FP32/human-pose-estimation-3d-0001.bin + Select inference device @@ -277,14 +278,14 @@ select device from dropdown list for running inference using OpenVINO .. code:: ipython3 core = ov.Core() - + device = widgets.Dropdown( options=core.available_devices + ["AUTO"], value="AUTO", description="Device:", disabled=False, ) - + device @@ -319,7 +320,7 @@ created to infer the compiled model. compiled_model = core.compile_model(model=model, device_name=device.value) infer_request = compiled_model.create_infer_request() input_tensor_name = model.inputs[0].get_any_name() - + # get input and output names of nodes input_layer = compiled_model.input(0) output_layers = list(compiled_model.outputs) @@ -359,25 +360,25 @@ input for the 3D model. This is how you obtain the output heat maps, PAF def model_infer(scaled_img, stride): """ Run model inference on the input image - + Parameters: scaled_img: resized image according to the input size of the model stride: int, the stride of the window """ - + # Remove excess space from the picture img = scaled_img[ 0 : scaled_img.shape[0] - (scaled_img.shape[0] % stride), 0 : scaled_img.shape[1] - (scaled_img.shape[1] % stride), ] - + img = np.transpose(img, (2, 0, 1))[None,] infer_request.infer({input_tensor_name: img}) # A set of three inference results is obtained results = {name: infer_request.get_tensor(name).data[:] for name in {"features", "heatmaps", "pafs"}} # Get the results results = (results["features"][0], results["heatmaps"][0], results["pafs"][0]) - + return results Draw 2D Pose Overlays @@ -417,8 +418,8 @@ from Open Model Zoo. [13, 14], # neck - r_hip - r_knee - r_ankle ] ) - - + + body_edges_2d = np.array( [ [0, 1], # neck - nose @@ -440,25 +441,25 @@ from Open Model Zoo. [13, 14], # neck - r_hip - r_knee - r_ankle ] ) - - + + def draw_poses(frame, poses_2d, scaled_img, use_popup): """ Draw 2D pose overlays on the image to visualize estimated poses. Joints are drawn as circles and limbs are drawn as lines. - + :param frame: the input image :param poses_2d: array of human joint pairs """ for pose in poses_2d: pose = np.array(pose[0:-1]).reshape((-1, 3)).transpose() was_found = pose[2] > 0 - + pose[0], pose[1] = ( pose[0] * frame.shape[1] / scaled_img.shape[1], pose[1] * frame.shape[0] / scaled_img.shape[0], ) - + # Draw joints. for edge in body_edges_2d: if was_found[edge[0]] and was_found[edge[1]]: @@ -481,7 +482,7 @@ from Open Model Zoo. -1, cv2.LINE_AA, ) - + return frame Main Processing Function @@ -498,18 +499,18 @@ webcam feed or a video file. """ 2D image as input, using OpenVINO as inference backend, get joints 3D coordinates, and draw 3D human skeleton in the scene - + :param source: The webcam number to feed the video stream with primary webcam set to "0", or the video path. :param flip: To be used by VideoPlayer function for flipping capture image. :param use_popup: False for showing encoded frames over this notebook, True for creating a popup window. :param skip_frames: Number of frames to skip at the beginning of the video. """ - + focal_length = -1 # default stride = 8 player = None skeleton_set = None - + try: # create video player to play with target fps video_path # get the frame from camera @@ -517,16 +518,16 @@ webcam feed or a video file. player = utils.VideoPlayer(source, flip=flip, fps=30, skip_first_frames=skip_frames) # start capturing player.start() - + input_image = player.next() # set the window size resize_scale = 450 / input_image.shape[1] windows_width = int(input_image.shape[1] * resize_scale) windows_height = int(input_image.shape[0] * resize_scale) - + # use visualization library engine3D = engine.Engine3js(grid=True, axis=True, view_width=windows_width, view_height=windows_height) - + if use_popup: # display the 3D human pose in this notebook, and origin frame in popup window display(engine3D.renderer) @@ -536,43 +537,43 @@ webcam feed or a video file. # set the 2D image box, show both human pose and image in the notebook imgbox = widgets.Image(format="jpg", height=windows_height, width=windows_width) display(widgets.HBox([engine3D.renderer, imgbox])) - + skeleton = engine.Skeleton(body_edges=body_edges) - + processing_times = collections.deque() - + while True: # grab the frame frame = player.next() if frame is None: print("Source ended") break - + # resize image and change dims to fit neural network input # (see https://github.com/openvinotoolkit/open_model_zoo/tree/master/models/public/human-pose-estimation-3d-0001) scaled_img = cv2.resize(frame, dsize=(model.inputs[0].shape[3], model.inputs[0].shape[2])) - + if focal_length < 0: # Focal length is unknown focal_length = np.float32(0.8 * scaled_img.shape[1]) - + # inference start start_time = time.time() # get results inference_result = model_infer(scaled_img, stride) - + # inference stop stop_time = time.time() processing_times.append(stop_time - start_time) # Process the point to point coordinates of the data poses_3d, poses_2d = engine.parse_poses(inference_result, 1, stride, focal_length, True) - + # use processing times from last 200 frames if len(processing_times) > 200: processing_times.popleft() - + processing_time = np.mean(processing_times) * 1000 fps = 1000 / processing_time - + if len(poses_3d) > 0: # From here, you can rotate the 3D point positions using the function "draw_poses", # or you can directly make the correct mapping below to properly display the object image on the screen @@ -585,28 +586,28 @@ webcam feed or a video file. -y + np.ones(poses_3d[:, 2::4].shape) * 100, -x, ) - + poses_3d = poses_3d.reshape(poses_3d.shape[0], 19, -1)[:, :, 0:3] people = skeleton(poses_3d=poses_3d) - + try: engine3D.scene_remove(skeleton_set) except Exception: pass - + engine3D.scene_add(people) skeleton_set = people - + # draw 2D frame = draw_poses(frame, poses_2d, scaled_img, use_popup) - + else: try: engine3D.scene_remove(skeleton_set) skeleton_set = None except Exception: pass - + cv2.putText( frame, f"Inference time: {processing_time:.1f}ms ({fps:.1f} FPS)", @@ -617,7 +618,7 @@ webcam feed or a video file. 1, cv2.LINE_AA, ) - + if use_popup: cv2.imshow(title, frame) key = cv2.waitKey(1) @@ -631,9 +632,9 @@ webcam feed or a video file. frame, params=[cv2.IMWRITE_JPEG_QUALITY, 90], )[1].tobytes() - + engine3D.renderer.render(engine3D.scene, engine3D.cam) - + except KeyboardInterrupt: print("Interrupted") except RuntimeError as e: @@ -680,10 +681,10 @@ picture on the left to interact. .. code:: ipython3 USE_WEBCAM = False - + cam_id = 0 video_path = "https://github.com/intel-iot-devkit/sample-videos/raw/master/face-demographics-walking.mp4" - + source = cam_id if USE_WEBCAM else video_path - + run_pose_estimation(source=source, flip=isinstance(source, int), use_popup=False) diff --git a/docs/notebooks/3D-segmentation-point-clouds-with-output.rst b/docs/notebooks/3D-segmentation-point-clouds-with-output.rst index 6447ad00919e1d..19c23d62bf48f2 100644 --- a/docs/notebooks/3D-segmentation-point-clouds-with-output.rst +++ b/docs/notebooks/3D-segmentation-point-clouds-with-output.rst @@ -216,7 +216,7 @@ chair for example. .. parsed-literal:: - /tmp/ipykernel_16799/2434168836.py:12: UserWarning: No data for colormapping provided via 'c'. Parameters 'cmap' will be ignored + /tmp/ipykernel_1006815/2434168836.py:12: UserWarning: No data for colormapping provided via 'c'. Parameters 'cmap' will be ignored ax.scatter3D(X, Y, Z, s=5, cmap="jet", marker="o", label="chair") @@ -317,7 +317,7 @@ select device from dropdown list for running inference using OpenVINO .. parsed-literal:: - /tmp/ipykernel_16799/2804603389.py:23: UserWarning: No data for colormapping provided via 'c'. Parameters 'cmap' will be ignored + /tmp/ipykernel_1006815/2804603389.py:23: UserWarning: No data for colormapping provided via 'c'. Parameters 'cmap' will be ignored ax.scatter(XCur, YCur, ZCur, s=5, cmap="jet", marker="o", label=classes[i]) diff --git a/docs/notebooks/action-recognition-webcam-with-output_files/action-recognition-webcam-with-output_22_0.png b/docs/notebooks/action-recognition-webcam-with-output_files/action-recognition-webcam-with-output_22_0.png index 29fb7787324c8b..c504ef5bf83bf4 100644 --- a/docs/notebooks/action-recognition-webcam-with-output_files/action-recognition-webcam-with-output_22_0.png +++ b/docs/notebooks/action-recognition-webcam-with-output_files/action-recognition-webcam-with-output_22_0.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:1f9f7a4ff050de5ac1c035ff13546573f526abbc3d4b1e157edb3a278caba746 -size 69060 +oid sha256:d012f16703650c5879a6b9393860504b16b8515f1ed19faff89c314e5e129ab7 +size 69133 diff --git a/docs/notebooks/all_notebooks_paths.txt b/docs/notebooks/all_notebooks_paths.txt index ebe064c809a46b..8a7ff229db33f3 100644 --- a/docs/notebooks/all_notebooks_paths.txt +++ b/docs/notebooks/all_notebooks_paths.txt @@ -15,6 +15,7 @@ notebooks/convert-to-openvino/convert-to-openvino.ipynb notebooks/convert-to-openvino/legacy-mo-convert-to-openvino.ipynb notebooks/cross-lingual-books-alignment/cross-lingual-books-alignment.ipynb notebooks/ct-segmentation-quantize/ct-segmentation-quantize-nncf.ipynb +notebooks/ddcolor-image-colorization/ddcolor-image-colorization.ipynb notebooks/decidiffusion-image-generation/decidiffusion-image-generation.ipynb notebooks/depth-anything/depth-anything.ipynb notebooks/detectron2-to-openvino/detectron2-to-openvino.ipynb @@ -36,9 +37,9 @@ notebooks/hello-npu/hello-npu.ipynb notebooks/hello-segmentation/hello-segmentation.ipynb notebooks/hello-world/hello-world.ipynb notebooks/hugging-face-hub/hugging-face-hub.ipynb +notebooks/hunyuan-dit-image-generation/hunyuan-dit-image-generation.ipynb notebooks/image-bind/image-bind.ipynb notebooks/image-classification-quantization/image-classification-quantization.ipynb -notebooks/image-inpainting/image-inpainting.ipynb notebooks/instant-id/instant-id.ipynb notebooks/instruct-pix2pix-image-editing/instruct-pix2pix-image-editing.ipynb notebooks/knowledge-graphs-conve/knowledge-graphs-conve.ipynb @@ -51,6 +52,7 @@ notebooks/llava-multimodal-chatbot/llava-multimodal-chatbot.ipynb notebooks/llava-multimodal-chatbot/videollava-multimodal-chatbot.ipynb notebooks/llava-next-multimodal-chatbot/llava-next-multimodal-chatbot.ipynb notebooks/llm-agent-langchain/llm-agent-langchain.ipynb +notebooks/llm-chatbot/llm-chatbot-generate-api.ipynb notebooks/llm-chatbot/llm-chatbot.ipynb notebooks/llm-question-answering/llm-question-answering.ipynb notebooks/llm-rag-langchain/llm-rag-langchain.ipynb @@ -60,9 +62,9 @@ notebooks/mms-massively-multilingual-speech/mms-massively-multilingual-speech.ip notebooks/mobileclip-video-search/mobileclip-video-search.ipynb notebooks/mobilevlm-language-assistant/mobilevlm-language-assistant.ipynb notebooks/model-server/model-server.ipynb -notebooks/model-tools/model-tools.ipynb notebooks/music-generation/music-generation.ipynb notebooks/named-entity-recognition/named-entity-recognition.ipynb +notebooks/nano-llava-multimodal-chatbot/nano-llava-multimodal-chatbot.ipynb notebooks/object-detection-webcam/object-detection.ipynb notebooks/oneformer-segmentation/oneformer-segmentation.ipynb notebooks/openvino-api/openvino-api.ipynb @@ -73,6 +75,8 @@ notebooks/optimize-preprocessing/optimize-preprocessing.ipynb notebooks/paddle-ocr-webcam/paddle-ocr-webcam.ipynb notebooks/paddle-to-openvino/paddle-to-openvino-classification.ipynb notebooks/paint-by-example/paint-by-example.ipynb +notebooks/parler-tts-text-to-speech/parler-tts-text-to-speech.ipynb +notebooks/person-counting-webcam/person-counting.ipynb notebooks/person-tracking-webcam/person-tracking.ipynb notebooks/photo-maker/photo-maker.ipynb notebooks/pix2struct-docvqa/pix2struct-docvqa.ipynb @@ -80,6 +84,7 @@ notebooks/pose-estimation-webcam/pose-estimation.ipynb notebooks/pyannote-speaker-diarization/pyannote-speaker-diarization.ipynb notebooks/pytorch-post-training-quantization-nncf/pytorch-post-training-quantization-nncf.ipynb notebooks/pytorch-quantization-aware-training/pytorch-quantization-aware-training.ipynb +notebooks/pytorch-quantization-sparsity-aware-training/pytorch-quantization-sparsity-aware-training.ipynb notebooks/pytorch-to-openvino/pytorch-onnx-to-openvino.ipynb notebooks/pytorch-to-openvino/pytorch-to-openvino.ipynb notebooks/qrcode-monster/qrcode-monster.ipynb @@ -87,6 +92,7 @@ notebooks/quantizing-model-with-accuracy-control/speech-recognition-quantization notebooks/quantizing-model-with-accuracy-control/yolov8-quantization-with-accuracy-control.ipynb notebooks/riffusion-text-to-music/riffusion-text-to-music.ipynb notebooks/rmbg-background-removal/rmbg-background-removal.ipynb +notebooks/s3d-mil-nce-text-to-video-retrieval/s3d-mil-nce-text-to-video-retrieval.ipynb notebooks/sdxl-turbo/sdxl-turbo.ipynb notebooks/segment-anything/segment-anything.ipynb notebooks/siglip-zero-shot-image-classification/siglip-zero-shot-image-classification.ipynb @@ -95,6 +101,7 @@ notebooks/softvc-voice-conversion/softvc-voice-conversion.ipynb notebooks/sound-generation-audioldm2/sound-generation-audioldm2.ipynb notebooks/sparsity-optimization/sparsity-optimization.ipynb notebooks/speculative-sampling/speculative-sampling.ipynb +notebooks/speechbrain-emotion-recognition/speechbrain-emotion-recognition.ipynb notebooks/speech-recognition-quantization/speech-recognition-quantization-data2vec.ipynb notebooks/speech-recognition-quantization/speech-recognition-quantization-wav2vec2.ipynb notebooks/speech-to-text/speech-to-text.ipynb @@ -119,7 +126,6 @@ notebooks/tensorflow-hub/tensorflow-hub.ipynb notebooks/tensorflow-object-detection-to-openvino/tensorflow-instance-segmentation-to-openvino.ipynb notebooks/tensorflow-object-detection-to-openvino/tensorflow-object-detection-to-openvino.ipynb notebooks/tensorflow-quantization-aware-training/tensorflow-quantization-aware-training.ipynb -notebooks/text-prediction/text-prediction.ipynb notebooks/tflite-selfie-segmentation/tflite-selfie-segmentation.ipynb notebooks/tflite-to-openvino/tflite-to-openvino.ipynb notebooks/tiny-sd-image-generation/tiny-sd-image-generation.ipynb @@ -132,9 +138,9 @@ notebooks/vision-image-colorization/vision-image-colorization.ipynb notebooks/vision-monodepth/vision-monodepth.ipynb notebooks/vision-paddlegan-anime/vision-paddlegan-anime.ipynb notebooks/vision-paddlegan-superresolution/vision-paddlegan-superresolution.ipynb -notebooks/whisper-subtitles-generation/whisper-convert.ipynb -notebooks/whisper-subtitles-generation/whisper-nncf-quantize.ipynb +notebooks/whisper-subtitles-generation/whisper-subtitles-generation.ipynb notebooks/wuerstchen-image-generation/wuerstchen-image-generation.ipynb +notebooks/yolov10-optimization/yolov10-optimization.ipynb notebooks/yolov7-optimization/yolov7-optimization.ipynb notebooks/yolov8-optimization/yolov8-instance-segmentation.ipynb notebooks/yolov8-optimization/yolov8-keypoint-detection.ipynb diff --git a/docs/notebooks/amused-lightweight-text-to-image-with-output.rst b/docs/notebooks/amused-lightweight-text-to-image-with-output.rst index d44c1c231284d5..f4a4d2a52d7d7f 100644 --- a/docs/notebooks/amused-lightweight-text-to-image-with-output.rst +++ b/docs/notebooks/amused-lightweight-text-to-image-with-output.rst @@ -65,12 +65,12 @@ Load and run the original pipeline import torch from diffusers import AmusedPipeline - - + + pipe = AmusedPipeline.from_pretrained( "amused/amused-256", ) - + prompt = "kind smiling ghost" image = pipe(prompt, generator=torch.Generator("cpu").manual_seed(8)).images[0] image.save("text2image_256.png") @@ -78,8 +78,8 @@ Load and run the original pipeline .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`. - warnings.warn( + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-708/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/transformers/transformer_2d.py:34: FutureWarning: `Transformer2DModelOutput` is deprecated and will be removed in version 1.0.0. Importing `Transformer2DModelOutput` from `diffusers.models.transformer_2d` is deprecated and this will be removed in a future version. Please use `from diffusers.models.modeling_outputs import Transformer2DModelOutput`, instead. + deprecate("Transformer2DModelOutput", "1.0.0", deprecation_message) @@ -130,8 +130,8 @@ Define paths for converted models: .. code:: ipython3 from pathlib import Path - - + + TRANSFORMER_OV_PATH = Path("models/transformer_ir.xml") TEXT_ENCODER_OV_PATH = Path("models/text_encoder_ir.xml") VQVAE_OV_PATH = Path("models/vqvae_ir.xml") @@ -144,10 +144,10 @@ file. .. code:: ipython3 import torch - + import openvino as ov - - + + def convert(model: torch.nn.Module, xml_path: str, example_input): xml_path = Path(xml_path) if not xml_path.exists(): @@ -155,7 +155,7 @@ file. with torch.no_grad(): converted_model = ov.convert_model(model, example_input=example_input) ov.save_model(converted_model, xml_path, compress_to_fp16=False) - + # cleanup memory torch._C._jit_clear_class_registry() torch.jit._recursive.concrete_type_store = torch.jit._recursive.ConcreteTypeStore() @@ -172,17 +172,17 @@ Convert the Text Encoder def __init__(self, text_encoder): super().__init__() self.text_encoder = text_encoder - + def forward(self, input_ids=None, return_dict=None, output_hidden_states=None): outputs = self.text_encoder( input_ids=input_ids, return_dict=return_dict, output_hidden_states=output_hidden_states, ) - + return outputs.text_embeds, outputs.last_hidden_state, outputs.hidden_states - - + + input_ids = pipe.tokenizer( prompt, return_tensors="pt", @@ -190,41 +190,41 @@ Convert the Text Encoder truncation=True, max_length=pipe.tokenizer.model_max_length, ) - + input_example = { "input_ids": input_ids.input_ids, "return_dict": torch.tensor(True), "output_hidden_states": torch.tensor(True), } - + convert(TextEncoderWrapper(pipe.text_encoder), TEXT_ENCODER_OV_PATH, input_example) .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_utils.py:4371: FutureWarning: `_is_quantized_training_enabled` is going to be deprecated in transformers 4.39.0. Please use `model.hf_quantizer.is_trainable` instead + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-708/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_utils.py:4481: FutureWarning: `_is_quantized_training_enabled` is going to be deprecated in transformers 4.39.0. Please use `model.hf_quantizer.is_trainable` instead warnings.warn( - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_attn_mask_utils.py:86: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-708/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_attn_mask_utils.py:86: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if input_shape[-1] > 1 or self.sliding_window is not None: - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_attn_mask_utils.py:162: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-708/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_attn_mask_utils.py:162: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if past_key_values_length > 0: - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/clip/modeling_clip.py:620: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-708/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/clip/modeling_clip.py:622: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! encoder_states = () if output_hidden_states else None - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/clip/modeling_clip.py:625: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-708/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/clip/modeling_clip.py:627: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if output_hidden_states: - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/clip/modeling_clip.py:279: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-708/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/clip/modeling_clip.py:276: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if attn_weights.size() != (bsz * self.num_heads, tgt_len, src_len): - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/clip/modeling_clip.py:287: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-708/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/clip/modeling_clip.py:284: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if causal_attention_mask.size() != (bsz, 1, tgt_len, src_len): - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/clip/modeling_clip.py:319: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-708/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/clip/modeling_clip.py:316: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if attn_output.size() != (bsz * self.num_heads, tgt_len, self.head_dim): - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/clip/modeling_clip.py:648: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-708/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/clip/modeling_clip.py:650: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if output_hidden_states: - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/clip/modeling_clip.py:651: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-708/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/clip/modeling_clip.py:653: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if not return_dict: - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/clip/modeling_clip.py:742: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-708/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/clip/modeling_clip.py:745: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if not return_dict: - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/clip/modeling_clip.py:1227: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-708/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/clip/modeling_clip.py:1230: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if not return_dict: @@ -239,7 +239,7 @@ Convert the U-ViT transformer def __init__(self, transformer): super().__init__() self.transformer = transformer - + def forward( self, latents=None, @@ -253,21 +253,21 @@ Convert the U-ViT transformer pooled_text_emb=pooled_text_emb, encoder_hidden_states=encoder_hidden_states, ) - - + + shape = (1, 16, 16) latents = torch.full(shape, pipe.scheduler.config.mask_token_id, dtype=torch.long) latents = torch.cat([latents] * 2) - - + + example_input = { "latents": latents, "micro_conds": torch.rand([2, 5], dtype=torch.float32), "pooled_text_emb": torch.rand([2, 768], dtype=torch.float32), "encoder_hidden_states": torch.rand([2, 77, 768], dtype=torch.float32), } - - + + pipe.transformer.eval() w_transformer = TransformerWrapper(pipe.transformer) convert(w_transformer, TRANSFORMER_OV_PATH, example_input) @@ -286,7 +286,7 @@ suitable. This function repeats part of ``AmusedPipeline``. shape = (1, 16, 16) latents = torch.full(shape, pipe.scheduler.config.mask_token_id, dtype=torch.long) model_input = torch.cat([latents] * 2) - + model_output = pipe.transformer( model_input, micro_conds=torch.rand([2, 5], dtype=torch.float32), @@ -296,50 +296,50 @@ suitable. This function repeats part of ``AmusedPipeline``. guidance_scale = 10.0 uncond_logits, cond_logits = model_output.chunk(2) model_output = uncond_logits + guidance_scale * (cond_logits - uncond_logits) - + latents = pipe.scheduler.step( model_output=model_output, timestep=torch.tensor(0), sample=latents, ).prev_sample - + return latents - - + + class VQVAEWrapper(torch.nn.Module): def __init__(self, vqvae): super().__init__() self.vqvae = vqvae - + def forward(self, latents=None, force_not_quantize=True, shape=None): outputs = self.vqvae.decode( latents, force_not_quantize=force_not_quantize, shape=shape.tolist(), ) - + return outputs - - + + latents = get_latents() example_vqvae_input = { "latents": latents, "force_not_quantize": torch.tensor(True), "shape": torch.tensor((1, 16, 16, 64)), } - + convert(VQVAEWrapper(pipe.vqvae), VQVAE_OV_PATH, example_vqvae_input) .. parsed-literal:: - /tmp/ipykernel_17572/3779428577.py:34: TracerWarning: Converting a tensor to a Python list might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /tmp/ipykernel_1007584/3779428577.py:34: TracerWarning: Converting a tensor to a Python list might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! shape=shape.tolist(), - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/vq_model.py:144: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-708/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/autoencoders/vq_model.py:144: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if not force_not_quantize: - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/upsampling.py:149: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-708/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/upsampling.py:146: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! assert hidden_states.shape[1] == self.channels - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/upsampling.py:165: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-708/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/models/upsampling.py:162: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if hidden_states.shape[0] >= 64: @@ -353,8 +353,8 @@ Select device from dropdown list for running inference using OpenVINO. .. code:: ipython3 import ipywidgets as widgets - - + + core = ov.Core() device = widgets.Dropdown( options=core.available_devices + ["AUTO"], @@ -362,7 +362,7 @@ Select device from dropdown list for running inference using OpenVINO. description="Device:", disabled=False, ) - + device @@ -387,29 +387,29 @@ wrapper classes return ``torch.Tensor``\ s instead of ``np.array``\ s. .. code:: ipython3 from collections import namedtuple - - + + class ConvTextEncoderWrapper(torch.nn.Module): def __init__(self, text_encoder, config): super().__init__() self.config = config self.text_encoder = text_encoder - + def forward(self, input_ids=None, return_dict=None, output_hidden_states=None): inputs = { "input_ids": input_ids, "return_dict": return_dict, "output_hidden_states": output_hidden_states, } - + outs = self.text_encoder(inputs) - + outputs = namedtuple("CLIPTextModelOutput", ("text_embeds", "last_hidden_state", "hidden_states")) - + text_embeds = torch.from_numpy(outs[0]) last_hidden_state = torch.from_numpy(outs[1]) hidden_states = list(torch.from_numpy(out) for out in outs.values())[2:] - + return outputs(text_embeds, last_hidden_state, hidden_states) .. code:: ipython3 @@ -419,7 +419,7 @@ wrapper classes return ``torch.Tensor``\ s instead of ``np.array``\ s. super().__init__() self.config = config self.transformer = transformer - + def forward(self, latents=None, micro_conds=None, pooled_text_emb=None, encoder_hidden_states=None, **kwargs): outputs = self.transformer( { @@ -430,7 +430,7 @@ wrapper classes return ``torch.Tensor``\ s instead of ``np.array``\ s. }, share_inputs=False, ) - + return torch.from_numpy(outputs[0]) .. code:: ipython3 @@ -441,17 +441,17 @@ wrapper classes return ``torch.Tensor``\ s instead of ``np.array``\ s. self.vqvae = vqvae self.dtype = dtype self.config = config - + def decode(self, latents=None, force_not_quantize=True, shape=None): inputs = { "latents": latents, "force_not_quantize": force_not_quantize, "shape": torch.tensor(shape), } - + outs = self.vqvae(inputs) outs = namedtuple("VQVAE", "sample")(torch.from_numpy(outs[0])) - + return outs And insert wrappers instances in the pipeline: @@ -459,25 +459,25 @@ And insert wrappers instances in the pipeline: .. code:: ipython3 prompt = "kind smiling ghost" - + transformer = pipe.transformer vqvae = pipe.vqvae text_encoder = pipe.text_encoder - + pipe.__dict__["_internal_dict"]["_execution_device"] = pipe._execution_device # this is to avoid some problem that can occur in the pipeline pipe.register_modules( text_encoder=ConvTextEncoderWrapper(ov_text_encoder, text_encoder.config), transformer=ConvTransformerWrapper(ov_transformer, transformer.config), vqvae=ConvVQVAEWrapper(ov_vqvae, vqvae.dtype, vqvae.config), ) - + image = pipe(prompt, generator=torch.Generator("cpu").manual_seed(8)).images[0] image.save("text2image_256.png") .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/configuration_utils.py:139: FutureWarning: Accessing config attribute `_execution_device` directly via 'AmusedPipeline' object attribute is deprecated. Please access '_execution_device' over 'AmusedPipeline's config object instead, e.g. 'scheduler.config._execution_device'. + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-708/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/configuration_utils.py:140: FutureWarning: Accessing config attribute `_execution_device` directly via 'AmusedPipeline' object attribute is deprecated. Please access '_execution_device' over 'AmusedPipeline's config object instead, e.g. 'scheduler.config._execution_device'. deprecate("direct config name access", "1.0.0", deprecation_message, standard_warn=False) @@ -540,13 +540,9 @@ improve model inference speed. .. code:: ipython3 QUANTIZED_TRANSFORMER_OV_PATH = Path(str(TRANSFORMER_OV_PATH).replace(".xml", "_quantized.xml")) - - to_quantize = widgets.Checkbox( - value=True, - description="Quantization", - disabled=False, - ) - + + skip_for_device = "GPU" in device.value + to_quantize = widgets.Checkbox(value=not skip_for_device, description="Quantization", disabled=skip_for_device) to_quantize @@ -561,12 +557,12 @@ improve model inference speed. .. code:: ipython3 import requests - + r = requests.get( url="https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/latest/utils/skip_kernel_extension.py", ) open("skip_kernel_extension.py", "w").write(r.text) - + %load_ext skip_kernel_extension Prepare calibration dataset @@ -575,49 +571,49 @@ Prepare calibration dataset We use a portion of -`conceptual_captions `__ +`conceptual_captions `__ dataset from Hugging Face as calibration data. To collect intermediate model inputs for calibration we customize ``CompiledModel``. .. code:: ipython3 %%skip not $to_quantize.value - + import datasets from tqdm.auto import tqdm from typing import Any, Dict, List import pickle import numpy as np - - + + def disable_progress_bar(pipeline, disable=True): if not hasattr(pipeline, "_progress_bar_config"): pipeline._progress_bar_config = {'disable': disable} else: pipeline._progress_bar_config['disable'] = disable - - + + class CompiledModelDecorator(ov.CompiledModel): def __init__(self, compiled_model: ov.CompiledModel, data_cache: List[Any] = None, keep_prob: float = 0.5): super().__init__(compiled_model) self.data_cache = data_cache if data_cache is not None else [] self.keep_prob = keep_prob - + def __call__(self, *args, **kwargs): if np.random.rand() <= self.keep_prob: self.data_cache.append(*args) return super().__call__(*args, **kwargs) - - + + def collect_calibration_data(ov_transformer_model, calibration_dataset_size: int) -> List[Dict]: calibration_dataset_filepath = Path(f"calibration_data/{calibration_dataset_size}.pkl") if not calibration_dataset_filepath.exists(): calibration_data = [] pipe.transformer.transformer = CompiledModelDecorator(ov_transformer_model, calibration_data, keep_prob=1.0) disable_progress_bar(pipe) - - dataset = datasets.load_dataset("conceptual_captions", split="train").shuffle(seed=42) - + + dataset = datasets.load_dataset("google-research-datasets/conceptual_captions", split="train", trust_remote_code=True).shuffle(seed=42) + # Run inference for data collection pbar = tqdm(total=calibration_dataset_size) for batch in dataset: @@ -628,14 +624,14 @@ model inputs for calibration we customize ``CompiledModel``. pbar.update(len(calibration_data) - pbar.n) if pbar.n >= calibration_dataset_size: break - + pipe.transformer.transformer = ov_transformer_model disable_progress_bar(pipe, disable=False) - + calibration_dataset_filepath.parent.mkdir(exist_ok=True, parents=True) with open(calibration_dataset_filepath, 'wb') as f: pickle.dump(calibration_data, f) - + with open(calibration_dataset_filepath, 'rb') as f: calibration_data = pickle.load(f) return calibration_data @@ -651,14 +647,14 @@ model. .. code:: ipython3 %%skip not $to_quantize.value - + from nncf.quantization.advanced_parameters import AdvancedSmoothQuantParameters from nncf.quantization.range_estimator import RangeEstimatorParameters, StatisticsCollectorParameters, StatisticsType, \ AggregatorType import nncf - + CALIBRATION_DATASET_SIZE = 12 * 25 - + if not QUANTIZED_TRANSFORMER_OV_PATH.exists(): calibration_data = collect_calibration_data(ov_transformer, CALIBRATION_DATASET_SIZE) quantized_model = nncf.quantize( @@ -690,14 +686,6 @@ model. INFO:nncf:NNCF initialized successfully. Supported frameworks detected: torch, onnx, openvino -.. parsed-literal:: - - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/datasets/load.py:1486: FutureWarning: The repository for conceptual_captions contains custom code which must be executed to correctly load the dataset. You can inspect the repository content at https://hf.co/datasets/conceptual_captions - You can avoid this message in future by passing the argument `trust_remote_code=True`. - Passing `trust_remote_code=True` will be mandatory to load this dataset from the next major release of `datasets`. - warnings.warn( - - .. parsed-literal:: @@ -706,7 +694,7 @@ model. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/configuration_utils.py:139: FutureWarning: Accessing config attribute `_execution_device` directly via 'AmusedPipeline' object attribute is deprecated. Please access '_execution_device' over 'AmusedPipeline's config object instead, e.g. 'scheduler.config._execution_device'. + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-708/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/configuration_utils.py:140: FutureWarning: Accessing config attribute `_execution_device` directly via 'AmusedPipeline' object attribute is deprecated. Please access '_execution_device' over 'AmusedPipeline's config object instead, e.g. 'scheduler.config._execution_device'. deprecate("direct config name access", "1.0.0", deprecation_message, standard_warn=False) @@ -756,9 +744,9 @@ model. INFO:nncf:3 ignored nodes were found by types in the NNCFGraph INFO:nncf:182 ignored nodes were found by name in the NNCFGraph - INFO:nncf:Not adding activation input quantizer for operation: 37 __module.transformer.embed.conv/aten::_convolution/Convolution - INFO:nncf:Not adding activation input quantizer for operation: 2883 __module.transformer.mlm_layer.conv1/aten::_convolution/Convolution - INFO:nncf:Not adding activation input quantizer for operation: 3243 __module.transformer.mlm_layer.conv2/aten::_convolution/Convolution + INFO:nncf:Not adding activation input quantizer for operation: 120 __module.transformer.embed.conv/aten::_convolution/Convolution + INFO:nncf:Not adding activation input quantizer for operation: 2154 __module.transformer.mlm_layer.conv1/aten::_convolution/Convolution + INFO:nncf:Not adding activation input quantizer for operation: 2993 __module.transformer.mlm_layer.conv2/aten::_convolution/Convolution @@ -784,17 +772,17 @@ model. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/experimental/tensor/tensor.py:84: RuntimeWarning: invalid value encountered in multiply + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-708/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/experimental/tensor/tensor.py:92: RuntimeWarning: invalid value encountered in multiply return Tensor(self.data * unwrap_tensor_data(other)) - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/experimental/tensor/tensor.py:84: RuntimeWarning: invalid value encountered in multiply + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-708/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/experimental/tensor/tensor.py:92: RuntimeWarning: invalid value encountered in multiply return Tensor(self.data * unwrap_tensor_data(other)) - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/experimental/tensor/tensor.py:84: RuntimeWarning: invalid value encountered in multiply + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-708/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/experimental/tensor/tensor.py:92: RuntimeWarning: invalid value encountered in multiply return Tensor(self.data * unwrap_tensor_data(other)) - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/experimental/tensor/tensor.py:84: RuntimeWarning: invalid value encountered in multiply + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-708/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/experimental/tensor/tensor.py:92: RuntimeWarning: invalid value encountered in multiply return Tensor(self.data * unwrap_tensor_data(other)) - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/experimental/tensor/tensor.py:84: RuntimeWarning: invalid value encountered in multiply + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-708/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/experimental/tensor/tensor.py:92: RuntimeWarning: invalid value encountered in multiply return Tensor(self.data * unwrap_tensor_data(other)) - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/experimental/tensor/tensor.py:84: RuntimeWarning: invalid value encountered in multiply + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-708/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/experimental/tensor/tensor.py:92: RuntimeWarning: invalid value encountered in multiply return Tensor(self.data * unwrap_tensor_data(other)) @@ -812,21 +800,21 @@ Demo generation with quantized pipeline .. code:: ipython3 %%skip not $to_quantize.value - + original_ov_transformer_model = pipe.transformer.transformer pipe.transformer.transformer = core.compile_model(QUANTIZED_TRANSFORMER_OV_PATH, device.value) - + image = pipe(prompt, generator=torch.Generator('cpu').manual_seed(8)).images[0] image.save('text2image_256_quantized.png') - + pipe.transformer.transformer = original_ov_transformer_model - + display(image) .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/configuration_utils.py:139: FutureWarning: Accessing config attribute `_execution_device` directly via 'AmusedPipeline' object attribute is deprecated. Please access '_execution_device' over 'AmusedPipeline's config object instead, e.g. 'scheduler.config._execution_device'. + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-708/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/configuration_utils.py:140: FutureWarning: Accessing config attribute `_execution_device` directly via 'AmusedPipeline' object attribute is deprecated. Please access '_execution_device' over 'AmusedPipeline's config object instead, e.g. 'scheduler.config._execution_device'. deprecate("direct config name access", "1.0.0", deprecation_message, standard_warn=False) @@ -857,24 +845,24 @@ a rough estimate of generation quality. .. code:: ipython3 %%skip not $to_quantize.value - + from torchmetrics.image.inception import InceptionScore from torchvision import transforms as transforms from itertools import islice import time - + VALIDATION_DATASET_SIZE = 100 - + def compute_inception_score(ov_transformer_model_path, validation_set_size, batch_size=100): original_ov_transformer_model = pipe.transformer.transformer pipe.transformer.transformer = core.compile_model(ov_transformer_model_path, device.value) - + disable_progress_bar(pipe) - dataset = datasets.load_dataset("conceptual_captions", "unlabeled", split="validation").shuffle(seed=42) + dataset = datasets.load_dataset("google-research-datasets/conceptual_captions", "unlabeled", split="validation", trust_remote_code=True).shuffle(seed=42) dataset = islice(dataset, validation_set_size) - + inception_score = InceptionScore(normalize=True, splits=1) - + images = [] infer_times = [] for batch in tqdm(dataset, total=validation_set_size, desc="Computing Inception Score"): @@ -886,21 +874,21 @@ a rough estimate of generation quality. infer_times.append(time.perf_counter() - start_time) image = transforms.ToTensor()(image) images.append(image) - + mean_perf_time = sum(infer_times) / len(infer_times) - + while len(images) > 0: images_batch = torch.stack(images[-batch_size:]) images = images[:-batch_size] inception_score.update(images_batch) kl_mean, kl_std = inception_score.compute() - + pipe.transformer.transformer = original_ov_transformer_model disable_progress_bar(pipe, disable=False) - + return kl_mean, mean_perf_time - - + + original_inception_score, original_time = compute_inception_score(TRANSFORMER_OV_PATH, VALIDATION_DATASET_SIZE) print(f"Original pipeline Inception Score: {original_inception_score}") quantized_inception_score, quantized_time = compute_inception_score(QUANTIZED_TRANSFORMER_OV_PATH, VALIDATION_DATASET_SIZE) @@ -910,7 +898,7 @@ a rough estimate of generation quality. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torchmetrics/utilities/prints.py:43: UserWarning: Metric `InceptionScore` will save all extracted features in buffer. For large datasets this may lead to large memory footprint. + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-708/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torchmetrics/utilities/prints.py:43: UserWarning: Metric `InceptionScore` will save all extracted features in buffer. For large datasets this may lead to large memory footprint. warnings.warn(\*args, \*\*kwargs) # noqa: B028 @@ -922,9 +910,9 @@ a rough estimate of generation quality. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/configuration_utils.py:139: FutureWarning: Accessing config attribute `_execution_device` directly via 'AmusedPipeline' object attribute is deprecated. Please access '_execution_device' over 'AmusedPipeline's config object instead, e.g. 'scheduler.config._execution_device'. + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-708/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/configuration_utils.py:140: FutureWarning: Accessing config attribute `_execution_device` directly via 'AmusedPipeline' object attribute is deprecated. Please access '_execution_device' over 'AmusedPipeline's config object instead, e.g. 'scheduler.config._execution_device'. deprecate("direct config name access", "1.0.0", deprecation_message, standard_warn=False) - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torchmetrics/image/inception.py:176: UserWarning: std(): degrees of freedom is <= 0. Correction should be strictly less than the reduction factor (input numel divided by output numel). (Triggered internally at ../aten/src/ATen/native/ReduceOps.cpp:1807.) + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-708/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torchmetrics/image/inception.py:176: UserWarning: std(): degrees of freedom is <= 0. Correction should be strictly less than the reduction factor (input numel divided by output numel). (Triggered internally at ../aten/src/ATen/native/ReduceOps.cpp:1807.) return kl.mean(), kl.std() @@ -955,13 +943,13 @@ Below you can select which pipeline to run: original or quantized. .. code:: ipython3 quantized_model_present = QUANTIZED_TRANSFORMER_OV_PATH.exists() - + use_quantized_model = widgets.Checkbox( value=True if quantized_model_present else False, description="Use quantized pipeline", disabled=not quantized_model_present, ) - + use_quantized_model @@ -977,18 +965,18 @@ Below you can select which pipeline to run: original or quantized. import gradio as gr import numpy as np - + pipe.transformer.transformer = core.compile_model( QUANTIZED_TRANSFORMER_OV_PATH if use_quantized_model.value else TRANSFORMER_OV_PATH, device.value, ) - - + + def generate(prompt, seed, _=gr.Progress(track_tqdm=True)): image = pipe(prompt, generator=torch.Generator("cpu").manual_seed(seed)).images[0] return image - - + + demo = gr.Interface( generate, [ @@ -1015,7 +1003,7 @@ Below you can select which pipeline to run: original or quantized. .. parsed-literal:: Running on local URL: http://127.0.0.1:7860 - + To create a public link, set `share=True` in `launch()`. diff --git a/docs/notebooks/animate-anyone-with-output.rst b/docs/notebooks/animate-anyone-with-output.rst index c07b1321d80c42..4240a1d02cc673 100644 --- a/docs/notebooks/animate-anyone-with-output.rst +++ b/docs/notebooks/animate-anyone-with-output.rst @@ -34,14 +34,13 @@ Learn more in `GitHub repo `__ and `paper `__. -.. container:: alert alert-warning +.. warning:: + + This tutorial requires at least **96 GB** of RAM for model conversion and **40 GB** for + inference. Changing the values of ``HEIGHT<``, ``WIDTH`` and ``VIDEO_LENGTH`` variables + will change the memory consumption but will also affect accuracy. - :: -

! WARNING !

-

- This tutorial requires at least 96 GB of RAM for model conversion and 40 GB for inference. Changing the values of HEIGHT, WIDTH and VIDEO_LENGTH variables will change the memory consumption but will also affect accuracy. -

Table of contents: ^^^^^^^^^^^^^^^^^^ @@ -64,7 +63,8 @@ Table of contents: - `Video post-processing <#video-post-processing>`__ - `Interactive inference <#interactive-inference>`__ -.. |image0| image:: https://github.com/openvinotoolkit/openvino_notebooks/raw/latest/notebooks/animate-anyone/animate-anyone.gif +.. |image0| image:: https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/latest/notebooks/animate-anyone/animate-anyone.gif + Prerequisites ------------- @@ -153,11 +153,11 @@ Note that we clone a fork of original repo with tweaked forward methods. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/utils/outputs.py:63: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead. + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-707/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/utils/outputs.py:63: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead. torch.utils._pytree._register_pytree_node( - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/utils/outputs.py:63: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead. + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-707/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/utils/outputs.py:63: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead. torch.utils._pytree._register_pytree_node( - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/utils/outputs.py:63: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead. + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-707/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/diffusers/utils/outputs.py:63: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead. torch.utils._pytree._register_pytree_node( @@ -308,13 +308,13 @@ Download weights .. parsed-literal:: - .gitattributes: 0%| | 0.00/1.52k [00:00`__. Weight @@ -443,7 +449,7 @@ compression parameters. More details about weights compression can be found in `OpenVINO documentation `__. -.. |image1| image:: https://humanaigc.github.io/animate-anyone/static/images/f2_img.png +.. |image01| image:: https://humanaigc.github.io/animate-anyone/static/images/f2_img.png .. code:: ipython3 @@ -502,7 +508,6 @@ of the pipeline, it will be better to convert them to separate models. .. parsed-literal:: - WARNING:nncf:NNCF provides best results with torch==2.2.*, while current torch version is 2.3.0+cpu. If you encounter issues, consider switching to torch==2.2.* INFO:nncf:Statistics of the bitwidth distribution: ┍━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┑ │ Num bits (N) │ % all parameters (layers) │ % ratio-defining parameters (layers) │ @@ -839,7 +844,7 @@ required for both reference and denoising UNets. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_utils.py:4371: FutureWarning: `_is_quantized_training_enabled` is going to be deprecated in transformers 4.39.0. Please use `model.hf_quantizer.is_trainable` instead + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-707/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/modeling_utils.py:4481: FutureWarning: `_is_quantized_training_enabled` is going to be deprecated in transformers 4.39.0. Please use `model.hf_quantizer.is_trainable` instead warnings.warn( @@ -1210,7 +1215,7 @@ Video post-processing .. raw:: html diff --git a/docs/notebooks/async-api-with-output.rst b/docs/notebooks/async-api-with-output.rst index b7d53ad0cd10c1..9a6bc4f791cd80 100644 --- a/docs/notebooks/async-api-with-output.rst +++ b/docs/notebooks/async-api-with-output.rst @@ -352,7 +352,7 @@ Test performance in Sync Mode .. parsed-literal:: Source ended - average throuput in sync mode: 43.35 fps + average throuput in sync mode: 59.76 fps Async Mode @@ -491,7 +491,7 @@ Test the performance in Async Mode .. parsed-literal:: Source ended - average throuput in async mode: 73.97 fps + average throuput in async mode: 104.09 fps Compare the performance @@ -634,5 +634,5 @@ Test the performance with ``AsyncInferQueue`` .. parsed-literal:: - average throughput in async mode with async infer queue: 111.33 fps + average throughput in async mode with async infer queue: 149.07 fps diff --git a/docs/notebooks/async-api-with-output_files/async-api-with-output_23_0.png b/docs/notebooks/async-api-with-output_files/async-api-with-output_23_0.png index 07fc6bb05537ec..947168a13876a1 100644 --- a/docs/notebooks/async-api-with-output_files/async-api-with-output_23_0.png +++ b/docs/notebooks/async-api-with-output_files/async-api-with-output_23_0.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c6eb6b07a2e43cfab480087829f6babef1e7050550997c85a7a6824f8c308cc3 -size 30403 +oid sha256:828b054c2b0cdfb183cd60781179cd06e288e55fe15ef96e8b8295814ac58e1d +size 29389 diff --git a/docs/notebooks/auto-device-with-output.rst b/docs/notebooks/auto-device-with-output.rst index 06c0ef2defb0bd..c139bff3032350 100644 --- a/docs/notebooks/auto-device-with-output.rst +++ b/docs/notebooks/auto-device-with-output.rst @@ -96,7 +96,7 @@ Import modules and create Core core = ov.Core() - if "GPU" not in core.available_devices: + if not any("GPU" in device for device in core.available_devices): display( Markdown( '
Warning: A GPU device is not available. This notebook requires GPU device to have meaningful results.
' @@ -186,15 +186,16 @@ By default, ``compile_model`` API will select **AUTO** as .. parsed-literal:: - [23:28:01.3129]I[plugin.cpp:418][AUTO] device:CPU, config:LOG_LEVEL=LOG_INFO - [23:28:01.3129]I[plugin.cpp:418][AUTO] device:CPU, config:PERFORMANCE_HINT=LATENCY - [23:28:01.3130]I[plugin.cpp:418][AUTO] device:CPU, config:PERFORMANCE_HINT_NUM_REQUESTS=0 - [23:28:01.3130]I[plugin.cpp:418][AUTO] device:CPU, config:PERF_COUNT=NO - [23:28:01.3130]I[plugin.cpp:423][AUTO] device:CPU, priority:0 - [23:28:01.3130]I[schedule.cpp:17][AUTO] scheduler starting - [23:28:01.3130]I[auto_schedule.cpp:131][AUTO] select device:CPU - [23:28:01.4657]I[auto_schedule.cpp:109][AUTO] device:CPU compiling model finished - [23:28:01.4659]I[plugin.cpp:451][AUTO] underlying hardware does not support hardware context + [22:42:34.9458]I[plugin.cpp:421][AUTO] device:CPU, config:LOG_LEVEL=LOG_INFO + [22:42:34.9458]I[plugin.cpp:421][AUTO] device:CPU, config:PERFORMANCE_HINT=LATENCY + [22:42:34.9458]I[plugin.cpp:421][AUTO] device:CPU, config:PERFORMANCE_HINT_NUM_REQUESTS=0 + [22:42:34.9459]I[plugin.cpp:421][AUTO] device:CPU, config:PERF_COUNT=NO + [22:42:34.9459]I[plugin.cpp:426][AUTO] device:CPU, priority:0 + [22:42:34.9459]I[schedule.cpp:17][AUTO] scheduler starting + [22:42:34.9459]I[auto_schedule.cpp:134][AUTO] select device:CPU + [22:42:35.1105]I[auto_schedule.cpp:336][AUTO] Device: [CPU]: Compile model took 164.490887 ms + [22:42:35.1108]I[auto_schedule.cpp:112][AUTO] device:CPU compiling model finished + [22:42:35.1109]I[plugin.cpp:454][AUTO] underlying hardware does not support hardware context Successfully compiled model without a device_name. @@ -208,7 +209,7 @@ By default, ``compile_model`` API will select **AUTO** as .. parsed-literal:: Deleted compiled_model - [23:28:01.4767]I[schedule.cpp:303][AUTO] scheduler ending + [22:42:35.1221]I[schedule.cpp:308][AUTO] scheduler ending Explicitly pass AUTO as device_name to Core::compile_model API @@ -318,7 +319,7 @@ Load the model to GPU device and perform inference .. code:: ipython3 - if "GPU" not in core.available_devices: + if not any("GPU" in device for device in core.available_devices): print(f"A GPU device is not available. Available devices are: {core.available_devices}") else: # Start time. @@ -366,7 +367,7 @@ executed on CPU until GPU is ready. .. parsed-literal:: - Time to load model using AUTO device and get first inference: 0.18 seconds. + Time to load model using AUTO device and get first inference: 0.17 seconds. .. code:: ipython3 @@ -538,12 +539,12 @@ Loop for inference and update the FPS/Latency every Compiling Model for AUTO device with THROUGHPUT hint Start inference, 6 groups of FPS/latency will be measured over 10s intervals - throughput: 177.51fps, latency: 32.04ms, time interval: 10.01s - throughput: 179.73fps, latency: 32.54ms, time interval: 10.01s - throughput: 178.74fps, latency: 32.73ms, time interval: 10.01s - throughput: 179.46fps, latency: 32.59ms, time interval: 10.01s - throughput: 178.98fps, latency: 32.74ms, time interval: 10.02s - throughput: 178.58fps, latency: 32.79ms, time interval: 10.01s + throughput: 180.70fps, latency: 31.51ms, time interval: 10.01s + throughput: 180.34fps, latency: 32.54ms, time interval: 10.00s + throughput: 179.29fps, latency: 32.60ms, time interval: 10.01s + throughput: 180.59fps, latency: 32.42ms, time interval: 10.01s + throughput: 179.95fps, latency: 32.56ms, time interval: 10.01s + throughput: 181.03fps, latency: 32.33ms, time interval: 10.00s Done @@ -589,12 +590,12 @@ Loop for inference and update the FPS/Latency for each Compiling Model for AUTO Device with LATENCY hint Start inference, 6 groups fps/latency will be out with 10s interval - throughput: 136.40fps, latency: 6.83ms, time interval: 10.01s - throughput: 137.96fps, latency: 6.81ms, time interval: 10.00s - throughput: 137.97fps, latency: 6.80ms, time interval: 10.00s - throughput: 137.97fps, latency: 6.80ms, time interval: 10.00s - throughput: 138.06fps, latency: 6.80ms, time interval: 10.00s - throughput: 133.29fps, latency: 7.06ms, time interval: 10.01s + throughput: 137.68fps, latency: 6.68ms, time interval: 10.00s + throughput: 140.68fps, latency: 6.66ms, time interval: 10.00s + throughput: 140.55fps, latency: 6.67ms, time interval: 10.00s + throughput: 140.49fps, latency: 6.67ms, time interval: 10.00s + throughput: 140.62fps, latency: 6.66ms, time interval: 10.01s + throughput: 140.67fps, latency: 6.66ms, time interval: 10.00s Done diff --git a/docs/notebooks/auto-device-with-output_files/auto-device-with-output_27_0.png b/docs/notebooks/auto-device-with-output_files/auto-device-with-output_27_0.png index dbe3a0edb3855e..3b474b8efbae73 100644 --- a/docs/notebooks/auto-device-with-output_files/auto-device-with-output_27_0.png +++ b/docs/notebooks/auto-device-with-output_files/auto-device-with-output_27_0.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:d644b71f335dd26763dfad14f93ba1ff32ffe30cfdbe3ac06d1c7346aaba3985 -size 27550 +oid sha256:b81dc92d1b9ea2b743b2dd768141d5dca2b822764f0482673cd98d4c60775e3e +size 26796 diff --git a/docs/notebooks/auto-device-with-output_files/auto-device-with-output_28_0.png b/docs/notebooks/auto-device-with-output_files/auto-device-with-output_28_0.png index 0e617d97958ff3..e17540e8246d82 100644 --- a/docs/notebooks/auto-device-with-output_files/auto-device-with-output_28_0.png +++ b/docs/notebooks/auto-device-with-output_files/auto-device-with-output_28_0.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:b3e6b932de9fd81a384cccede0ed571c920f68e6a9176ce2d84cee626bb03f05 -size 40115 +oid sha256:794493b6bb153b468b8d933cf8a205adb934792ffa3a67246c8de6016d9c1b8a +size 40060 diff --git a/docs/notebooks/clip-zero-shot-classification-with-output.rst b/docs/notebooks/clip-zero-shot-classification-with-output.rst index 495813ec267ead..9ff557539bb327 100644 --- a/docs/notebooks/clip-zero-shot-classification-with-output.rst +++ b/docs/notebooks/clip-zero-shot-classification-with-output.rst @@ -102,9 +102,9 @@ tokenizer and preparing the images. .. code:: ipython3 import platform - + %pip install -q --extra-index-url https://download.pytorch.org/whl/cpu "gradio>=4.19" "openvino>=2023.1.0" "transformers[torch]>=4.30" "datasets" "nncf>=2.6.0" "torch>=2.1" Pillow - + if platform.system() != "Windows": %pip install -q "matplotlib>=3.4" else: @@ -119,7 +119,7 @@ tokenizer and preparing the images. .. code:: ipython3 from transformers import CLIPProcessor, CLIPModel - + # load pre-trained model model = CLIPModel.from_pretrained("openai/clip-vit-base-patch16") # load preprocessor for model input @@ -141,8 +141,8 @@ tokenizer and preparing the images. import matplotlib.pyplot as plt import numpy as np from PIL import Image - - + + def visualize_result(image: Image, labels: List[str], probs: np.ndarray, top: int = 5): """ Utility function for visualization classification results @@ -160,7 +160,7 @@ tokenizer and preparing the images. plt.subplot(8, 8, 1) plt.imshow(image) plt.axis("off") - + plt.subplot(8, 8, 2) y = np.arange(top_probs.shape[-1]) plt.grid() @@ -189,17 +189,17 @@ similarity score for the final result. import requests from pathlib import Path - - + + sample_path = Path("data/coco.jpg") sample_path.parent.mkdir(parents=True, exist_ok=True) r = requests.get("https://storage.openvinotoolkit.org/repositories/openvino_notebooks/data/data/image/coco.jpg") - + with sample_path.open("wb") as f: f.write(r.content) - + image = Image.open(sample_path) - + input_labels = [ "cat", "dog", @@ -213,9 +213,9 @@ similarity score for the final result. "computer", ] text_descriptions = [f"This is a photo of a {label}" for label in input_labels] - + inputs = processor(text=text_descriptions, images=[image], return_tensors="pt", padding=True) - + results = model(**inputs) logits_per_image = results["logits_per_image"] # this is the image-text similarity score probs = logits_per_image.softmax(dim=1).detach().numpy() # we can take the softmax to get the label probabilities @@ -243,10 +243,10 @@ save it on disk for the next usage with ``ov.save_model``. .. code:: ipython3 import openvino as ov - + fp16_model_path = Path("clip-vit-base-patch16.xml") model.config.torchscript = True - + if not fp16_model_path.exists(): ov_model = ov.convert_model(model, example_input=dict(inputs)) ov.save_model(ov_model, fp16_model_path) @@ -263,7 +263,7 @@ same input data from the example above with PyTorch. .. code:: ipython3 from scipy.special import softmax - + # create OpenVINO core object instance core = ov.Core() @@ -277,14 +277,14 @@ select device from dropdown list for running inference using OpenVINO .. code:: ipython3 import ipywidgets as widgets - + device = widgets.Dropdown( options=core.available_devices + ["AUTO"], value="AUTO", description="Device:", disabled=False, ) - + device @@ -317,8 +317,6 @@ Great! Looks like we got the same result. Quantize model to INT8 using NNCF --------------------------------- - ## Quantize model to INT8 using -NNCF The goal of this part of tutorial is to demonstrate how to speed up the model by applying 8-bit post-training quantization from @@ -349,7 +347,7 @@ inference faster. The optimization process contains the following steps: description="Quantization", disabled=False, ) - + to_quantize @@ -368,7 +366,7 @@ inference faster. The optimization process contains the following steps: url="https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/latest/utils/skip_kernel_extension.py", ) open("skip_kernel_extension.py", "w").write(r.text) - + %load_ext skip_kernel_extension Prepare datasets @@ -384,16 +382,16 @@ model. .. code:: ipython3 %%skip not $to_quantize.value - + import requests from io import BytesIO import numpy as np from PIL import Image from requests.packages.urllib3.exceptions import InsecureRequestWarning requests.packages.urllib3.disable_warnings(InsecureRequestWarning) - + max_length = model.config.text_config.max_position_embeddings - + def check_text_data(data): """ Check if the given data is text-based. @@ -403,7 +401,7 @@ model. if isinstance(data, list): return all(isinstance(x, str) for x in data) return False - + def get_pil_from_url(url): """ Downloads and converts an image from a URL to a PIL Image object. @@ -411,7 +409,7 @@ model. response = requests.get(url, verify=False, timeout=20) image = Image.open(BytesIO(response.content)) return image.convert("RGB") - + def collate_fn(example, image_column="image_url", text_column="caption"): """ Preprocesses an example by loading and transforming image and text data. @@ -422,10 +420,10 @@ model. """ assert len(example) == 1 example = example[0] - + if not check_text_data(example[text_column]): raise ValueError("Text data is not valid") - + url = example[image_column] try: image = get_pil_from_url(url) @@ -434,7 +432,7 @@ model. return None except Exception: return None - + inputs = processor(text=example[text_column], images=[image], return_tensors="pt", padding=True) if inputs['input_ids'].shape[1] > max_length: return None @@ -443,11 +441,11 @@ model. .. code:: ipython3 %%skip not $to_quantize.value - + import torch from datasets import load_dataset from tqdm.notebook import tqdm - + def prepare_calibration_data(dataloader, init_steps): """ This function prepares calibration data from a dataloader for a specified number of initialization steps. @@ -470,13 +468,13 @@ model. } ) return data - - + + def prepare_dataset(opt_init_steps=50, max_train_samples=1000): """ Prepares a vision-text dataset for quantization. """ - dataset = load_dataset("conceptual_captions") + dataset = load_dataset("google-research-datasets/conceptual_captions", trust_remote_code=True) train_dataset = dataset["train"].shuffle(seed=42) dataloader = torch.utils.data.DataLoader(train_dataset, collate_fn=collate_fn, batch_size=1) calibration_data = prepare_calibration_data(dataloader, opt_init_steps) @@ -485,14 +483,14 @@ model. .. code:: ipython3 %%skip not $to_quantize.value - + import logging import nncf - + core = ov.Core() - + nncf.set_log_level(logging.ERROR) - + int8_model_path = 'clip-vit-base-patch16_int8.xml' calibration_data = prepare_dataset() ov_model = core.read_model(fp16_model_path) @@ -535,12 +533,12 @@ Create a quantized model from the pre-trained ``FP16`` model. .. code:: ipython3 %%skip not $to_quantize.value - + if len(calibration_data) == 0: raise RuntimeError( 'Calibration dataset is empty. Please check internet connection and try to download images manually.' ) - + calibration_dataset = nncf.Dataset(calibration_data) quantized_model = nncf.quantize( model=ov_model, @@ -654,7 +652,7 @@ the same input data that we used before. .. code:: ipython3 %%skip not $to_quantize.value - + # compile model for loading on device compiled_model = core.compile_model(quantized_model, device.value) # run inference on preprocessed data and get image-text similarity score @@ -679,9 +677,9 @@ Compare File Size .. code:: ipython3 %%skip not $to_quantize.value - + from pathlib import Path - + fp16_ir_model_size = Path(fp16_model_path).with_suffix(".bin").stat().st_size / 1024 / 1024 quantized_model_size = Path(int8_model_path).with_suffix(".bin").stat().st_size / 1024 / 1024 print(f"FP16 IR model size: {fp16_ir_model_size:.2f} MB") @@ -711,9 +709,9 @@ up of the dynamic quantized models. .. code:: ipython3 %%skip not $to_quantize.value - + import time - + def calculate_inference_time(model_path, calibration_data): model = core.compile_model(model_path, device.value) inference_time = [] @@ -728,7 +726,7 @@ up of the dynamic quantized models. .. code:: ipython3 %%skip not $to_quantize.value - + fp16_latency = calculate_inference_time(fp16_model_path, calibration_data) int8_latency = calculate_inference_time(int8_model_path, calibration_data) print(f"Performance speed up: {fp16_latency / int8_latency:.3f}") @@ -742,7 +740,6 @@ up of the dynamic quantized models. Interactive demo ---------------- - ## Interactive demo Now, it is your turn! You can provide your own image and comma-separated list of labels for zero-shot classification. @@ -754,13 +751,13 @@ example, ``cat,dog,bird``) .. code:: ipython3 import gradio as gr - + model_path = Path("clip-vit-base-patch16-int8.xml") if not model_path.exists(): model_path = Path("clip-vit-base-patch16.xml") compiled_model = core.compile_model(model_path, device.value) - - + + def classify(image, text): """Classify image using classes listing. Args: @@ -774,10 +771,10 @@ example, ``cat,dog,bird``) inputs = processor(text=text_descriptions, images=[image], return_tensors="np", padding=True) ov_logits_per_image = compiled_model(dict(inputs))[0] probs = softmax(ov_logits_per_image, axis=1)[0] - + return {label: float(prob) for label, prob in zip(labels, probs)} - - + + demo = gr.Interface( classify, [ diff --git a/docs/notebooks/convert-to-openvino-with-output.rst b/docs/notebooks/convert-to-openvino-with-output.rst index 60b730e4f491cf..cfb48f56a06e72 100644 --- a/docs/notebooks/convert-to-openvino-with-output.rst +++ b/docs/notebooks/convert-to-openvino-with-output.rst @@ -35,7 +35,7 @@ Table of contents: .. parsed-literal:: - Requirement already satisfied: pip in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (24.0) + Requirement already satisfied: pip in /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-708/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages (24.0) Note: you may need to restart the kernel to use updated packages. Note: you may need to restart the kernel to use updated packages. @@ -181,13 +181,11 @@ NLP model from Hugging Face and export it in ONNX format: .. parsed-literal:: - 2024-05-15 23:49:16.064636: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. - 2024-05-15 23:49:16.099980: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. + 2024-06-19 23:03:42.267090: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. + 2024-06-19 23:03:42.302578: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. - 2024-05-15 23:49:16.617080: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`. - warnings.warn( - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/distilbert/modeling_distilbert.py:234: TracerWarning: torch.tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. + 2024-06-19 23:03:42.815203: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-708/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/transformers/models/distilbert/modeling_distilbert.py:231: TracerWarning: torch.tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect. mask, torch.tensor(torch.finfo(scores.dtype).min) @@ -664,12 +662,12 @@ frameworks conversion guides. .. parsed-literal:: - 2024-05-15 23:49:36.583572: E tensorflow/compiler/xla/stream_executor/cuda/cuda_driver.cc:266] failed call to cuInit: CUDA_ERROR_COMPAT_NOT_SUPPORTED_ON_DEVICE: forward compatibility was attempted on non supported HW - 2024-05-15 23:49:36.583606: I tensorflow/compiler/xla/stream_executor/cuda/cuda_diagnostics.cc:168] retrieving CUDA diagnostic information for host: iotg-dev-workstation-07 - 2024-05-15 23:49:36.583610: I tensorflow/compiler/xla/stream_executor/cuda/cuda_diagnostics.cc:175] hostname: iotg-dev-workstation-07 - 2024-05-15 23:49:36.583842: I tensorflow/compiler/xla/stream_executor/cuda/cuda_diagnostics.cc:199] libcuda reported version is: 470.223.2 - 2024-05-15 23:49:36.583866: I tensorflow/compiler/xla/stream_executor/cuda/cuda_diagnostics.cc:203] kernel reported version is: 470.182.3 - 2024-05-15 23:49:36.583871: E tensorflow/compiler/xla/stream_executor/cuda/cuda_diagnostics.cc:312] kernel version 470.182.3 does not match DSO version 470.223.2 -- cannot find working devices in this configuration + 2024-06-19 23:04:02.230970: E tensorflow/compiler/xla/stream_executor/cuda/cuda_driver.cc:266] failed call to cuInit: CUDA_ERROR_COMPAT_NOT_SUPPORTED_ON_DEVICE: forward compatibility was attempted on non supported HW + 2024-06-19 23:04:02.231002: I tensorflow/compiler/xla/stream_executor/cuda/cuda_diagnostics.cc:168] retrieving CUDA diagnostic information for host: iotg-dev-workstation-07 + 2024-06-19 23:04:02.231006: I tensorflow/compiler/xla/stream_executor/cuda/cuda_diagnostics.cc:175] hostname: iotg-dev-workstation-07 + 2024-06-19 23:04:02.231217: I tensorflow/compiler/xla/stream_executor/cuda/cuda_diagnostics.cc:199] libcuda reported version is: 470.223.2 + 2024-06-19 23:04:02.231235: I tensorflow/compiler/xla/stream_executor/cuda/cuda_diagnostics.cc:203] kernel reported version is: 470.182.3 + 2024-06-19 23:04:02.231238: E tensorflow/compiler/xla/stream_executor/cuda/cuda_diagnostics.cc:312] kernel version 470.182.3 does not match DSO version 470.223.2 -- cannot find working devices in this configuration Migration from Legacy conversion API @@ -727,6 +725,13 @@ Resnet50 model that was exported to the ONNX format: ov_model = mo.convert_model(ONNX_CV_MODEL_PATH, layout="nchw") +.. parsed-literal:: + + [ INFO ] MO command line tool is considered as the legacy conversion API as of OpenVINO 2023.2 release. + In 2025.0 MO command line tool and openvino.tools.mo.convert_model() will be removed. Please use OpenVINO Model Converter (OVC) or openvino.convert_model(). OVC represents a lightweight alternative of MO and provides simplified model conversion API. + Find more information about transition from MO to OVC at https://docs.openvino.ai/2023.2/openvino_docs_OV_Converter_UG_prepare_model_convert_model_MO_OVC_transition.html + + .. parsed-literal:: huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks... @@ -770,6 +775,17 @@ and the layout of an original model: # alternatively use source_layout and target_layout parameters ov_model = mo.convert_model(ONNX_CV_MODEL_PATH, source_layout="nchw", target_layout="nhwc") + +.. parsed-literal:: + + [ INFO ] MO command line tool is considered as the legacy conversion API as of OpenVINO 2023.2 release. + In 2025.0 MO command line tool and openvino.tools.mo.convert_model() will be removed. Please use OpenVINO Model Converter (OVC) or openvino.convert_model(). OVC represents a lightweight alternative of MO and provides simplified model conversion API. + Find more information about transition from MO to OVC at https://docs.openvino.ai/2023.2/openvino_docs_OV_Converter_UG_prepare_model_convert_model_MO_OVC_transition.html + [ INFO ] MO command line tool is considered as the legacy conversion API as of OpenVINO 2023.2 release. + In 2025.0 MO command line tool and openvino.tools.mo.convert_model() will be removed. Please use OpenVINO Model Converter (OVC) or openvino.convert_model(). OVC represents a lightweight alternative of MO and provides simplified model conversion API. + Find more information about transition from MO to OVC at https://docs.openvino.ai/2023.2/openvino_docs_OV_Converter_UG_prepare_model_convert_model_MO_OVC_transition.html + + Specifying Mean and Scale Values ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -808,6 +824,14 @@ more examples. scale_values=[255 * x for x in [0.229, 0.224, 0.225]], ) + +.. parsed-literal:: + + [ INFO ] MO command line tool is considered as the legacy conversion API as of OpenVINO 2023.2 release. + In 2025.0 MO command line tool and openvino.tools.mo.convert_model() will be removed. Please use OpenVINO Model Converter (OVC) or openvino.convert_model(). OVC represents a lightweight alternative of MO and provides simplified model conversion API. + Find more information about transition from MO to OVC at https://docs.openvino.ai/2023.2/openvino_docs_OV_Converter_UG_prepare_model_convert_model_MO_OVC_transition.html + + Reversing Input Channels ^^^^^^^^^^^^^^^^^^^^^^^^ @@ -838,6 +862,14 @@ the color channels before inference. ov_model = mo.convert_model(ONNX_CV_MODEL_PATH, reverse_input_channels=True) + +.. parsed-literal:: + + [ INFO ] MO command line tool is considered as the legacy conversion API as of OpenVINO 2023.2 release. + In 2025.0 MO command line tool and openvino.tools.mo.convert_model() will be removed. Please use OpenVINO Model Converter (OVC) or openvino.convert_model(). OVC represents a lightweight alternative of MO and provides simplified model conversion API. + Find more information about transition from MO to OVC at https://docs.openvino.ai/2023.2/openvino_docs_OV_Converter_UG_prepare_model_convert_model_MO_OVC_transition.html + + Cutting Off Parts of a Model ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/docs/notebooks/convnext-classification-with-output.rst b/docs/notebooks/convnext-classification-with-output.rst index 692982069bda82..a7c4bd19ccf556 100644 --- a/docs/notebooks/convnext-classification-with-output.rst +++ b/docs/notebooks/convnext-classification-with-output.rst @@ -50,9 +50,9 @@ Prerequisites .. parsed-literal:: - DEPRECATION: pytorch-lightning 1.6.5 has a non-standard dependency specifier torch>=1.8.*. pip 24.1 will enforce this behaviour change. A possible replacement is to upgrade to a newer version of pytorch-lightning or contact the author to suggest that they release a version with a conforming dependency specifiers. Discussion can be found at https://github.com/pypa/pip/issues/12063 + DEPRECATION: pytorch-lightning 1.6.3 has a non-standard dependency specifier torch>=1.8.*. pip 24.1 will enforce this behaviour change. A possible replacement is to upgrade to a newer version of pytorch-lightning or contact the author to suggest that they release a version with a conforming dependency specifiers. Discussion can be found at https://github.com/pypa/pip/issues/12063 Note: you may need to restart the kernel to use updated packages. - DEPRECATION: pytorch-lightning 1.6.5 has a non-standard dependency specifier torch>=1.8.*. pip 24.1 will enforce this behaviour change. A possible replacement is to upgrade to a newer version of pytorch-lightning or contact the author to suggest that they release a version with a conforming dependency specifiers. Discussion can be found at https://github.com/pypa/pip/issues/12063 + DEPRECATION: pytorch-lightning 1.6.3 has a non-standard dependency specifier torch>=1.8.*. pip 24.1 will enforce this behaviour change. A possible replacement is to upgrade to a newer version of pytorch-lightning or contact the author to suggest that they release a version with a conforming dependency specifiers. Discussion can be found at https://github.com/pypa/pip/issues/12063 Note: you may need to restart the kernel to use updated packages. @@ -183,7 +183,7 @@ And print results Predicted Class: 281 Predicted Label: n02123045 tabby, tabby cat - Predicted Probability: 0.4661690592765808 + Predicted Probability: 0.5351971983909607 Convert the model to OpenVINO Intermediate representation format diff --git a/docs/notebooks/cross-lingual-books-alignment-with-output.rst b/docs/notebooks/cross-lingual-books-alignment-with-output.rst index fb79d3ab19683e..f80b3eb809c33f 100644 --- a/docs/notebooks/cross-lingual-books-alignment-with-output.rst +++ b/docs/notebooks/cross-lingual-books-alignment-with-output.rst @@ -197,7 +197,7 @@ which in a raw format looks like this: .. parsed-literal:: - '\ufeffThe Project Gutenberg eBook of Anna Karenina\r\n \r\nThis ebook is for the use of anyone anywhere in the United States and\r\nmost other parts of the world at no cost and with almost no restrictions\r\nwhatsoever. You may copy it, give it away or re-use it under the terms\r\nof the Project Gutenberg License included with this ebook or online\r\nat www.gutenberg.org. If you are not located in the United States,\r\nyou will have to check the laws of the country where you are located\r\nbefore using this eBook.\r\n\r\nTitle: Anna Karenina\r\n\r\n\r\nAuthor: graf Leo Tolstoy\r\n\r\nTranslator: Constance Garnett\r\n\r\nRelease date: July 1, 1998 [eBook #1399]\r\n Most recently updated: April 9, 2023\r\n\r\nLanguage: English\r\n\r\n\r\n\r\n*** START OF THE PROJECT GUTENBERG EBOOK ANNA KARENINA \*\*\*\r\n[Illustration]\r\n\r\n\r\n\r\n\r\n ANNA KARENINA \r\n\r\n by Leo Tolstoy \r\n\r\n Translated by Constance Garnett \r\n\r\nContents\r\n\r\n\r\n PART ONE\r\n PART TWO\r\n PART THREE\r\n PART FOUR\r\n PART FIVE\r\n PART SIX\r\n PART SEVEN\r\n PART EIGHT\r\n\r\n\r\n\r\n\r\nPART ONE\r\n\r\nChapter 1\r\n\r\n\r\nHappy families are all alike; every unhappy family is unhappy in its\r\nown way.\r\n\r\nEverything was in confusion in the Oblonskys’ house. The wife had\r\ndiscovered that the husband was carrying on an intrigue with a French\r\ngirl, who had been a governess in their family, and she had announced\r\nto her husband that she could not go on living in the same house with\r\nhim. This position of affairs had now lasted three days, and not only\r\nthe husband and wife themselves, but all the me' + '\ufeffThe Project Gutenberg eBook of Anna Karenina\r\n \r\nThis ebook is for the use of anyone anywhere in the United States and\r\nmost other parts of the world at no cost and with almost no restrictions\r\nwhatsoever. You may copy it, give it away or re-use it under the terms\r\nof the Project Gutenberg License included with this ebook or online\r\nat www.gutenberg.org. If you are not located in the United States,\r\nyou will have to check the laws of the country where you are located\r\nbefore using this eBook.\r\n\r\nTitle: Anna Karenina\r\n\r\n\r\nAuthor: graf Leo Tolstoy\r\n\r\nTranslator: Constance Garnett\r\n\r\nRelease date: July 1, 1998 [eBook #1399]\r\n Most recently updated: April 9, 2023\r\n\r\nLanguage: English\r\n\r\n\r\n\r\n\*\*\* START OF THE PROJECT GUTENBERG EBOOK ANNA KARENINA \*\*\*\r\n[Illustration]\r\n\r\n\r\n\r\n\r\n ANNA KARENINA \r\n\r\n by Leo Tolstoy \r\n\r\n Translated by Constance Garnett \r\n\r\nContents\r\n\r\n\r\n PART ONE\r\n PART TWO\r\n PART THREE\r\n PART FOUR\r\n PART FIVE\r\n PART SIX\r\n PART SEVEN\r\n PART EIGHT\r\n\r\n\r\n\r\n\r\nPART ONE\r\n\r\nChapter 1\r\n\r\n\r\nHappy families are all alike; every unhappy family is unhappy in its\r\nown way.\r\n\r\nEverything was in confusion in the Oblonskys’ house. The wife had\r\ndiscovered that the husband was carrying on an intrigue with a French\r\ngirl, who had been a governess in their family, and she had announced\r\nto her husband that she could not go on living in the same house with\r\nhim. This position of affairs had now lasted three days, and not only\r\nthe husband and wife themselves, but all the me' @@ -210,7 +210,7 @@ which in a raw format looks like this: .. parsed-literal:: - 'The Project Gutenberg EBook of Anna Karenina, 1. Band, by Leo N. Tolstoi\r\n\r\nThis eBook is for the use of anyone anywhere at no cost and with\r\nalmost no restrictions whatsoever. You may copy it, give it away or\r\nre-use it under the terms of the Project Gutenberg License included\r\nwith this eBook or online at www.gutenberg.org\r\n\r\n\r\nTitle: Anna Karenina, 1. Band\r\n\r\nAuthor: Leo N. Tolstoi\r\n\r\nRelease Date: February 18, 2014 [EBook #44956]\r\n\r\nLanguage: German\r\n\r\nCharacter set encoding: ISO-8859-1\r\n\r\n\*\*\* START OF THIS PROJECT GUTENBERG EBOOK ANNA KARENINA, 1. BAND \*\*\*\r\n\r\n\r\n\r\n\r\nProduced by Norbert H. Langkau, Jens Nordmann and the\r\nOnline Distributed Proofreading Team at http://www.pgdp.net\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n Anna Karenina.\r\n\r\n\r\n Roman aus dem Russischen\r\n\r\n des\r\n\r\n Grafen Leo N. Tolstoi.\r\n\r\n\r\n\r\n Nach der siebenten Auflage übersetzt\r\n\r\n von\r\n\r\n Hans Moser.\r\n\r\n\r\n Erster Band.\r\n\r\n\r\n\r\n Leipzig\r\n\r\n Druck und Verlag von Philipp Reclam jun.\r\n\r\n \* \* \* \* *\r\n\r\n\r\n\r\n\r\n Erster Teil.\r\n\r\n »Die Rache ist mein, ich will vergelten.«\r\n\r\n 1.\r\n\r\n\r\nAlle glücklichen Familien sind einander ähnlich; jede unglücklich' + 'The Project Gutenberg EBook of Anna Karenina, 1. Band, by Leo N. Tolstoi\r\n\r\nThis eBook is for the use of anyone anywhere at no cost and with\r\nalmost no restrictions whatsoever. You may copy it, give it away or\r\nre-use it under the terms of the Project Gutenberg License included\r\nwith this eBook or online at www.gutenberg.org\r\n\r\n\r\nTitle: Anna Karenina, 1. Band\r\n\r\nAuthor: Leo N. Tolstoi\r\n\r\nRelease Date: February 18, 2014 [EBook #44956]\r\n\r\nLanguage: German\r\n\r\nCharacter set encoding: ISO-8859-1\r\n\r\n\*\*\* START OF THIS PROJECT GUTENBERG EBOOK ANNA KARENINA, 1. BAND \*\*\*\r\n\r\n\r\n\r\n\r\nProduced by Norbert H. Langkau, Jens Nordmann and the\r\nOnline Distributed Proofreading Team at http://www.pgdp.net\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n Anna Karenina.\r\n\r\n\r\n Roman aus dem Russischen\r\n\r\n des\r\n\r\n Grafen Leo N. Tolstoi.\r\n\r\n\r\n\r\n Nach der siebenten Auflage übersetzt\r\n\r\n von\r\n\r\n Hans Moser.\r\n\r\n\r\n Erster Band.\r\n\r\n\r\n\r\n Leipzig\r\n\r\n Druck und Verlag von Philipp Reclam jun.\r\n\r\n * * * * *\r\n\r\n\r\n\r\n\r\n Erster Teil.\r\n\r\n »Die Rache ist mein, ich will vergelten.«\r\n\r\n 1.\r\n\r\n\r\nAlle glücklichen Familien sind einander ähnlich; jede unglücklich' @@ -407,12 +407,12 @@ languages. It has the same architecture as the BERT model but has been trained on a different task: to produce identical embeddings for translation pairs. -|image1| +|image01| This makes LaBSE a great choice for our task and it can be reused for different language pairs still producing good results. -.. |image1| image:: https://github.com/openvinotoolkit/openvino_notebooks/assets/29454499/627d3a39-7076-479f-a7b1-392f49a0b83e +.. |image01| image:: https://github.com/openvinotoolkit/openvino_notebooks/assets/29454499/627d3a39-7076-479f-a7b1-392f49a0b83e .. code:: ipython3 diff --git a/docs/notebooks/ct-segmentation-quantize-nncf-with-output.rst b/docs/notebooks/ct-segmentation-quantize-nncf-with-output.rst index c2991dc95a36dd..72da3069d39ff1 100644 --- a/docs/notebooks/ct-segmentation-quantize-nncf-with-output.rst +++ b/docs/notebooks/ct-segmentation-quantize-nncf-with-output.rst @@ -152,10 +152,10 @@ Imports .. parsed-literal:: - 2024-05-15 23:50:56.536543: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. - 2024-05-15 23:50:56.573924: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. + 2024-06-19 23:05:23.286795: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. + 2024-06-19 23:05:23.321455: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. - 2024-05-15 23:50:57.174238: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT + 2024-06-19 23:05:23.900660: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT .. parsed-literal:: @@ -435,7 +435,7 @@ this notebook. .. parsed-literal:: [ WARNING ] Please fix your imports. Module %s has been moved to %s. The old module will be deleted in version %s. - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/monai/networks/nets/basic_unet.py:168: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-708/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/monai/networks/nets/basic_unet.py:168: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if x_e.shape[-i - 1] != x_0.shape[-i - 1]: @@ -534,18 +534,18 @@ Convert quantized model to OpenVINO IR model and save it. .. parsed-literal:: - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/torch/quantization/layers.py:337: TracerWarning: Converting a tensor to a Python number might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-708/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/torch/quantization/layers.py:339: TracerWarning: Converting a tensor to a Python number might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! return self._level_low.item() - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/torch/quantization/layers.py:345: TracerWarning: Converting a tensor to a Python number might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-708/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/nncf/torch/quantization/layers.py:347: TracerWarning: Converting a tensor to a Python number might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! return self._level_high.item() - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/monai/networks/nets/basic_unet.py:168: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-708/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/monai/networks/nets/basic_unet.py:168: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs! if x_e.shape[-i - 1] != x_0.shape[-i - 1]: - /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/jit/_trace.py:1116: TracerWarning: Output nr 1. of the traced function does not match the corresponding output of the Python function. Detailed error: + /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-708/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/torch/jit/_trace.py:1116: TracerWarning: Output nr 1. of the traced function does not match the corresponding output of the Python function. Detailed error: Tensor-likes are not close! - Mismatched elements: 247888 / 262144 (94.6%) - Greatest absolute difference: 6.055658936500549 at index (0, 0, 158, 273) (up to 1e-05 allowed) - Greatest relative difference: 13834.454856259192 at index (0, 0, 343, 273) (up to 1e-05 allowed) + Mismatched elements: 249385 / 262144 (95.1%) + Greatest absolute difference: 3.856969118118286 at index (0, 0, 411, 46) (up to 1e-05 allowed) + Greatest relative difference: 60858.36910067906 at index (0, 0, 89, 134) (up to 1e-05 allowed) _check_trace( @@ -668,18 +668,18 @@ be run in the notebook with ``! benchmark_app`` or [ INFO ] Parsing input parameters [Step 2/11] Loading OpenVINO Runtime [ INFO ] OpenVINO: - [ INFO ] Build ................................. 2024.1.0-15008-f4afc983258-releases/2024/1 + [ INFO ] Build ................................. 2024.2.0-15519-5c0f38f83f6-releases/2024/2 [ INFO ] [ INFO ] Device info: [ INFO ] AUTO - [ INFO ] Build ................................. 2024.1.0-15008-f4afc983258-releases/2024/1 + [ INFO ] Build ................................. 2024.2.0-15519-5c0f38f83f6-releases/2024/2 [ INFO ] [ INFO ] [Step 3/11] Setting device configuration [ WARNING ] Performance hint was not explicitly specified in command line. Device(AUTO) performance hint will be set to PerformanceMode.LATENCY. [Step 4/11] Reading model files [ INFO ] Loading model files - [ INFO ] Read model took 8.68 ms + [ INFO ] Read model took 9.09 ms [ INFO ] Original model I/O parameters: [ INFO ] Model inputs: [ INFO ] x (node: x) : f32 / [...] / [?,?,?,?] @@ -693,7 +693,7 @@ be run in the notebook with ``! benchmark_app`` or [ INFO ] Model outputs: [ INFO ] ***NO_NAME*** (node: __module.final_conv/aten::_convolution/Add) : f32 / [...] / [?,1,16..,16..] [Step 7/11] Loading the model to the device - [ INFO ] Compile model took 149.48 ms + [ INFO ] Compile model took 185.62 ms [Step 8/11] Querying optimal runtime parameters [ INFO ] Model: [ INFO ] NETWORK_NAME: Model0 @@ -728,9 +728,9 @@ be run in the notebook with ``! benchmark_app`` or [Step 9/11] Creating infer requests and preparing input tensors [ ERROR ] Input x is dynamic. Provide data shapes! Traceback (most recent call last): - File "/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/tools/benchmark/main.py", line 486, in main + File "/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-708/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/tools/benchmark/main.py", line 486, in main data_queue = get_input_data(paths_to_input, app_inputs_info) - File "/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/tools/benchmark/utils/inputs_filling.py", line 123, in get_input_data + File "/opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-708/.workspace/scm/ov-notebook/.venv/lib/python3.8/site-packages/openvino/tools/benchmark/utils/inputs_filling.py", line 123, in get_input_data raise Exception(f"Input {info.name} is dynamic. Provide data shapes!") Exception: Input x is dynamic. Provide data shapes! @@ -747,18 +747,18 @@ be run in the notebook with ``! benchmark_app`` or [ INFO ] Parsing input parameters [Step 2/11] Loading OpenVINO Runtime [ INFO ] OpenVINO: - [ INFO ] Build ................................. 2024.1.0-15008-f4afc983258-releases/2024/1 + [ INFO ] Build ................................. 2024.2.0-15519-5c0f38f83f6-releases/2024/2 [ INFO ] [ INFO ] Device info: [ INFO ] AUTO - [ INFO ] Build ................................. 2024.1.0-15008-f4afc983258-releases/2024/1 + [ INFO ] Build ................................. 2024.2.0-15519-5c0f38f83f6-releases/2024/2 [ INFO ] [ INFO ] [Step 3/11] Setting device configuration [ WARNING ] Performance hint was not explicitly specified in command line. Device(AUTO) performance hint will be set to PerformanceMode.LATENCY. [Step 4/11] Reading model files [ INFO ] Loading model files - [ INFO ] Read model took 10.82 ms + [ INFO ] Read model took 10.55 ms [ INFO ] Original model I/O parameters: [ INFO ] Model inputs: [ INFO ] x (node: x) : f32 / [...] / [1,1,512,512] @@ -772,7 +772,7 @@ be run in the notebook with ``! benchmark_app`` or [ INFO ] Model outputs: [ INFO ] ***NO_NAME*** (node: __module.final_conv/aten::_convolution/Add) : f32 / [...] / [1,1,512,512] [Step 7/11] Loading the model to the device - [ INFO ] Compile model took 257.40 ms + [ INFO ] Compile model took 226.95 ms [Step 8/11] Querying optimal runtime parameters [ INFO ] Model: [ INFO ] NETWORK_NAME: Model49 @@ -809,17 +809,17 @@ be run in the notebook with ``! benchmark_app`` or [ INFO ] Fill input 'x' with random values [Step 10/11] Measuring performance (Start inference synchronously, limits: 15000 ms duration) [ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop). - [ INFO ] First inference took 27.38 ms + [ INFO ] First inference took 28.80 ms [Step 11/11] Dumping statistics report [ INFO ] Execution Devices:['CPU'] - [ INFO ] Count: 959 iterations - [ INFO ] Duration: 15011.96 ms + [ INFO ] Count: 971 iterations + [ INFO ] Duration: 15015.25 ms [ INFO ] Latency: - [ INFO ] Median: 15.40 ms - [ INFO ] Average: 15.45 ms - [ INFO ] Min: 15.18 ms - [ INFO ] Max: 17.18 ms - [ INFO ] Throughput: 63.88 FPS + [ INFO ] Median: 15.21 ms + [ INFO ] Average: 15.27 ms + [ INFO ] Min: 14.93 ms + [ INFO ] Max: 17.10 ms + [ INFO ] Throughput: 64.67 FPS Visually Compare Inference Results @@ -904,7 +904,7 @@ seed is displayed to enable reproducing specific runs of this cell. .. parsed-literal:: - Visualizing results with seed 1715809926 + Visualizing results with seed 1718831192 @@ -988,7 +988,7 @@ performs inference, and displays the results on the frames loaded in .. parsed-literal:: Loaded model to AUTO in 0.21 seconds. - Total time for 68 frames: 2.72 seconds, fps:25.33 + Total time for 68 frames: 2.38 seconds, fps:28.96 References diff --git a/docs/notebooks/ct-segmentation-quantize-nncf-with-output_files/ct-segmentation-quantize-nncf-with-output_37_1.png b/docs/notebooks/ct-segmentation-quantize-nncf-with-output_files/ct-segmentation-quantize-nncf-with-output_37_1.png index b3af95598c688c..a71579735b16da 100644 --- a/docs/notebooks/ct-segmentation-quantize-nncf-with-output_files/ct-segmentation-quantize-nncf-with-output_37_1.png +++ b/docs/notebooks/ct-segmentation-quantize-nncf-with-output_files/ct-segmentation-quantize-nncf-with-output_37_1.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:28010020834c2072a301b1a4eab4743fe594249fd6868e415af89e0dbc74892e -size 383860 +oid sha256:b504e816b8d13dae3137547bf06b49ff832d21e402fdaa6beb4d23f042004a89 +size 383906 diff --git a/docs/notebooks/ddcolor-image-colorization-with-output.rst b/docs/notebooks/ddcolor-image-colorization-with-output.rst new file mode 100644 index 00000000000000..b05d0e30d4c49b --- /dev/null +++ b/docs/notebooks/ddcolor-image-colorization-with-output.rst @@ -0,0 +1,739 @@ +Colorize grayscale images using DDColor and OpenVINO +====================================================== + +Image colorization is the process of adding color to grayscale images. +Initially captured in black and white, these images are transformed into +vibrant, lifelike representations by estimating RGB colors. This +technology enhances both aesthetic appeal and perceptual quality. +Historically, artists manually applied colors to monochromatic +photographs, a painstaking task that could take up to a month for a +single image. However, with advancements in information technology and +the rise of deep neural networks, automated image colorization has +become increasingly important. + +DDColor is one of the most progressive methods of image colorization in +our days. It is a novel approach using dual decoders: a pixel decoder +and a query-based color decoder, that stands out in its ability to +produce photo-realistic colorization, particularly in complex scenes +with multiple objects and diverse contexts. |image0| + +More details about this approach can be found in original model +`repository `__ and +`paper `__. + +In this tutorial we consider how to convert and run DDColor using +OpenVINO. Additionally, we will demonstrate how to optimize this model +using `NNCF `__. + +🪄 Let’s start to explore magic of image colorization! #### Table of +contents: + +- `Prerequisites <#prerequisites>`__ +- `Load PyTorch model <#load-pytorch-model>`__ +- `Run PyTorch model inference <#run-pytorch-model-inference>`__ +- `Convert PyTorch model to OpenVINO Intermediate + Representation <#convert-pytorch-model-to-openvino-intermediate-representation>`__ +- `Run OpenVINO model inference <#run-openvino-model-inference>`__ +- `Optimize OpenVINO model using + NNCF <#optimize-openvino-model-using-nncf>`__ + + - `Collect quantization dataset <#collect-quantization-dataset>`__ + - `Perform model quantization <#perform-model-quantization>`__ + +- `Run INT8 model inference <#run-int8-model-inference>`__ +- `Compare FP16 and INT8 model + size <#compare-fp16-and-int8-model-size>`__ +- `Compare inference time of the FP16 and INT8 + models <#compare-inference-time-of-the-fp16-and-int8-models>`__ +- `Interactive inference <#interactive-inference>`__ + +.. |image0| image:: https://github.com/piddnad/DDColor/raw/master/assets/network_arch.jpg + +Prerequisites +------------- + + + +.. code:: ipython3 + + import platform + + %pip install -q "nncf>=2.11.0" "torch>=2.1" "torchvision" "timm" "opencv_python" "pillow" "PyYAML" "scipy" "scikit-image" "datasets" "gradio>=4.19" --extra-index-url https://download.pytorch.org/whl/cpu + %pip install -Uq --pre "openvino" --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly + + if platform.python_version_tuple()[1] in ["8", "9"]: + %pip install -q "gradio-imageslider<=0.0.17" "typing-extensions>=4.9.0" + else: + %pip install -q "gradio-imageslider" + + +.. parsed-literal:: + + Note: you may need to restart the kernel to use updated packages. + ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts. + openvino-dev 2024.2.0 requires openvino==2024.2.0, but you have openvino 2024.3.0.dev20240619 which is incompatible. + Note: you may need to restart the kernel to use updated packages. + Note: you may need to restart the kernel to use updated packages. + + +.. code:: ipython3 + + import sys + from pathlib import Path + + repo_dir = Path("DDColor") + + if not repo_dir.exists(): + !git clone https://github.com/piddnad/DDColor.git + + sys.path.append(str(repo_dir)) + + +.. parsed-literal:: + + Cloning into 'DDColor'... + remote: Enumerating objects: 230, done. + remote: Counting objects: 100% (76/76), done. + remote: Compressing objects: 100% (39/39), done. + remote: Total 230 (delta 54), reused 40 (delta 36), pack-reused 154 + Receiving objects: 100% (230/230), 13.34 MiB | 23.96 MiB/s, done. + Resolving deltas: 100% (75/75), done. + + +.. code:: ipython3 + + try: + from inference.colorization_pipeline_hf import DDColorHF, ImageColorizationPipelineHF + except Exception: + from inference.colorization_pipeline_hf import DDColorHF, ImageColorizationPipelineHF + +Load PyTorch model +------------------ + + + +There are several models from DDColor’s family provided in `model +repository `__. +We will use DDColor-T, the most lightweight version of ddcolor model, +but demonstrated in the tutorial steps are also applicable to other +models from DDColor family. + +.. code:: ipython3 + + import torch + + model_name = "ddcolor_paper_tiny" + + ddcolor_model = DDColorHF.from_pretrained(f"piddnad/{model_name}") + + + colorizer = ImageColorizationPipelineHF(model=ddcolor_model, input_size=512) + + ddcolor_model.to("cpu") + colorizer.device = torch.device("cpu") + +Run PyTorch model inference +--------------------------- + + + +.. code:: ipython3 + + import cv2 + import PIL + + IMG_PATH = "DDColor/assets/test_images/Ansel Adams _ Moore Photography.jpeg" + + + img = cv2.imread(IMG_PATH) + + PIL.Image.fromarray(img[:, :, ::-1]) + + + + +.. image:: ddcolor-image-colorization-with-output_files/ddcolor-image-colorization-with-output_8_0.png + + + +.. code:: ipython3 + + image_out = colorizer.process(img) + PIL.Image.fromarray(image_out[:, :, ::-1]) + + + + +.. image:: ddcolor-image-colorization-with-output_files/ddcolor-image-colorization-with-output_9_0.png + + + +Convert PyTorch model to OpenVINO Intermediate Representation +------------------------------------------------------------- + + + +OpenVINO supports PyTorch models via conversion to OpenVINO Intermediate +Representation (IR). OpenVINO model conversion API should be used for +these purposes. ``ov.convert_model`` function accepts original PyTorch +model instance and example input for tracing and returns ``ov.Model`` +representing this model in OpenVINO framework. Converted model can be +used for saving on disk using ``ov.save_model`` function or directly +loading on device using ``core.complie_model``. + +.. code:: ipython3 + + import openvino as ov + import torch + + OV_COLORIZER_PATH = Path("ddcolor.xml") + + if not OV_COLORIZER_PATH.exists(): + ov_model = ov.convert_model(ddcolor_model, example_input=torch.ones((1, 3, 512, 512)), input=[1, 3, 512, 512]) + ov.save_model(ov_model, OV_COLORIZER_PATH) + +Run OpenVINO model inference +---------------------------- + + + +Select one of supported devices for inference using dropdown list. + +.. code:: ipython3 + + import ipywidgets as widgets + + core = ov.Core() + + device = widgets.Dropdown( + options=core.available_devices + ["AUTO"], + value="AUTO", + description="Device:", + disabled=False, + ) + + device + + + + +.. parsed-literal:: + + Dropdown(description='Device:', index=1, options=('CPU', 'AUTO'), value='AUTO') + + + +.. code:: ipython3 + + compiled_model = core.compile_model(OV_COLORIZER_PATH, device.value) + +.. code:: ipython3 + + import cv2 + import numpy as np + import torch + import torch.nn.functional as F + + + def process(img, compiled_model): + # Preprocess input image + height, width = img.shape[:2] + + # Normalize to [0, 1] range + img = (img / 255.0).astype(np.float32) + orig_l = cv2.cvtColor(img, cv2.COLOR_BGR2Lab)[:, :, :1] # (h, w, 1) + + # Resize rgb image -> lab -> get grey -> rgb + img = cv2.resize(img, (512, 512)) + img_l = cv2.cvtColor(img, cv2.COLOR_BGR2Lab)[:, :, :1] + img_gray_lab = np.concatenate((img_l, np.zeros_like(img_l), np.zeros_like(img_l)), axis=-1) + img_gray_rgb = cv2.cvtColor(img_gray_lab, cv2.COLOR_LAB2RGB) + + # Transpose HWC -> CHW and add batch dimension + tensor_gray_rgb = torch.from_numpy(img_gray_rgb.transpose((2, 0, 1))).float().unsqueeze(0) + + # Run model inference + output_ab = compiled_model(tensor_gray_rgb)[0] + + # Postprocess result + # resize ab -> concat original l -> rgb + output_ab_resize = F.interpolate(torch.from_numpy(output_ab), size=(height, width))[0].float().numpy().transpose(1, 2, 0) + output_lab = np.concatenate((orig_l, output_ab_resize), axis=-1) + output_bgr = cv2.cvtColor(output_lab, cv2.COLOR_LAB2BGR) + + output_img = (output_bgr * 255.0).round().astype(np.uint8) + + return output_img + +.. code:: ipython3 + + ov_processed_img = process(img, compiled_model) + PIL.Image.fromarray(ov_processed_img[:, :, ::-1]) + + + + +.. image:: ddcolor-image-colorization-with-output_files/ddcolor-image-colorization-with-output_16_0.png + + + +Optimize OpenVINO model using NNCF +---------------------------------- + + + +`NNCF `__ enables +post-training quantization by adding quantization layers into model +graph and then using a subset of the training dataset to initialize the +parameters of these additional quantization layers. Quantized operations +are executed in ``INT8`` instead of ``FP32``/``FP16`` making model +inference faster. + +The optimization process contains the following steps: + +1. Create a calibration dataset for quantization. +2. Run ``nncf.quantize()`` to obtain quantized model. +3. Save the ``INT8`` model using ``openvino.save_model()`` function. + +Please select below whether you would like to run quantization to +improve model inference speed. + +.. code:: ipython3 + + to_quantize = widgets.Checkbox( + value=True, + description="Quantization", + disabled=False, + ) + + to_quantize + + + + +.. parsed-literal:: + + Checkbox(value=True, description='Quantization') + + + +.. code:: ipython3 + + import requests + + OV_INT8_COLORIZER_PATH = Path("ddcolor_int8.xml") + compiled_int8_model = None + + r = requests.get( + url="https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/latest/utils/skip_kernel_extension.py", + ) + open("skip_kernel_extension.py", "w").write(r.text) + + %load_ext skip_kernel_extension + +Collect quantization dataset +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + + +We use a portion of +`ummagumm-a/colorization_dataset `__ +dataset from Hugging Face as calibration data. + +.. code:: ipython3 + + %%skip not $to_quantize.value + + from datasets import load_dataset + + subset_size = 300 + calibration_data = [] + + if not OV_INT8_COLORIZER_PATH.exists(): + dataset = load_dataset("ummagumm-a/colorization_dataset", split="train", streaming=True).shuffle(seed=42).take(subset_size) + for idx, batch in enumerate(dataset): + if idx >= subset_size: + break + img = np.array(batch["conditioning_image"]) + img = (img / 255.0).astype(np.float32) + img = cv2.resize(img, (512, 512)) + img_l = cv2.cvtColor(np.stack([img, img, img], axis=2), cv2.COLOR_BGR2Lab)[:, :, :1] + img_gray_lab = np.concatenate((img_l, np.zeros_like(img_l), np.zeros_like(img_l)), axis=-1) + img_gray_rgb = cv2.cvtColor(img_gray_lab, cv2.COLOR_LAB2RGB) + + image = np.expand_dims(img_gray_rgb.transpose((2, 0, 1)).astype(np.float32), axis=0) + calibration_data.append(image) + +Perform model quantization +~~~~~~~~~~~~~~~~~~~~~~~~~~ + + + +.. code:: ipython3 + + %%skip not $to_quantize.value + + import nncf + + if not OV_INT8_COLORIZER_PATH.exists(): + ov_model = core.read_model(OV_COLORIZER_PATH) + quantized_model = nncf.quantize( + model=ov_model, + subset_size=subset_size, + calibration_dataset=nncf.Dataset(calibration_data), + ) + ov.save_model(quantized_model, OV_INT8_COLORIZER_PATH) + + +.. parsed-literal:: + + INFO:nncf:NNCF initialized successfully. Supported frameworks detected: torch, tensorflow, onnx, openvino + + +.. parsed-literal:: + + 2024-06-19 23:08:49.715756: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. + 2024-06-19 23:08:49.755077: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. + To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. + 2024-06-19 23:08:50.152911: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT + + + +.. parsed-literal:: + + Output() + + + +.. raw:: html + + 

+
+
+
+
+.. raw:: html
+
+    
+    

+
+
+
+
+.. parsed-literal::
+
+    Output()
+
+
+
+.. raw:: html
+
+    

+
+
+
+
+.. raw:: html
+
+    
+    

+
+
+
+Run INT8 model inference
+------------------------
+
+
+
+.. code:: ipython3
+
+    from IPython.display import display
+    
+    if OV_INT8_COLORIZER_PATH.exists():
+        compiled_int8_model = core.compile_model(OV_INT8_COLORIZER_PATH, device.value)
+        img = cv2.imread("DDColor/assets/test_images/Ansel Adams _ Moore Photography.jpeg")
+        img_out = process(img, compiled_int8_model)
+        display(PIL.Image.fromarray(img_out[:, :, ::-1]))
+
+
+
+.. image:: ddcolor-image-colorization-with-output_files/ddcolor-image-colorization-with-output_25_0.png
+
+
+Compare FP16 and INT8 model size
+--------------------------------
+
+
+
+.. code:: ipython3
+
+    fp16_ir_model_size = OV_COLORIZER_PATH.with_suffix(".bin").stat().st_size / 2**20
+    
+    print(f"FP16 model size: {fp16_ir_model_size:.2f} MB")
+    
+    if OV_INT8_COLORIZER_PATH.exists():
+        quantized_model_size = OV_INT8_COLORIZER_PATH.with_suffix(".bin").stat().st_size / 2**20
+        print(f"INT8 model size: {quantized_model_size:.2f} MB")
+        print(f"Model compression rate: {fp16_ir_model_size / quantized_model_size:.3f}")
+
+
+.. parsed-literal::
+
+    FP16 model size: 104.89 MB
+    INT8 model size: 52.97 MB
+    Model compression rate: 1.980
+
+
+Compare inference time of the FP16 and INT8 models
+--------------------------------------------------
+
+
+
+To measure the inference performance of OpenVINO FP16 and INT8 models,
+use `Benchmark
+Tool `__.
+
+   **NOTE**: For the most accurate performance estimation, it is
+   recommended to run ``benchmark_app`` in a terminal/command prompt
+   after closing other applications.
+
+.. code:: ipython3
+
+    !benchmark_app  -m $OV_COLORIZER_PATH -d $device.value -api async -shape "[1,3,512,512]" -t 15
+
+
+.. parsed-literal::
+
+    [Step 1/11] Parsing and validating input arguments
+    [ INFO ] Parsing input parameters
+    [Step 2/11] Loading OpenVINO Runtime
+    [ INFO ] OpenVINO:
+    [ INFO ] Build ................................. 2024.3.0-15743-15257f1bac1
+    [ INFO ] 
+    [ INFO ] Device info:
+    [ INFO ] AUTO
+    [ INFO ] Build ................................. 2024.3.0-15743-15257f1bac1
+    [ INFO ] 
+    [ INFO ] 
+    [Step 3/11] Setting device configuration
+    [ WARNING ] Performance hint was not explicitly specified in command line. Device(AUTO) performance hint will be set to PerformanceMode.THROUGHPUT.
+    [Step 4/11] Reading model files
+    [ INFO ] Loading model files
+    [ INFO ] Read model took 41.59 ms
+    [ INFO ] Original model I/O parameters:
+    [ INFO ] Model inputs:
+    [ INFO ]     x (node: x) : f32 / [...] / [1,3,512,512]
+    [ INFO ] Model outputs:
+    [ INFO ]     ***NO_NAME*** (node: __module.refine_net.0.0/aten::_convolution/Add) : f32 / [...] / [1,2,512,512]
+    [Step 5/11] Resizing model to match image sizes and given batch
+    [ INFO ] Model batch size: 1
+    [ INFO ] Reshaping model: 'x': [1,3,512,512]
+    [ INFO ] Reshape model took 0.04 ms
+    [Step 6/11] Configuring input of the model
+    [ INFO ] Model inputs:
+    [ INFO ]     x (node: x) : u8 / [N,C,H,W] / [1,3,512,512]
+    [ INFO ] Model outputs:
+    [ INFO ]     ***NO_NAME*** (node: __module.refine_net.0.0/aten::_convolution/Add) : f32 / [...] / [1,2,512,512]
+    [Step 7/11] Loading the model to the device
+    [ INFO ] Compile model took 1539.09 ms
+    [Step 8/11] Querying optimal runtime parameters
+    [ INFO ] Model:
+    [ INFO ]   NETWORK_NAME: Model0
+    [ INFO ]   EXECUTION_DEVICES: ['CPU']
+    [ INFO ]   PERFORMANCE_HINT: PerformanceMode.THROUGHPUT
+    [ INFO ]   OPTIMAL_NUMBER_OF_INFER_REQUESTS: 6
+    [ INFO ]   MULTI_DEVICE_PRIORITIES: CPU
+    [ INFO ]   CPU:
+    [ INFO ]     AFFINITY: Affinity.CORE
+    [ INFO ]     CPU_DENORMALS_OPTIMIZATION: False
+    [ INFO ]     CPU_SPARSE_WEIGHTS_DECOMPRESSION_RATE: 1.0
+    [ INFO ]     DYNAMIC_QUANTIZATION_GROUP_SIZE: 0
+    [ INFO ]     ENABLE_CPU_PINNING: True
+    [ INFO ]     ENABLE_HYPER_THREADING: True
+    [ INFO ]     EXECUTION_DEVICES: ['CPU']
+    [ INFO ]     EXECUTION_MODE_HINT: ExecutionMode.PERFORMANCE
+    [ INFO ]     INFERENCE_NUM_THREADS: 24
+    [ INFO ]     INFERENCE_PRECISION_HINT: 
+    [ INFO ]     KV_CACHE_PRECISION: 
+    [ INFO ]     LOG_LEVEL: Level.NO
+    [ INFO ]     MODEL_DISTRIBUTION_POLICY: set()
+    [ INFO ]     NETWORK_NAME: Model0
+    [ INFO ]     NUM_STREAMS: 6
+    [ INFO ]     OPTIMAL_NUMBER_OF_INFER_REQUESTS: 6
+    [ INFO ]     PERFORMANCE_HINT: THROUGHPUT
+    [ INFO ]     PERFORMANCE_HINT_NUM_REQUESTS: 0
+    [ INFO ]     PERF_COUNT: NO
+    [ INFO ]     SCHEDULING_CORE_TYPE: SchedulingCoreType.ANY_CORE
+    [ INFO ]   MODEL_PRIORITY: Priority.MEDIUM
+    [ INFO ]   LOADED_FROM_CACHE: False
+    [ INFO ]   PERF_COUNT: False
+    [Step 9/11] Creating infer requests and preparing input tensors
+    [ WARNING ] No input files were given for input 'x'!. This input will be filled with random values!
+    [ INFO ] Fill input 'x' with random values 
+    [Step 10/11] Measuring performance (Start inference asynchronously, 6 inference requests, limits: 15000 ms duration)
+    [ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop).
+    [ INFO ] First inference took 546.12 ms
+    [Step 11/11] Dumping statistics report
+    [ INFO ] Execution Devices:['CPU']
+    [ INFO ] Count:            72 iterations
+    [ INFO ] Duration:         16826.49 ms
+    [ INFO ] Latency:
+    [ INFO ]    Median:        1401.31 ms
+    [ INFO ]    Average:       1400.26 ms
+    [ INFO ]    Min:           1333.57 ms
+    [ INFO ]    Max:           1475.06 ms
+    [ INFO ] Throughput:   4.28 FPS
+
+
+.. code:: ipython3
+
+    if OV_INT8_COLORIZER_PATH.exists():
+        !benchmark_app  -m $OV_INT8_COLORIZER_PATH -d $device.value -api async -shape "[1,3,512,512]" -t 15
+
+
+.. parsed-literal::
+
+    [Step 1/11] Parsing and validating input arguments
+    [ INFO ] Parsing input parameters
+    [Step 2/11] Loading OpenVINO Runtime
+    [ INFO ] OpenVINO:
+    [ INFO ] Build ................................. 2024.3.0-15743-15257f1bac1
+    [ INFO ] 
+    [ INFO ] Device info:
+    [ INFO ] AUTO
+    [ INFO ] Build ................................. 2024.3.0-15743-15257f1bac1
+    [ INFO ] 
+    [ INFO ] 
+    [Step 3/11] Setting device configuration
+    [ WARNING ] Performance hint was not explicitly specified in command line. Device(AUTO) performance hint will be set to PerformanceMode.THROUGHPUT.
+    [Step 4/11] Reading model files
+    [ INFO ] Loading model files
+    [ INFO ] Read model took 67.10 ms
+    [ INFO ] Original model I/O parameters:
+    [ INFO ] Model inputs:
+    [ INFO ]     x (node: x) : f32 / [...] / [1,3,512,512]
+    [ INFO ] Model outputs:
+    [ INFO ]     ***NO_NAME*** (node: __module.refine_net.0.0/aten::_convolution/Add) : f32 / [...] / [1,2,512,512]
+    [Step 5/11] Resizing model to match image sizes and given batch
+    [ INFO ] Model batch size: 1
+    [ INFO ] Reshaping model: 'x': [1,3,512,512]
+    [ INFO ] Reshape model took 0.04 ms
+    [Step 6/11] Configuring input of the model
+    [ INFO ] Model inputs:
+    [ INFO ]     x (node: x) : u8 / [N,C,H,W] / [1,3,512,512]
+    [ INFO ] Model outputs:
+    [ INFO ]     ***NO_NAME*** (node: __module.refine_net.0.0/aten::_convolution/Add) : f32 / [...] / [1,2,512,512]
+    [Step 7/11] Loading the model to the device
+    [ INFO ] Compile model took 2751.87 ms
+    [Step 8/11] Querying optimal runtime parameters
+    [ INFO ] Model:
+    [ INFO ]   NETWORK_NAME: Model0
+    [ INFO ]   EXECUTION_DEVICES: ['CPU']
+    [ INFO ]   PERFORMANCE_HINT: PerformanceMode.THROUGHPUT
+    [ INFO ]   OPTIMAL_NUMBER_OF_INFER_REQUESTS: 6
+    [ INFO ]   MULTI_DEVICE_PRIORITIES: CPU
+    [ INFO ]   CPU:
+    [ INFO ]     AFFINITY: Affinity.CORE
+    [ INFO ]     CPU_DENORMALS_OPTIMIZATION: False
+    [ INFO ]     CPU_SPARSE_WEIGHTS_DECOMPRESSION_RATE: 1.0
+    [ INFO ]     DYNAMIC_QUANTIZATION_GROUP_SIZE: 0
+    [ INFO ]     ENABLE_CPU_PINNING: True
+    [ INFO ]     ENABLE_HYPER_THREADING: True
+    [ INFO ]     EXECUTION_DEVICES: ['CPU']
+    [ INFO ]     EXECUTION_MODE_HINT: ExecutionMode.PERFORMANCE
+    [ INFO ]     INFERENCE_NUM_THREADS: 24
+    [ INFO ]     INFERENCE_PRECISION_HINT: 
+    [ INFO ]     KV_CACHE_PRECISION: 
+    [ INFO ]     LOG_LEVEL: Level.NO
+    [ INFO ]     MODEL_DISTRIBUTION_POLICY: set()
+    [ INFO ]     NETWORK_NAME: Model0
+    [ INFO ]     NUM_STREAMS: 6
+    [ INFO ]     OPTIMAL_NUMBER_OF_INFER_REQUESTS: 6
+    [ INFO ]     PERFORMANCE_HINT: THROUGHPUT
+    [ INFO ]     PERFORMANCE_HINT_NUM_REQUESTS: 0
+    [ INFO ]     PERF_COUNT: NO
+    [ INFO ]     SCHEDULING_CORE_TYPE: SchedulingCoreType.ANY_CORE
+    [ INFO ]   MODEL_PRIORITY: Priority.MEDIUM
+    [ INFO ]   LOADED_FROM_CACHE: False
+    [ INFO ]   PERF_COUNT: False
+    [Step 9/11] Creating infer requests and preparing input tensors
+    [ WARNING ] No input files were given for input 'x'!. This input will be filled with random values!
+    [ INFO ] Fill input 'x' with random values 
+    [Step 10/11] Measuring performance (Start inference asynchronously, 6 inference requests, limits: 15000 ms duration)
+    [ INFO ] Benchmarking in inference only mode (inputs filling are not included in measurement loop).
+    [ INFO ] First inference took 280.80 ms
+    [Step 11/11] Dumping statistics report
+    [ INFO ] Execution Devices:['CPU']
+    [ INFO ] Count:            156 iterations
+    [ INFO ] Duration:         15712.29 ms
+    [ INFO ] Latency:
+    [ INFO ]    Median:        596.19 ms
+    [ INFO ]    Average:       603.03 ms
+    [ INFO ]    Min:           567.92 ms
+    [ INFO ]    Max:           680.14 ms
+    [ INFO ] Throughput:   9.93 FPS
+
+
+Interactive inference
+---------------------
+
+
+
+.. code:: ipython3
+
+    import gradio as gr
+    from gradio_imageslider import ImageSlider
+    from functools import partial
+    
+    
+    def generate(image, use_int8=True):
+        image_in = cv2.imread(image)
+        image_out = process(image_in, compiled_model if not use_int8 else compiled_int8_model)
+        image_in_pil = PIL.Image.fromarray(cv2.cvtColor(image_in, cv2.COLOR_BGR2RGB))
+        image_out_pil = PIL.Image.fromarray(cv2.cvtColor(image_out, cv2.COLOR_BGR2RGB))
+        return (image_in_pil, image_out_pil)
+    
+    
+    with gr.Blocks() as demo:
+        with gr.Row(equal_height=False):
+            image = gr.Image(type="filepath")
+            with gr.Column():
+                output_image = ImageSlider(show_label=True, type="filepath", interactive=False, label="FP16 model output")
+                button = gr.Button(value="Run{}".format(" FP16 model" if compiled_int8_model is not None else ""))
+            with gr.Column(visible=compiled_int8_model is not None):
+                output_image_int8 = ImageSlider(show_label=True, type="filepath", interactive=False, label="INT8 model output")
+                button_i8 = gr.Button(value="Run INT8 model")
+        button.click(fn=partial(generate, use_int8=False), inputs=[image], outputs=[output_image])
+        button_i8.click(fn=partial(generate, use_int8=True), inputs=[image], outputs=[output_image_int8])
+        examples = gr.Examples(
+            [
+                "DDColor/assets/test_images/New York Riverfront December 15, 1931.jpg",
+                "DDColor/assets/test_images/Audrey Hepburn.jpg",
+                "DDColor/assets/test_images/Acrobats Balance On Top Of The Empire State Building, 1934.jpg",
+            ],
+            inputs=[image],
+        )
+    
+    
+    if __name__ == "__main__":
+        try:
+            demo.queue().launch(debug=False)
+        except Exception:
+            demo.queue().launch(share=True, debug=False)
+    # if you are launching remotely, specify server_name and server_port
+    # demo.launch(server_name='your server name', server_port='server port in int')
+    # Read more in the docs: https://gradio.app/docs/
+
+
+.. parsed-literal::
+
+    Running on local URL:  http://127.0.0.1:7860
+    
+    To create a public link, set `share=True` in `launch()`.
+
+
+
+
+
+
+
diff --git a/docs/notebooks/ddcolor-image-colorization-with-output_files/ddcolor-image-colorization-with-output_16_0.jpg b/docs/notebooks/ddcolor-image-colorization-with-output_files/ddcolor-image-colorization-with-output_16_0.jpg
new file mode 100644
index 00000000000000..4941cb1f61198d
--- /dev/null
+++ b/docs/notebooks/ddcolor-image-colorization-with-output_files/ddcolor-image-colorization-with-output_16_0.jpg
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e152195f834e9ac26e7df50e12c80cf13ee8d747e62dd29edf85b6fabb98cf84
+size 164898
diff --git a/docs/notebooks/ddcolor-image-colorization-with-output_files/ddcolor-image-colorization-with-output_16_0.png b/docs/notebooks/ddcolor-image-colorization-with-output_files/ddcolor-image-colorization-with-output_16_0.png
new file mode 100644
index 00000000000000..df05154ccae067
--- /dev/null
+++ b/docs/notebooks/ddcolor-image-colorization-with-output_files/ddcolor-image-colorization-with-output_16_0.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f5d2b934619e33836937b6f9fe32987a37ca7506ecaf7d044d8286b6212bebe5
+size 1497558
diff --git a/docs/notebooks/ddcolor-image-colorization-with-output_files/ddcolor-image-colorization-with-output_25_0.jpg b/docs/notebooks/ddcolor-image-colorization-with-output_files/ddcolor-image-colorization-with-output_25_0.jpg
new file mode 100644
index 00000000000000..ec566f6d4a8eab
--- /dev/null
+++ b/docs/notebooks/ddcolor-image-colorization-with-output_files/ddcolor-image-colorization-with-output_25_0.jpg
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:734019b3e2c4db0cc8cab97a3790860e72e3e9b107c498727819a4432ac215df
+size 167190
diff --git a/docs/notebooks/ddcolor-image-colorization-with-output_files/ddcolor-image-colorization-with-output_25_0.png b/docs/notebooks/ddcolor-image-colorization-with-output_files/ddcolor-image-colorization-with-output_25_0.png
new file mode 100644
index 00000000000000..ca800b3d2b30bd
--- /dev/null
+++ b/docs/notebooks/ddcolor-image-colorization-with-output_files/ddcolor-image-colorization-with-output_25_0.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bd8956b996d25efec8ca81a9cea60e276961ec963497ae48168373629a54d840
+size 1522254
diff --git a/docs/notebooks/ddcolor-image-colorization-with-output_files/ddcolor-image-colorization-with-output_8_0.jpg b/docs/notebooks/ddcolor-image-colorization-with-output_files/ddcolor-image-colorization-with-output_8_0.jpg
new file mode 100644
index 00000000000000..8fb53b4e8d0997
--- /dev/null
+++ b/docs/notebooks/ddcolor-image-colorization-with-output_files/ddcolor-image-colorization-with-output_8_0.jpg
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:614b930200ec51d4631b7c2a3cc743e18a8f3f94c3ea47673ef1fca37028af31
+size 152058
diff --git a/docs/notebooks/ddcolor-image-colorization-with-output_files/ddcolor-image-colorization-with-output_8_0.png b/docs/notebooks/ddcolor-image-colorization-with-output_files/ddcolor-image-colorization-with-output_8_0.png
new file mode 100644
index 00000000000000..d28e544903e803
--- /dev/null
+++ b/docs/notebooks/ddcolor-image-colorization-with-output_files/ddcolor-image-colorization-with-output_8_0.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0c105b02bbf0b3c82c253b6b4e92d7b28518775279197cbf239ca40dd52c9e9e
+size 792834
diff --git a/docs/notebooks/ddcolor-image-colorization-with-output_files/ddcolor-image-colorization-with-output_9_0.jpg b/docs/notebooks/ddcolor-image-colorization-with-output_files/ddcolor-image-colorization-with-output_9_0.jpg
new file mode 100644
index 00000000000000..77e9e43ffd6303
--- /dev/null
+++ b/docs/notebooks/ddcolor-image-colorization-with-output_files/ddcolor-image-colorization-with-output_9_0.jpg
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a92ff02af12364a2b15169f96657de93c6a172e20174f7b8a857578d58405f9
+size 164917
diff --git a/docs/notebooks/ddcolor-image-colorization-with-output_files/ddcolor-image-colorization-with-output_9_0.png b/docs/notebooks/ddcolor-image-colorization-with-output_files/ddcolor-image-colorization-with-output_9_0.png
new file mode 100644
index 00000000000000..5e84b839762374
--- /dev/null
+++ b/docs/notebooks/ddcolor-image-colorization-with-output_files/ddcolor-image-colorization-with-output_9_0.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:82894174823bc19a70eb250cf7238d542f2d81127e6bd3760d210c94796f2968
+size 1497540
diff --git a/docs/notebooks/decidiffusion-image-generation-with-output.rst b/docs/notebooks/decidiffusion-image-generation-with-output.rst
index 9e878c71157966..6273895cf3cb73 100644
--- a/docs/notebooks/decidiffusion-image-generation-with-output.rst
+++ b/docs/notebooks/decidiffusion-image-generation-with-output.rst
@@ -89,7 +89,7 @@ install required packages
 
 .. code:: ipython3
 
-    %pip install -q --extra-index-url https://download.pytorch.org/whl/cpu  "diffusers" "transformers" "torch>=2.1" "pillow" "openvino>=2023.1.0" "gradio>=4.19" "datasets>=2.14.6" "huggingface-hub>=0.19.4" "nncf>=2.7.0" "peft==0.6.2" "opencv-python"
+    %pip install -q --extra-index-url https://download.pytorch.org/whl/cpu  "diffusers<0.29.0" "transformers" "torch>=2.1" "pillow" "openvino>=2023.1.0" "gradio>=4.19" "datasets>=2.14.6" "huggingface-hub>=0.19.4" "nncf>=2.7.0" "peft==0.6.2" "opencv-python"
 
 Prepare DeciDiffusion models for OpenVINO format conversion
 -----------------------------------------------------------
@@ -1174,12 +1174,8 @@ improve model inference speed.
 
 .. code:: ipython3
 
-    to_quantize = widgets.Checkbox(
-        value=True,
-        description="Quantization",
-        disabled=False,
-    )
-    
+    skip_for_device = "GPU" in device.value
+    to_quantize = widgets.Checkbox(value=not skip_for_device, description="Quantization", disabled=skip_for_device)
     to_quantize
 
 
@@ -1214,7 +1210,7 @@ Prepare calibration dataset
 
 
 We use a portion of
-`conceptual_captions `__
+`conceptual_captions `__
 dataset from Hugging Face as calibration data. To collect intermediate
 model inputs for calibration we should customize ``CompiledModel``.
 
@@ -1249,7 +1245,7 @@ model inputs for calibration we should customize ``CompiledModel``.
         pipeline.unet = CompiledModelDecorator(original_unet, prob=0.3)
         pipeline.set_progress_bar_config(disable=True)
     
-        dataset = datasets.load_dataset("conceptual_captions", split="train", streaming=True).shuffle(seed=42)
+        dataset = datasets.load_dataset("google-research-datasets/conceptual_captions", split="train", streaming=True, trust_remote_code=True).shuffle(seed=42)
     
         pbar = tqdm(total=subset_size)
         for batch in dataset:
@@ -1444,7 +1440,7 @@ pipelines, we use median inference time on calibration subset.
     import time
     
     validation_size = 10
-    calibration_dataset = datasets.load_dataset("conceptual_captions", split="train", streaming=True)
+    calibration_dataset = datasets.load_dataset("conceptual_captions", split="train", streaming=True, trust_remote_code=True)
     validation_data = []
     for idx, batch in enumerate(calibration_dataset):
         if idx >= validation_size:
diff --git a/docs/notebooks/depth-anything-with-output.rst b/docs/notebooks/depth-anything-with-output.rst
index d583ea6772020c..dcb43a18ff863d 100644
--- a/docs/notebooks/depth-anything-with-output.rst
+++ b/docs/notebooks/depth-anything-with-output.rst
@@ -75,13 +75,13 @@ Prerequisites
 .. parsed-literal::
 
     Cloning into 'Depth-Anything'...
-    remote: Enumerating objects: 421, done.
-    remote: Counting objects: 100% (144/144), done.
-    remote: Compressing objects: 100% (105/105), done.
-    remote: Total 421 (delta 101), reused 43 (delta 39), pack-reused 277
-    Receiving objects: 100% (421/421), 237.89 MiB | 27.94 MiB/s, done.
-    Resolving deltas: 100% (144/144), done.
-    /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/notebooks/depth-anything/Depth-Anything
+    remote: Enumerating objects: 430, done.
+    remote: Counting objects: 100% (153/153), done.
+    remote: Compressing objects: 100% (112/112), done.
+    remote: Total 430 (delta 107), reused 47 (delta 41), pack-reused 277
+    Receiving objects: 100% (430/430), 237.89 MiB | 23.98 MiB/s, done.
+    Resolving deltas: 100% (150/150), done.
+    /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-708/.workspace/scm/ov-notebook/notebooks/depth-anything/Depth-Anything
     Note: you may need to restart the kernel to use updated packages.
     Note: you may need to restart the kernel to use updated packages.
     WARNING: typer 0.12.3 does not provide the extra 'all'
@@ -273,13 +273,13 @@ loading on device using ``core.complie_model``.
 
 .. parsed-literal::
 
-    /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/notebooks/depth-anything/Depth-Anything/torchhub/facebookresearch_dinov2_main/dinov2/layers/patch_embed.py:73: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!
+    /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-708/.workspace/scm/ov-notebook/notebooks/depth-anything/Depth-Anything/torchhub/facebookresearch_dinov2_main/dinov2/layers/patch_embed.py:73: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!
       assert H % patch_H == 0, f"Input image height {H} is not a multiple of patch height {patch_H}"
-    /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/notebooks/depth-anything/Depth-Anything/torchhub/facebookresearch_dinov2_main/dinov2/layers/patch_embed.py:74: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!
+    /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-708/.workspace/scm/ov-notebook/notebooks/depth-anything/Depth-Anything/torchhub/facebookresearch_dinov2_main/dinov2/layers/patch_embed.py:74: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!
       assert W % patch_W == 0, f"Input image width {W} is not a multiple of patch width: {patch_W}"
-    /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/notebooks/depth-anything/Depth-Anything/torchhub/facebookresearch_dinov2_main/vision_transformer.py:183: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!
+    /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-708/.workspace/scm/ov-notebook/notebooks/depth-anything/Depth-Anything/torchhub/facebookresearch_dinov2_main/vision_transformer.py:183: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!
       if npatch == N and w == h:
-    /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/notebooks/depth-anything/Depth-Anything/depth_anything/dpt.py:133: TracerWarning: Converting a tensor to a Python integer might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!
+    /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-708/.workspace/scm/ov-notebook/notebooks/depth-anything/Depth-Anything/depth_anything/dpt.py:133: TracerWarning: Converting a tensor to a Python integer might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!
       out = F.interpolate(out, (int(patch_h * 14), int(patch_w * 14)), mode="bilinear", align_corners=True)
 
 
@@ -571,7 +571,7 @@ Run inference on video
 
 .. parsed-literal::
 
-    Processed 60 frames in 13.28 seconds. Total FPS (including video processing): 4.52.Inference FPS: 10.54 
+    Processed 60 frames in 13.15 seconds. Total FPS (including video processing): 4.56.Inference FPS: 10.55 
     Video saved to 'output/Coco Walking in Berkeley_depth_anything.mp4'.
 
 
@@ -598,7 +598,7 @@ Run inference on video
 .. parsed-literal::
 
     Showing video saved at
-    /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/notebooks/depth-anything/Depth-Anything/output/Coco Walking in Berkeley_depth_anything.mp4
+    /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-708/.workspace/scm/ov-notebook/notebooks/depth-anything/Depth-Anything/output/Coco Walking in Berkeley_depth_anything.mp4
     If you cannot see the video in your browser, please click on the following link to download the video 
 
 
@@ -612,7 +612,7 @@ Run inference on video
 .. raw:: html
 
     
 
@@ -735,10 +735,10 @@ quantization code below may take some time.
 
 .. parsed-literal::
 
-    2024-05-15 23:54:08.057348: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.
-    2024-05-15 23:54:08.091208: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
+    2024-06-19 23:17:58.857728: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.
+    2024-06-19 23:17:58.890483: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
     To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.
-    2024-05-15 23:54:08.654880: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT
+    2024-06-19 23:17:59.450018: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT
 
 
 
@@ -902,10 +902,10 @@ data.
 
 .. parsed-literal::
 
-    Processed 60 frames in 12.70 seconds. Total FPS (including video processing): 4.73.Inference FPS: 12.89 
+    Processed 60 frames in 12.57 seconds. Total FPS (including video processing): 4.77.Inference FPS: 12.87 
     Video saved to 'output/Coco Walking in Berkeley_depth_anything_int8.mp4'.
     Showing video saved at
-    /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-681/.workspace/scm/ov-notebook/notebooks/depth-anything/Depth-Anything/output/Coco Walking in Berkeley_depth_anything.mp4
+    /opt/home/k8sworker/ci-ai/cibuilds/ov-notebook/OVNotebookOps-708/.workspace/scm/ov-notebook/notebooks/depth-anything/Depth-Anything/output/Coco Walking in Berkeley_depth_anything.mp4
     If you cannot see the video in your browser, please click on the following link to download the video 
 
 
@@ -919,7 +919,7 @@ data.
 .. raw:: html
 
     
 
@@ -985,9 +985,9 @@ Tool =2023.1.0"
 
diff --git a/docs/notebooks/detectron2-to-openvino-with-output_files/detectron2-to-openvino-with-output_22_0.jpg b/docs/notebooks/detectron2-to-openvino-with-output_files/detectron2-to-openvino-with-output_22_0.jpg
index f80eeeb0a2bd87..19419d46d85eb3 100644
--- a/docs/notebooks/detectron2-to-openvino-with-output_files/detectron2-to-openvino-with-output_22_0.jpg
+++ b/docs/notebooks/detectron2-to-openvino-with-output_files/detectron2-to-openvino-with-output_22_0.jpg
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:402297f642d1ae24c5f34c947b243308e80a2353a297ff4fbc29a80a91447f3d
-size 57749
+oid sha256:7886faf5ed54b2fb91c911f1716f9875e62f036803ec85448f93b2f9b02bd0d3
+size 59068
diff --git a/docs/notebooks/detectron2-to-openvino-with-output_files/detectron2-to-openvino-with-output_22_0.png b/docs/notebooks/detectron2-to-openvino-with-output_files/detectron2-to-openvino-with-output_22_0.png
index 914c56acc0b474..4e1de70f09db69 100644
--- a/docs/notebooks/detectron2-to-openvino-with-output_files/detectron2-to-openvino-with-output_22_0.png
+++ b/docs/notebooks/detectron2-to-openvino-with-output_files/detectron2-to-openvino-with-output_22_0.png
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bfdce03dd80512c030e07a93860311e984178cc6c00aea85904d97a9d85fd91b
-size 507627
+oid sha256:5639c9302d12180ff79fa18c5a5e31784bb76a1d94aa3553f19109c1360440f7
+size 509180
diff --git a/docs/notebooks/detectron2-to-openvino-with-output_files/detectron2-to-openvino-with-output_32_0.jpg b/docs/notebooks/detectron2-to-openvino-with-output_files/detectron2-to-openvino-with-output_32_0.jpg
index de3cdef72f0010..827c03a14bc5cf 100644
--- a/docs/notebooks/detectron2-to-openvino-with-output_files/detectron2-to-openvino-with-output_32_0.jpg
+++ b/docs/notebooks/detectron2-to-openvino-with-output_files/detectron2-to-openvino-with-output_32_0.jpg
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d467c3878c59234b1e763ec286626e3036584d037e32857396ec873b73347d3c
-size 53573
+oid sha256:1fb2a4369b870763dd0900d35e91c7ccae5dfe1cbddd677fc35a6734a32f2819
+size 56638
diff --git a/docs/notebooks/detectron2-to-openvino-with-output_files/detectron2-to-openvino-with-output_32_0.png b/docs/notebooks/detectron2-to-openvino-with-output_files/detectron2-to-openvino-with-output_32_0.png
index 2d5cc464a97747..051773914b73d4 100644
--- a/docs/notebooks/detectron2-to-openvino-with-output_files/detectron2-to-openvino-with-output_32_0.png
+++ b/docs/notebooks/detectron2-to-openvino-with-output_files/detectron2-to-openvino-with-output_32_0.png
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ef9d6efc8183a218693431a458cd84831e4820d02518cfd9d67f6c532ffc6f53
-size 456551
+oid sha256:15df1737526ddf424974c8e2d53766ad02be4115d44269f53c619164d0db02ac
+size 456714
diff --git a/docs/notebooks/distil-whisper-asr-with-output.rst b/docs/notebooks/distil-whisper-asr-with-output.rst
index d1090b4abbb780..a66c343925f252 100644
--- a/docs/notebooks/distil-whisper-asr-with-output.rst
+++ b/docs/notebooks/distil-whisper-asr-with-output.rst
@@ -105,7 +105,7 @@ using tokenizer.
 .. code:: ipython3
 
     import ipywidgets as widgets
-
+    
     model_ids = {
         "Distil-Whisper": [
             "distil-whisper/distil-large-v2",
@@ -126,14 +126,14 @@ using tokenizer.
             "openai/whisper-tiny.en",
         ],
     }
-
+    
     model_type = widgets.Dropdown(
         options=model_ids.keys(),
         value="Distil-Whisper",
         description="Model type:",
         disabled=False,
     )
-
+    
     model_type
 
 .. code:: ipython3
@@ -144,15 +144,15 @@ using tokenizer.
         description="Model:",
         disabled=False,
     )
-
+    
     model_id
 
 .. code:: ipython3
 
     from transformers import AutoProcessor, AutoModelForSpeechSeq2Seq
-
+    
     processor = AutoProcessor.from_pretrained(model_id.value)
-
+    
     pt_model = AutoModelForSpeechSeq2Seq.from_pretrained(model_id.value)
     pt_model.eval();
 
@@ -175,8 +175,8 @@ by Hugging Face datasets implementation.
 .. code:: ipython3
 
     from datasets import load_dataset
-
-
+    
+    
     def extract_input_features(sample):
         input_features = processor(
             sample["audio"]["array"],
@@ -184,9 +184,9 @@ by Hugging Face datasets implementation.
             return_tensors="pt",
         ).input_features
         return input_features
-
-
-    dataset = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
+    
+    
+    dataset = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation", trust_remote_code=True)
     sample = dataset[0]
     input_features = extract_input_features(sample)
 
@@ -202,10 +202,10 @@ for decoding predicted token_ids into text transcription.
 .. code:: ipython3
 
     import IPython.display as ipd
-
+    
     predicted_ids = pt_model.generate(input_features)
     transcription = processor.batch_decode(predicted_ids, skip_special_tokens=True)
-
+    
     display(ipd.Audio(sample["audio"]["array"], rate=sample["audio"]["sampling_rate"]))
     print(f"Reference: {sample['text']}")
     print(f"Result: {transcription[0]}")
@@ -214,7 +214,7 @@ for decoding predicted token_ids into text transcription.
 
 .. raw:: html
 
-
+