diff --git a/.azure-pipelines/linux-conda-CI.yml b/.azure-pipelines/linux-conda-CI.yml deleted file mode 100644 index 0d80114f..00000000 --- a/.azure-pipelines/linux-conda-CI.yml +++ /dev/null @@ -1,213 +0,0 @@ -# Python package -# Create and test a Python package on multiple Python versions. -# Add steps that analyze code, save the dist with the build record, publish to a PyPI-compatible index, and more: -# https://docs.microsoft.com/azure/devops/pipelines/languages/python - -trigger: -- master - -jobs: - -- job: 'Test' - timeoutInMinutes: 30 - pool: - vmImage: 'ubuntu-latest' - strategy: - matrix: - - Python312-1160-RT1180-xgb2-lgbm40: - python.version: '3.12' - ONNX_PATH: 'onnx==1.16.0' - ONNXRT_PATH: 'onnxruntime==1.18.0' - COREML_PATH: NONE - lightgbm.version: '>=4.0' - xgboost.version: '>=2' - numpy.version: '' - scipy.version: '' - - Python311-1160-RT1171-xgb2-lgbm40: - python.version: '3.11' - ONNX_PATH: 'onnx==1.16.0' - ONNXRT_PATH: 'onnxruntime==1.17.1' - COREML_PATH: NONE - lightgbm.version: '>=4.0' - xgboost.version: '>=2' - numpy.version: '' - scipy.version: '' - - Python311-1150-RT1163-xgb2-lgbm40: - python.version: '3.11' - ONNX_PATH: 'onnx==1.15.0' - ONNXRT_PATH: 'onnxruntime==1.16.3' - COREML_PATH: NONE - lightgbm.version: '>=4.0' - xgboost.version: '>=2' - numpy.version: '' - scipy.version: '' - - Python311-1150-RT1163-xgb175-lgbm40: - python.version: '3.11' - ONNX_PATH: 'onnx==1.15.0' - ONNXRT_PATH: 'onnxruntime==1.16.3' - COREML_PATH: NONE - lightgbm.version: '>=4.0' - xgboost.version: '>=1.7.5,<2' - numpy.version: '' - scipy.version: '' - - Python311-1141-RT1162-xgb175-lgbm40: - python.version: '3.11' - ONNX_PATH: 'onnx==1.14.1' - ONNXRT_PATH: 'onnxruntime==1.16.2' - COREML_PATH: NONE - lightgbm.version: '>=4.0' - xgboost.version: '>=1.7.5,<2' - numpy.version: '' - scipy.version: '' - - Python310-1141-RT1151-xgb175: - python.version: '3.10' - ONNX_PATH: 'onnx==1.14.1' - ONNXRT_PATH: 'onnxruntime==1.15.1' - COREML_PATH: NONE - lightgbm.version: '<4.0' - xgboost.version: '>=1.7.5,<2' - numpy.version: '' - scipy.version: '' - - Python310-1141-RT1140-xgb175: - python.version: '3.10' - ONNX_PATH: 'onnx==1.14.1' - ONNXRT_PATH: 'onnxruntime==1.14.0' - COREML_PATH: NONE - lightgbm.version: '<4.0' - xgboost.version: '>=1.7.5,<2' - numpy.version: '' - scipy.version: '' - - Python39-1141-RT1151-xgb175-scipy180: - python.version: '3.9' - ONNX_PATH: 'onnx==1.14.1' - ONNXRT_PATH: 'onnxruntime==1.15.1' - COREML_PATH: NONE - lightgbm.version: '>=4.0' - xgboost.version: '>=1.7.5,<2' - numpy.version: '' - scipy.version: '==1.8.0' - - - maxParallel: 3 - - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '$(python.version)' - architecture: 'x64' - - - script: | - python -m pip install --upgrade pip - pip install $(ONNX_PATH) $(ONNXRT_PATH) cython - pip install -r requirements.txt - displayName: 'Install dependencies' - - - script: | - pip install -r requirements-dev.txt - displayName: 'Install dependencies-dev' - - - script: | - python -m pip install --upgrade pip - pip install "xgboost$(xgboost.version)" - pip install "lightgbm$(lightgbm.version)" - pip install $(ONNX_PATH) - pip install $(ONNXRT_PATH) - pip install "numpy$(numpy.version)" - pip install "scipy$(scipy.version)" - displayName: 'Install xgboost, onnxruntime' - - - script: | - python -m pip install coloredlogs flatbuffers packaging sympy numpy protobuf - python -m pip install $(ONNXRT_PATH) - displayName: 'Install onnxruntime' - - - script: | - pip install -e . - displayName: 'local installation' - - - script: | - export PYTHONPATH=. - python -c "import onnxruntime;print('onnx:',onnx.__version__)" - python -c "import onnxconverter_common;print('cc:',onnxconverter_common.__version__)" - python -c "import onnx;print('onnx:',onnx.__version__)" - python -c "import onnxruntime;print('ort:',onnxruntime.__version__)" - python -c "import xgboost;print('xgboost:',xgboost.__version__)" - python -c "import lightgbm;print('lightgbm:',lightgbm.__version__)" - displayName: 'version' - - - script: | - export PYTHONPATH=. - pytest tests/baseline --durations=0 - displayName: 'pytest - baseline' - - - script: | - export PYTHONPATH=. - pytest tests/catboost --durations=0 - displayName: 'pytest - catboost' - - - script: | - export PYTHONPATH=. - pytest tests/lightgbm --durations=0 - displayName: 'pytest - lightgbm' - - - script: | - export PYTHONPATH=. - pytest tests/sparkml --durations=0 - displayName: 'pytest - sparkml' - - - script: | - export PYTHONPATH=. - pytest tests/utils --durations=0 - displayName: 'pytest - utils' - - - script: | - export PYTHONPATH=. - pytest tests/xgboost --durations=0 - displayName: 'pytest - xgboost' - - - script: | - export PYTHONPATH=. - pip install h2o - pytest tests/h2o --durations=0 - displayName: 'pytest - h2o' - - - script: | - export PYTHONPATH=. - pytest tests/svmlib --durations=0 - displayName: 'pytest - svmlib' - - - script: | - pip install torch --extra-index-url https://download.pytorch.org/whl/cpu - pip install hummingbird-ml --no-deps - displayName: 'Install hummingbird-ml' - - - script: | - export PYTHONPATH=. - pytest tests/hummingbirdml --durations=0 - displayName: 'pytest - hummingbirdml' - - - script: | - if [ '$(COREML_PATH)' == 'NONE' ] - then - echo "required version of coremltools does not work on python 3.10" - else - export PYTHONPATH=. - pip install $(COREML_PATH) - pytest tests/coreml --durations=0 - fi - displayName: 'pytest - coreml [$(COREML_PATH)]' - # condition: ne('$(COREML_PATH)', 'NONE') - - - task: PublishTestResults@2 - inputs: - testResultsFiles: '**/test-results.xml' - testRunTitle: 'Python $(python.version)' - condition: succeededOrFailed() diff --git a/.azure-pipelines/win32-conda-CI.yml b/.azure-pipelines/win32-conda-CI.yml deleted file mode 100644 index 7304d225..00000000 --- a/.azure-pipelines/win32-conda-CI.yml +++ /dev/null @@ -1,200 +0,0 @@ -# Python package -# Create and test a Python package on multiple Python versions. -# Add steps that analyze code, save the dist with the build record, publish to a PyPI-compatible index, and more: -# https://docs.microsoft.com/azure/devops/pipelines/languages/python - -trigger: -- master - -jobs: - -- job: 'Test' - timeoutInMinutes: 30 - pool: - vmImage: 'windows-latest' - strategy: - matrix: - - Python312-1160-RT1180: - python.version: '3.11' - ONNX_PATH: 'onnx==1.16.0' - ONNXRT_PATH: 'onnxruntime==1.18.0' - COREML_PATH: NONE - numpy.version: '' - xgboost.version: '>=2.0' - - Python311-1160-RT1171: - python.version: '3.11' - ONNX_PATH: 'onnx==1.16.0' - ONNXRT_PATH: 'onnxruntime==1.17.1' - COREML_PATH: NONE - numpy.version: '' - xgboost.version: '>=2.0' - - Python311-1150-RT1163: - python.version: '3.11' - ONNX_PATH: 'onnx==1.15.0' - ONNXRT_PATH: 'onnxruntime==1.16.3' - COREML_PATH: NONE - numpy.version: '' - xgboost.version: '==2.0.2' - - Python311-1150-RT1162: - python.version: '3.11' - ONNX_PATH: 'onnx==1.15.0' - ONNXRT_PATH: 'onnxruntime==1.16.2' - COREML_PATH: NONE - numpy.version: '' - xgboost.version: '==2.0.2' - - Python311-1141-RT1162: - python.version: '3.11' - ONNX_PATH: 'onnx==1.14.1' - ONNXRT_PATH: 'onnxruntime==1.16.2' - COREML_PATH: NONE - numpy.version: '' - xgboost.version: '==1.7.5' - - Python310-1141-RT1151: - python.version: '3.10' - ONNX_PATH: 'onnx==1.14.1' - ONNXRT_PATH: 'onnxruntime==1.15.1' - COREML_PATH: NONE - numpy.version: '' - xgboost.version: '==1.7.5' - - Python310-1141-RT1140: - python.version: '3.10' - ONNX_PATH: 'onnx==1.14.1' - ONNXRT_PATH: onnxruntime==1.14.0 - COREML_PATH: NONE - numpy.version: '' - xgboost.version: '==1.7.5' - - Python39-1141-RT1140: - python.version: '3.9' - ONNX_PATH: 'onnx==1.14.1' - ONNXRT_PATH: onnxruntime==1.14.0 - COREML_PATH: NONE - numpy.version: '' - xgboost.version: '==1.7.5' - - maxParallel: 3 - - steps: - - powershell: Write-Host "##vso[task.prependpath]$env:CONDA\Scripts" - displayName: Add conda to PATH - - - script: conda create --yes --quiet --name py$(python.version) -c conda-forge python=$(python.version) numpy protobuf scikit-learn scipy cython - displayName: Create Anaconda environment - - - script: | - call activate py$(python.version) - python -m pip install --upgrade pip numpy - echo Test numpy installation... && python -c "import numpy" - python -m pip install -r requirements.txt - displayName: 'Install dependencies (1)' - - - script: | - call activate py$(python.version) - python -m pip install -r requirements-dev.txt - displayName: 'Install dependencies-dev' - - - script: | - call activate py$(python.version) - python -m pip install --upgrade scikit-learn - python -m pip install --upgrade lightgbm - python -m pip install "xgboost$(xgboost.version)" - displayName: 'Install scikit-learn' - - - script: | - call activate py$(python.version) - python -m pip install %ONNX_PATH% - python -m pip install %ONNXRT_PATH% - python -m pip install "numpy$(numpy.version)" - displayName: 'Install onnxruntime' - - - script: | - call activate py$(python.version) - python -m pip install coloredlogs flatbuffers packaging sympy numpy protobuf - python -m pip install $(ONNXRT_PATH) - displayName: 'Install ort-nightly' - - - script: | - call activate py$(python.version) - python -m pip install -e . - export PYTHONPATH=. - python -c "import onnxconverter_common;print(onnxconverter_common.__version__)" - python -c "import onnx;print(onnx.__version__)" - python -c "import onnxruntime;print(onnxruntime.__version__)" - python -c "import xgboost;print(xgboost.__version__)" - python -c "import lightgbm;print(lightgbm.__version__)" - displayName: 'version' - - - script: | - call activate py$(python.version) - export PYTHONPATH=. - python -m pytest tests/baseline --durations=0 - displayName: 'pytest baseline' - - - script: | - call activate py$(python.version) - export PYTHONPATH=. - python -m pytest tests/catboost --durations=0 - displayName: 'pytest catboost' - - - script: | - call activate py$(python.version) - set PYTHONPATH=. - if "$(COREML_PATH)" neq "NONE" python -m pip install %COREML_PATH% - if "$(COREML_PATH)" neq "NONE" python -m pytest tests/coreml --durations=0 - displayName: 'pytest coreml - [$(COREML_PATH)]' - #condition: ne('$(COREML_PATH)', 'NONE') - - - script: | - call activate py$(python.version) - export PYTHONPATH=. - python -m pytest tests/lightgbm --durations=0 - displayName: 'pytest lightgbm' - - - script: | - call activate py$(python.version) - export PYTHONPATH=. - python -m pytest tests/sparkml --durations=0 - displayName: 'pytest sparkml' - - - script: | - call activate py$(python.version) - export PYTHONPATH=. - python -m pytest tests/svmlib --durations=0 - displayName: 'pytest svmlib' - - - script: | - call activate py$(python.version) - export PYTHONPATH=. - python -m pytest tests/utils --durations=0 - displayName: 'pytest utils' - - - script: | - call activate py$(python.version) - export PYTHONPATH=. - python -m pytest tests/xgboost --durations=0 - displayName: 'pytest xgboost' - - - script: | - call activate py$(python.version) - python -m pip install torch - python -m pip install hummingbird-ml --no-deps - displayName: 'Install hummingbird-ml' - - - script: | - call activate py$(python.version) - export PYTHONPATH=. - python -m pytest tests/hummingbirdml --durations=0 - displayName: 'pytest hummingbirdml' - - - task: PublishTestResults@2 - inputs: - testResultsFiles: '**/test-results.xml' - testRunTitle: 'Python $(python.version)' - condition: succeededOrFailed() diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..03f5e17a --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,138 @@ +name: CI +on: [push, pull_request] +jobs: + run: + name: Py==${{ matrix.python_version }} - sklearn${{ matrix.sklearn_version }} - ${{ matrix.onnxrt_version }} + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest] + python_version: ['3.12', '3.11', '3.10', '3.9'] + include: + - python_version: '3.12' + documentation: 0 + numpy_version: '>=1.21.1' + scipy_version: '>=1.7.0' + onnx_version: 'onnx==1.16.0' + onnxrt_version: 'onnxruntime==1.18.0' + sklearn_version: '>=1.4.2' + lgbm_version: ">=4" + xgboost_version: ">=2" + - python_version: '3.11' + documentation: 1 + numpy_version: '>=1.21.1' + scipy_version: '>=1.7.0' + onnx_version: 'onnx<1.16.0' + onnxrt_version: 'onnxruntime==1.17.3' + sklearn_version: '==1.3.2' + lgbm_version: ">=4" + xgboost_version: ">=2" + - python_version: '3.10' + documentation: 0 + numpy_version: '>=1.21.1' + scipy_version: '>=1.7.0' + onnx_version: 'onnx==1.14.1' + onnxrt_version: 'onnxruntime==1.16.0' + sklearn_version: '==1.2.2' + lgbm_version: "<4" + xgboost_version: "<2" + - python_version: '3.9' + documentation: 0 + numpy_version: '>=1.21.1' + scipy_version: '>=1.7.0' + onnx_version: 'onnx==1.13.0' + onnxrt_version: 'onnxruntime==1.14.0' + sklearn_version: '==1.1.3' + lgbm_version: "<4" + xgboost_version: "<2" + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python_version }} + + - uses: tlylt/install-graphviz@v1 + + - name: Install pandoc + if: matrix.documentation == 1 + run: sudo apt-get install -y pandoc + + - name: Install requirements + run: python -m pip install -r requirements.txt + + - name: Install requirements dev + run: python -m pip install -r requirements-dev.txt + + - name: Cache pip + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('requirements-dev.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + ${{ runner.os }}- + + - name: Install specific versions + run: pip install "${{ matrix.onnx_version }}" "${{ matrix.onnxrt_version }}" "numpy${{ matrix.numpy_version }}" "scikit-learn${{ matrix.sklearn_version}}" "scipy${{ matrix.scipy_version }}" "xgboost${{ matrix.xgboost_version }}" "lightgbm${{ matrix.lgbm_version }}" + + - name: Install + run: pip install -e . + + - name: versions + run: | + python -c "from numpy import __version__;print('numpy', __version__)" + python -c "from pandas import __version__;print('pandas', __version__)" + python -c "from scipy import __version__;print('scipy', __version__)" + python -c "from sklearn import __version__;print('sklearn', __version__)" + python -c "from onnxruntime import __version__;print('onnxruntime', __version__)" + python -c "from onnx import __version__;print('onnx', __version__)" + python -c "from lightgbm import __version__;print('lightgbm', __version__)" + python -c "from xgboost import __version__;print('xgboost', __version__)" + python -c "from catboost import __version__;print('catboost', __version__)" + python -c "from h2o import __version__;print('h2o', __version__)" + python -c "from hummingbirdml import __version__;print('hummingbirdml', __version__)" + python -c "import onnx.defs;print('onnx_opset_version', onnx.defs.onnx_opset_version())" + + - name: Run tests baseline + run: pytest --maxfail=10 --durations=10 tests/baseline + + - name: Run tests utils + run: pytest --maxfail=10 --durations=10 tests/utils + + - name: Run tests catboost + run: pytest --maxfail=10 --durations=10 tests/catboost + + - name: Run tests lightgbm + run: pytest --maxfail=10 --durations=10 tests/lightgbm + + - name: Run tests xgboost + run: pytest --maxfail=10 --durations=10 tests/xgboost + + - name: Run tests svmlib + run: pytest --maxfail=10 --durations=10 tests/svmlib + + - name: Run tests h2o + run: pytest --maxfail=10 --durations=10 tests/h2o + + - name: Run tests pysparkml + run: pytest --maxfail=10 --durations=10 tests/sparkml + + - name: Run tests hummingbirdml + run: | + pip install torch --extra-index-url https://download.pytorch.org/whl/cpu + pip install hummingbird-ml --no-deps + pytest --maxfail=10 --durations=10 tests/hummingbirdml + + - name: Run tests baseline + run: pytest --maxfail=10 --durations=10 tests/baseline + + - name: build + run: pip wheel . + + - uses: actions/upload-artifact@v4 + with: + path: ./dist/** diff --git a/onnxmltools/convert/README.md b/onnxmltools/convert/README.md index ebc828cb..ae77ab10 100644 --- a/onnxmltools/convert/README.md +++ b/onnxmltools/convert/README.md @@ -81,7 +81,7 @@ The shape mapping from Core ML to our IR obeys the following rules. Notice that the compiler can overwrite those rules at some stages like shape inference. An example is the label shape of a classifier. One may expect that its shape is `[1, 1].` Nevertheless, our shape inference may change it to `[1]`. The major reason is that the current definition of ZipMap, the operator used to generate the predicted probabilities, does not support batch size greater than one. Core ML's batch size, `N-axis`, is ignored because it is not related to graph structures. In fact, ONNX's batch size is rather equivalent to sequence axis in Core ML. By default, we use `N=1` for traditional machine learning models and `N='None'` for neural networks. To overwrite our default types, user can provide `initial_types` when calling `convert(...)` defined in `onnxmltools.convert.coreml.convert.py`. All Core ML's shape calculations are derived from [this document](https://apple.github.io/coremltools/coremlspecification/index.html) specifically for our type system. -Some more details about Core ML neural network operator can be found at this [page](https://github.com/apple/coremltools/blob/master/mlmodel/format/NeuralNetwork.proto) +Some more details about Core ML neural network operator can be found at this [page](https://github.com/apple/coremltools/blob/main/mlmodel/format/NeuralNetwork.proto) For scikit-learn, user may need to specify the input types for their models. In general, we expect `[1, C]` if the input is feature vector. diff --git a/onnxmltools/convert/xgboost/operator_converters/XGBoost.py b/onnxmltools/convert/xgboost/operator_converters/XGBoost.py index d241f186..81185428 100644 --- a/onnxmltools/convert/xgboost/operator_converters/XGBoost.py +++ b/onnxmltools/convert/xgboost/operator_converters/XGBoost.py @@ -120,10 +120,10 @@ def _add_node( # Split condition for sklearn # * if X_ptr[X_sample_stride * i + X_fx_stride * node.feature] <= node.threshold: - # * https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/tree/_tree.pyx#L946 + # * https://github.com/scikit-learn/scikit-learn/blob/main/sklearn/tree/_tree.pyx#L946 # Split condition for xgboost # * if (fvalue < split_value) - # * https://github.com/dmlc/xgboost/blob/master/include/xgboost/tree_model.h#L804 + # * https://github.com/dmlc/xgboost/blob/main/include/xgboost/tree_model.h#L804 attr_pairs["nodes_treeids"].append(tree_id) attr_pairs["nodes_nodeids"].append(node_id) @@ -332,7 +332,7 @@ def convert(scope, operator, container): if ncl <= 1: ncl = 2 if objective != "binary:hinge": - # See https://github.com/dmlc/xgboost/blob/master/src/common/math.h#L23. + # See https://github.com/dmlc/xgboost/blob/main/src/common/math.h#L23. attr_pairs["post_transform"] = "LOGISTIC" attr_pairs["class_ids"] = [0 for v in attr_pairs["class_treeids"]] if js_trees[0].get("leaf", None) == 0: @@ -344,7 +344,7 @@ def convert(scope, operator, container): else: attr_pairs["base_values"] = [base_score] else: - # See https://github.com/dmlc/xgboost/blob/master/src/common/math.h#L35. + # See https://github.com/dmlc/xgboost/blob/main/src/common/math.h#L35. attr_pairs["post_transform"] = "SOFTMAX" attr_pairs["base_values"] = [base_score for n in range(ncl)] attr_pairs["class_ids"] = [v % ncl for v in attr_pairs["class_treeids"]]