Refactoring of benchmarks

README.md

@@ -8,7 +8,7 @@ Benefits:
 - Full control of benchmarks suite through CLI
 - Flexible and powerful benchmark config structure
 - Available with advanced profiling tools, such as Intel(R) VTune* Profiler
-- Automated generation of benchmarks report
+- Automated benchmarks report generation
 
 ### 📜 Table of Contents
 
@@ -23,7 +23,7 @@ Benefits:
 
 ## 🔧 Create a Python Environment
 
-Create a suitable Python environment to use depending on the needed frameworks:
+How to create a usable Python environment with the following required frameworks:
 
 - **sklearn, sklearnex, and gradient boosting frameworks**:
 
@@ -44,7 +44,7 @@ conda env create -n rapids --solver=libmamba -f envs/conda-env-rapids.yml
 
 ### Benchmarks Runner
 
-Call required benchmarking cases from specified config using the `sklbench` module as a runner:
+How to run benchmarks using the `sklbench` module and a specific configuration:
 
 ```bash
 python -m sklbench --config configs/sklearn_example.json

azure-pipelines.yml

@@ -13,9 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #===============================================================================
-variables:
-  - name: python.version
-    value: "3.10"
 
 jobs:
   - job: Lint
@@ -35,9 +32,23 @@ jobs:
     dependsOn: Lint
     strategy:
       matrix:
-        Pip:
+        Python3.9_Pip:
+          PYTHON_VERSION: "3.9"
           PKG_MANAGER: "pip"
-        Conda:
+        Python3.10_Pip:
+          PYTHON_VERSION: "3.10"
+          PKG_MANAGER: "pip"
+        Python3.11_Pip:
+          PYTHON_VERSION: "3.11"
+          PKG_MANAGER: "pip"
+        Python3.9_Conda:
+          PYTHON_VERSION: "3.9"
+          PKG_MANAGER: "conda"
+        Python3.10_Conda:
+          PYTHON_VERSION: "3.10"
+          PKG_MANAGER: "conda"
+        Python3.11_Conda:
+          PYTHON_VERSION: "3.11"
           PKG_MANAGER: "conda"
     pool:
       vmImage: "ubuntu-latest"
@@ -47,9 +58,23 @@ jobs:
     dependsOn: Lint
     strategy:
       matrix:
-        Pip:
+        Python3.9_Pip:
+          PYTHON_VERSION: "3.9"
           PKG_MANAGER: "pip"
-        Conda:
+        Python3.10_Pip:
+          PYTHON_VERSION: "3.10"
+          PKG_MANAGER: "pip"
+        Python3.11_Pip:
+          PYTHON_VERSION: "3.11"
+          PKG_MANAGER: "pip"
+        Python3.9_Conda:
+          PYTHON_VERSION: "3.9"
+          PKG_MANAGER: "conda"
+        Python3.10_Conda:
+          PYTHON_VERSION: "3.10"
+          PKG_MANAGER: "conda"
+        Python3.11_Conda:
+          PYTHON_VERSION: "3.11"
           PKG_MANAGER: "conda"
     pool:
       vmImage: "windows-latest"

configs/README.md

@@ -2,7 +2,7 @@
 
 Benchmarking cases in `scikit-learn_bench` are defined by configuration files and stored in the `configs` directory of the repository.
 
-Config defines:
+The configuration file (config) defines:
  - Measurement and profiling parameters
  - Library and algorithm to use
  - Algorithm-specific parameters
@@ -172,4 +172,4 @@ You can remove specific parameter from subset of cases when stacking parameters
 ```
 
 ---
-[Documentation tree](../README.md#-documentation-tree)
+[Documentation tree](../README.md#-documentation)

docs/README.md

@@ -66,4 +66,4 @@ Benchmark parameters the following overwriting priority:
 3. Parameters set
 
 ---
-[Documentation tree](../README.md#-documentation-tree)
+[Documentation tree](../README.md#-documentation)

envs/conda-env-sklearn.yml

@@ -2,19 +2,15 @@ channels:
   - conda-forge
   - nodefaults
 dependencies:
-  - python=3.10
   # additional frameworks
   - xgboost
   - catboost
   - lightgbm
   - faiss-cpu
   - modin-all
-  - intel::scikit-learn-intelex
-  - intel::daal4py
-  - intel::dpctl
-  - intel::dpnp
+  - scikit-learn-intelex
   # sklbench dependencies
-  - scikit-learn=1.4.*
+  - scikit-learn<1.5
   - pandas
   - tabulate
   - fastparquet

envs/requirements-sklearn.txt

@@ -8,7 +8,7 @@ scikit-learn-intelex
 dpctl
 dpnp
 # sklbench dependencies
-scikit-learn==1.4.*
+scikit-learn<1.5
 pandas
 tabulate
 fastparquet

pyproject.toml

@@ -17,7 +17,7 @@
 
 [tool.black]
 line-length = 90
-target-version = ['py38', 'py39', 'py310', 'py311', 'py312']
+target-version = ['py39', 'py310', 'py311', 'py312']
 extend-ignore = 'E203'
 
 [tool.isort]

test-configuration-linux.yml

@@ -15,12 +15,12 @@
 #===============================================================================
 steps:
   - task: UsePythonVersion@0
-    displayName: "Use Python $(python.version)"
+    displayName: "Use Python $(PYTHON_VERSION)"
     inputs:
-      versionSpec: "$(python.version)"
+      versionSpec: "$(PYTHON_VERSION)"
   - script: |
-      conda create -y -n bench-env -c conda-forge -c nodefaults python=$(python.version)
-    displayName: Environment update
+      conda create -y -n bench-env -c conda-forge -c nodefaults python=$(PYTHON_VERSION)
+    displayName: Environment initialization
   - script: |
       source /usr/share/miniconda/etc/profile.d/conda.sh
       conda activate bench-env
@@ -38,7 +38,7 @@ steps:
   - script: |
       source /usr/share/miniconda/etc/profile.d/conda.sh
       conda activate bench-env
-      python -m sklbench -l DEBUG -p algorithm:library=sklearnex algorithm:estimator=KMeans data:dataset=a9a
+      python -m sklbench --report -l DEBUG -p algorithm:library=sklearn,sklearnex algorithm:estimator=PCA,KMeans,ElasticNet,KNeighborsClassifier data:dataset=skin_segmentation data:split_kwargs:train_size=5000 data:split_kwargs:test_size=5000
     displayName: CLI arguments run
   - script: |
       source /usr/share/miniconda/etc/profile.d/conda.sh

test-configuration-win.yml

@@ -15,14 +15,14 @@
 #===============================================================================
 steps:
   - task: UsePythonVersion@0
-    displayName: "Use Python $(python.version)"
+    displayName: "Use Python $(PYTHON_VERSION)"
     inputs:
-      versionSpec: "$(python.version)"
+      versionSpec: "$(PYTHON_VERSION)"
   - powershell: Write-Host "##vso[task.prependpath]$env:CONDA\Scripts"
     displayName: Add conda to Windows PATH
   - script: |
-      conda create -y -n bench-env -c conda-forge -c nodefaults python=$(python.version)
-    displayName: Environment update
+      conda create -y -n bench-env -c conda-forge -c nodefaults python=$(PYTHON_VERSION)
+    displayName: Environment initialization
   - script: |
       call activate bench-env
       pip install -r envs/requirements-sklearn.txt
@@ -37,7 +37,7 @@ steps:
     condition: eq(variables['PKG_MANAGER'], 'conda')
   - script: |
       call activate bench-env
-      python -m sklbench -l DEBUG -p algorithm:library=sklearnex algorithm:estimator=KMeans data:dataset=a9a
+      python -m sklbench --report -l DEBUG -p algorithm:library=sklearn,sklearnex algorithm:estimator=PCA,KMeans,ElasticNet,KNeighborsClassifier data:dataset=skin_segmentation data:split_kwargs:train_size=5000 data:split_kwargs:test_size=5000
     displayName: CLI arguments run
   - script: |
       call activate bench-env