Merge Lightning Qubit and Lightning Kokkos backends in a single improved repository

.github/CHANGELOG.md

@@ -2,6 +2,9 @@
 
 ### New features since last release
 
+* Complete overhaul of repository structure to facilitates integration of multiple backends. Refactoring efforts we directed to improve development performance, code reuse and decrease overall overhead to propagate changes through backends. New C++ modular build strategy allows for faster test builds restricted to a module. Update CI/CD actions concurrency strategy. Change minimal Python version to 3.9.
+  [(#472)] (https://github.com/PennyLaneAI/pennylane-lightning/pull/472)
+
 * Wheels are built with native support for sparse Hamiltonians.
   [(#470)] (https://github.com/PennyLaneAI/pennylane-lightning/pull/470)
 
@@ -11,6 +14,8 @@
 ### Breaking changes
 
 ### Improvements
+* Merge Lightning Qubit and Lightning Kokkos backends in the new repository.
+  [(#472)] (https://github.com/PennyLaneAI/pennylane-lightning/pull/472)
 
 ### Documentation
 
@@ -20,7 +25,7 @@
 
 This release contains contributions from (in alphabetical order):
 
-Vincent Michaud-Rioux
+Amintor Dusko, Vincent Michaud-Rioux, Lee J. O'Riordan
 
 ---
 

.github/workflows/build_and_cache_Kokkos_linux.yml

@@ -1,4 +1,4 @@
-name: Build and Cache Kokkos and Kokkos Kernels
+name: Build and Cache Kokkos
 
 env:
   GCC_VERSION: 11
@@ -17,20 +17,19 @@ on:
         description: "Kokkos version"
         value: ${{ jobs.linux-set-builder-matrix.outputs.kokkos_version }}
 
+concurrency:
+  group: build_and_cache_Kokkos_linux-${{ github.ref }}-${{ github.workflow }}
+  cancel-in-progress: true
+
 jobs:
   linux-set-builder-matrix:
     name: Set Kokkos builder matrix
     runs-on: ubuntu-22.04
 
     steps:
-      - name: Cancel Previous Runs
-        uses: styfle/cancel-workflow-action@0.10.0
-        with:
-          access_token: ${{ github.token }}
-
       - name: Kokkos execution strategy
         id: exec_model
-        run: echo "exec_model=[\"SERIAL\"]" >> $GITHUB_OUTPUT # We may also adopt [OPENMP, THREADS] in later iterations
+        run: echo "exec_model=[\"SERIAL\", \"OPENMP\"]" >> $GITHUB_OUTPUT
 
       - name: Kokkos version
         id: kokkos_version
@@ -47,15 +46,10 @@ jobs:
         exec_model: ${{ fromJson(needs.linux-set-builder-matrix.outputs.exec_model) }}
         kokkos_version: ${{ fromJson(needs.linux-set-builder-matrix.outputs.kokkos_version) }}
 
-    name: Kokkos core & kernels (${{ matrix.exec_model }})
+    name: Kokkos core (${{ matrix.exec_model }})
     runs-on: ${{ inputs.os }}
 
     steps:
-      - name: Cancel previous runs
-        uses: styfle/cancel-workflow-action@0.10.0
-        with:
-          access_token: ${{ github.token }}
-
       - name: Cache installation directories
         id: kokkos-cache
         uses: actions/cache@v3
@@ -71,10 +65,6 @@ jobs:
           git checkout ${{ matrix.kokkos_version }}
           cd -
           pushd . &> /dev/null
-          git clone https://github.com/kokkos/kokkos-kernels.git
-          cd kokkos-kernels
-          git checkout ${{ matrix.kokkos_version }}
-          cd -
 
       - name: Install dependencies (Ubuntu)
         if: ${{ (((inputs.os == 'ubuntu-latest') || (inputs.os == 'ubuntu-20.04') || (inputs.os == 'ubuntu-22.04')) && (steps.kokkos-cache.outputs.cache-hit != 'true')) }}
@@ -88,6 +78,7 @@ jobs:
           cd kokkos
           cmake -BBuild . -DCMAKE_INSTALL_PREFIX=${{ github.workspace}}/Kokkos_install/${{ matrix.exec_model }} \
                           -DKokkos_ENABLE_COMPLEX_ALIGN=OFF \
+                          -DKokkos_ENABLE_SERIAL=ON \
                           -DKokkos_ENABLE_${{ matrix.exec_model }}=ON \
                           -DKokkos_ENABLE_DEPRECATION_WARNINGS=OFF \
                           -DCMAKE_CXX_COMPILER="$(which g++-$GCC_VERSION)" \
@@ -96,19 +87,4 @@ jobs:
                           -G Ninja
           cmake --build ./Build --verbose
           cmake --install ./Build
-          cd -
-
-      - name: Build Kokkos kernels library (Ubuntu)
-        if: ${{ (((inputs.os == 'ubuntu-latest') || (inputs.os == 'ubuntu-20.04') || (inputs.os == 'ubuntu-22.04')) && (steps.kokkos-cache.outputs.cache-hit != 'true')) }}
-        run: |
-          mkdir -p ${{ github.workspace}}/Kokkos_install/${{ matrix.exec_model }}
-          cd kokkos-kernels
-          cmake -BBuild . -DCMAKE_INSTALL_PREFIX=${{ github.workspace}}/Kokkos_install/${{ matrix.exec_model }} \
-                          -DKokkos_ENABLE_${{ matrix.exec_model }}=ON \
-                          -DCMAKE_CXX_COMPILER="$(which g++-$GCC_VERSION)" \
-                          -DCMAKE_CXX_STANDARD=20 \
-                          -DCMAKE_PREFIX_PATH=${{ github.workspace}}/Kokkos_install/${{ matrix.exec_model }} \
-                          -DCMAKE_POSITION_INDEPENDENT_CODE=ON \
-                          -G Ninja
-          cmake --build ./Build --verbose
-          cmake --install ./Build
+          cd -

.github/workflows/dev_version_script.py

@@ -17,7 +17,7 @@
 
 import re
 
-VERSION_FILE_PATH = "pennylane_lightning/_version.py"
+VERSION_FILE_PATH = "pennylane_lightning/core/_version.py"
 
 rgx_ver = re.compile('^__version__ = "(.*?)"$')
 

.github/workflows/format.yml

@@ -5,26 +5,25 @@ on:
     branches:
       - master
 
+concurrency:
+  group: format-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   black:
     runs-on: ubuntu-22.04
     steps:
-      - name: Cancel Previous Runs
-        uses: styfle/cancel-workflow-action@0.10.0
-        with:
-          access_token: ${{ github.token }}
-
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: 3.8
+          python-version: '3.9'
 
       - name: Install dependencies
         run:
-            python -m pip install --upgrade pip
-            pip install click==8.0.4 black
+            python -m pip install click==8.0.4 black==23.7.0
 
-      - uses: actions/checkout@v3
+      - name: Checkout PennyLane-Lightning
+        uses: actions/checkout@v3
 
       - name: Run Black
         run: black -l 100 pennylane_lightning/ tests/ --check
@@ -34,11 +33,6 @@ jobs:
     runs-on: ubuntu-22.04
 
     steps:
-      - name: Cancel previous runs
-        uses: styfle/cancel-workflow-action@0.10.0
-        with:
-          access_token: ${{ github.token }}
-
       - name: Install dependencies
         run: sudo apt update && sudo apt -y install clang-format-14 python3
 
@@ -49,7 +43,7 @@ jobs:
         run: ./bin/format --check --cfversion 14 ./pennylane_lightning/src
 
   build_and_cache_Kokkos:
-    name: "Build and cache Kokkos and Kokkos Kernels"
+    name: "Build and cache Kokkos"
     uses: ./.github/workflows/build_and_cache_Kokkos_linux.yml
     with:
       os: ubuntu-22.04
@@ -59,25 +53,23 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-22.04]
+        pl_backend: ["lightning_qubit"]
         exec_model: ${{ fromJson(needs.build_and_cache_Kokkos.outputs.exec_model) }}
         kokkos_version: ${{ fromJson(needs.build_and_cache_Kokkos.outputs.kokkos_version) }}
+        exclude:
+          - exec_model: OPENMP
 
     name: Tidy (C++)
     runs-on: ${{ matrix.os }}
 
     steps:
-      - name: Cancel previous runs
-        uses: styfle/cancel-workflow-action@0.10.0
-        with:
-          access_token: ${{ github.token }}
-
       - name: Checkout PennyLane-Lightning
         uses: actions/checkout@v3
 
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: '3.8'
+          python-version: '3.9'
 
       - name: Restoring cached dependencies
         id: kokkos-cache
@@ -98,5 +90,11 @@ jobs:
 
       - name: Run clang-tidy compilation
         run: |
-          cmake -BBuild -DENABLE_CLANG_TIDY=ON -DCLANG_TIDY_BINARY=clang-tidy-14 -DBUILD_TESTS=ON -DENABLE_WARNINGS=ON -DCMAKE_CXX_COMPILER="$(which g++-10)" -G Ninja .
-          cmake --build ./Build
+          cmake -BBuild -G Ninja . \
+            -DENABLE_CLANG_TIDY=ON \
+            -DCLANG_TIDY_BINARY=clang-tidy-14 \
+            -DBUILD_TESTS=ON \
+            -DENABLE_WARNINGS=ON \
+            -DPL_BACKEND=${{ matrix.pl_backend }} \
+            -DCMAKE_CXX_COMPILER="$(which g++-10)"
+          cmake --build ./Build

.github/workflows/post_release_version_bump.yml

@@ -25,7 +25,7 @@ jobs:
       - name: Run version bump
         run: >
           python .github/workflows/vb_script.py
-          --version_path "./pennylane_lightning/_version.py"
+          --version_path "./pennylane_lightning/core/_version.py"
           --changelog_path "./.github/CHANGELOG.md" --post_release
 
       - name: Create Pull Request

.github/workflows/pre_release_version_bump.yml

@@ -24,7 +24,7 @@ jobs:
       - name: Run version bump
         run: >
           python .github/workflows/vb_script.py
-          --version_path "./pennylane_lightning/_version.py"
+          --version_path "./pennylane_lightning/core/_version.py"
           --changelog_path "./.github/CHANGELOG.md" --pre_release
 
       - name: Create Pull Request

.github/workflows/set_wheel_build_matrix.yml

@@ -1,7 +1,7 @@
 name: Set wheel build matrix
 
 env:
-  PYTHON3_MIN_VERSION: "8"
+  PYTHON3_MIN_VERSION: "9"
   PYTHON3_MAX_VERSION: "11"
 
 on:
@@ -14,17 +14,22 @@ on:
       python_version:
         description: "Python versions."
         value: ${{ jobs.set-builder-matrix.outputs.python_version }}
+      exec_model:
+        description: "The execution model for Kokkos."
+        value: ${{ jobs.set-builder-matrix.outputs.exec_model }}
+      kokkos_version:
+        description: "Kokkos version"
+        value: ${{ jobs.set-builder-matrix.outputs.kokkos_version }}
+
+concurrency:
+  group: set_wheel_build_matrix-${{ github.ref }}-${{ github.workflow }}
+  cancel-in-progress: true
 
 jobs:
   set-builder-matrix:
     runs-on: ubuntu-22.04
 
     steps:
-      - name: Cancel Previous Runs
-        uses: styfle/cancel-workflow-action@0.10.0
-        with:
-          access_token: ${{ github.token }}
-
       - name: Checkout PennyLane-Lightning
         uses: actions/checkout@v3
 
@@ -41,5 +46,15 @@ jobs:
               --max-version=3.${{ env.PYTHON3_MAX_VERSION }} --range)" >> $GITHUB_OUTPUT
           fi
 
+      - name: Kokkos execution strategy
+        id: exec_model
+        run: echo "exec_model=[\"OPENMP\"]" >> $GITHUB_OUTPUT # We may also adopt [THREADS] in later iterations
+
+      - name: Kokkos version
+        id: kokkos_version
+        run: echo "kokkos_version=[\"4.0.01\"]" >> $GITHUB_OUTPUT
+
     outputs:
       python_version: ${{ steps.pyver.outputs.python_version }}
+      exec_model: ${{ steps.exec_model.outputs.exec_model }}
+      kokkos_version: ${{ steps.kokkos_version.outputs.kokkos_version }}