Merge pull request #168 from ThrunGroup/dev

Merging branch `dev` into main; BanditPAM `v2.0.0` release; test failures are random intermittent failures to be expected

.github/workflows/mac_tests.yml

@@ -0,0 +1,73 @@
+name: MacOS - build package and run tests
+on: [push, pull_request]
+jobs:
+  build:
+    runs-on: macos-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"]
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install MacOS dependencies
+        run: |
+          brew install llvm libomp armadillo
+      - name: Install Python dependencies 
+        run: |
+          python -m pip install --upgrade pip
+          pip install pytest
+          pip install -r requirements.txt
+      - name: Install Armadillo 10.7.5+
+        run: |
+          cd ~
+          git clone https://gitlab.com/conradsnicta/armadillo-code.git
+          cd armadillo-code
+          sudo cmake .
+          sudo make install
+      - name: Install carma
+        run: |
+          cd ~
+          git clone https://github.com/RUrlus/carma.git
+          cd carma
+          mkdir build
+          cd build
+          sudo cmake -DCARMA_INSTALL_LIB=ON ..
+          sudo cmake --build . --config Release --target install
+          cd ~
+      - name: Install BanditPAM package
+        run: |
+          pip install -vvv -e .
+        env:
+          # The default compiler on the Github Ubuntu runners is gcc
+          # Would need to make a respective include change for clang 
+          CPLUS_INCLUDE_PATH: /usr/local/include/carma
+      - name: Downloading data files for tests
+        run: |
+          mkdir -p data
+          curl -XGET https://motiwari.com/banditpam_data/scRNA_1k.csv > data/scRNA_1k.csv
+          curl -XGET https://motiwari.com/banditpam_data/scrna_reformat.csv.gz > data/scrna_reformat.csv.gz
+          curl -XGET https://motiwari.com/banditpam_data/MNIST_100.csv > data/MNIST_100.csv
+          curl -XGET https://motiwari.com/banditpam_data/MNIST_1k.csv > data/MNIST_1k.csv
+          curl -XGET https://motiwari.com/banditpam_data/MNIST_10k.tar.gz > data/MNIST_10k.tar.gz
+          tar -xzvf data/MNIST_10k.tar.gz -C data
+          curl -XGET https://motiwari.com/banditpam_data/MNIST_70k.tar.gz > data/MNIST_70k.tar.gz
+          tar -xzvf data/MNIST_70k.tar.gz -C data
+      - name: Run smaller suite of test cases
+        run : |
+          pytest tests/test_smaller.py 
+      # - name: Run larger suite of test cases
+      #   run : |
+      #     pytest tests/test_larger.py 
+      - name: Verify that the C++ executable compiles and runs
+        run : |
+          git clone https://github.com/ThrunGroup/BanditPAM
+          cd BanditPAM
+          mkdir build
+          cd build
+          sudo cmake ..
+          sudo make
+          src/BanditPAM -f ../data/MNIST_1k.csv -k 5

.github/workflows/style_checks.yml

@@ -0,0 +1,33 @@
+name: Run style checks
+on: [push, pull_request]
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"]
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install style check dependencies
+        run: |
+          sudo apt update
+          sudo apt install -y clang-format cppcheck
+      - name: Install Python dependencies 
+        run: |
+          python -m pip install --upgrade pip
+          pip install flake8 pytest cpplint
+          pip install -r requirements.txt
+      - name: Lint with flake8
+        run: |
+          # stop the build if there are Python syntax errors or undefined names
+          flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+          # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+          flake8 . --count --exit-zero --max-complexity=10 --max-line-length=79 --statistics --exclude headers/carma
+      - name: Lint with cpplint
+        run: |
+          cpplint --exclude=headers/carma --exclude=build --filter=-legal/copyright,-build/c++11 --recursive .

.github/workflows/run_tests.yml → .github/workflows/ubuntu_tests.yml

@@ -1,12 +1,12 @@
-name: Build package and run tests
+name: Ubuntu - build package and run tests
 on: [push, pull_request]
 jobs:
   build:
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.6", "3.7", "3.8", "3.9"]
+        python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"]
     steps:
       - uses: actions/checkout@v2
       - name: Set up Python ${{ matrix.python-version }}
@@ -21,8 +21,8 @@ jobs:
       - name: Install Python dependencies 
         run: |
           python -m pip install --upgrade pip
-          pip install flake8 pytest cpplint
-          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+          pip install pytest
+          pip install -r requirements.txt
       - name: Install Armadillo 10.7.5+
         run: |
           cd ~
@@ -40,15 +40,6 @@ jobs:
           sudo cmake -DCARMA_INSTALL_LIB=ON ..
           sudo cmake --build . --config Release --target install
           cd ~
-      - name: Lint with flake8
-        run: |
-          # stop the build if there are Python syntax errors or undefined names
-          flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
-          # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
-          flake8 . --count --exit-zero --max-complexity=10 --max-line-length=79 --statistics --exclude headers/carma
-      - name: Lint with cpplint
-        run: |
-          cpplint --exclude=headers/carma --exclude=build --filter=-legal/copyright,-build/c++11 --recursive .
       - name: Install BanditPAM package
         run: |
           pip install -vvv -e .

.github/workflows/windows_tests.yml

@@ -0,0 +1,78 @@
+name: Windows - build package and run tests
+on: [push, pull_request] # This test is currently disabled until we can build Armadillo on Windows
+jobs:
+  build:
+    runs-on: windows-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"]
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      # - name: Install Windows dependencies
+      #   run: |
+      #     cmake --version
+      # - name: Install CPython from Anaconda # TODO: Need to update CMake? # TODO: See https://docs.conda.io/projects/conda/en/latest/user-guide/install/windows.html
+      #   run: |
+      #     curl -XGET https://repo.anaconda.com/archive/Anaconda3-2021.11-Windows-x86_64.exe > Anaconda3-2021.11-Windows-x86_64.exe
+      #     start /wait "" "Anaconda3-latest-Windows-x86_64.exe" /InstallationType=JustMe /AddToPath=1 /RegisterPython=1 /S /D=%UserProfile%\Anaconda3
+      #     conda create -n "$p{{ matrix.python-version }}" python=${{ matrix.python-version }}
+      #     conda activate p${{ matrix.python-version }}
+      - name: Install Python dependencies 
+        run: |
+          python -m pip install --upgrade pip
+          pip install pytest
+          pip install -r requirements.txt
+      - name: Install Armadillo 10.7.5+
+        run: |
+          cd ~
+          git clone https://gitlab.com/conradsnicta/armadillo-code.git
+          cd armadillo-code
+          cmake .
+      - name: Install carma
+        run: |
+          cd ~
+          git clone https://github.com/RUrlus/carma.git
+          cd carma
+          mkdir build
+          cd build
+          cmake -DCARMA_INSTALL_LIB=ON ..
+          cmake --build . --config Release --target install
+          cd ~
+      - name: Install BanditPAM package
+        run: |
+          pip install -vvv -e .
+        env:
+          # The default compiler on the Github Ubuntu runners is gcc
+          # Would need to make a respective include change for clang 
+          CPLUS_INCLUDE_PATH: /usr/local/include/carma
+      - name: Downloading data files for tests
+        run: |
+          mkdir -p data
+          curl -XGET https://motiwari.com/banditpam_data/scRNA_1k.csv > data/scRNA_1k.csv
+          curl -XGET https://motiwari.com/banditpam_data/scrna_reformat.csv.gz > data/scrna_reformat.csv.gz
+          curl -XGET https://motiwari.com/banditpam_data/MNIST_100.csv > data/MNIST_100.csv
+          curl -XGET https://motiwari.com/banditpam_data/MNIST_1k.csv > data/MNIST_1k.csv
+          curl -XGET https://motiwari.com/banditpam_data/MNIST_10k.tar.gz > data/MNIST_10k.tar.gz
+          tar -xzvf data/MNIST_10k.tar.gz -C data
+          curl -XGET https://motiwari.com/banditpam_data/MNIST_70k.tar.gz > data/MNIST_70k.tar.gz
+          tar -xzvf data/MNIST_70k.tar.gz -C data
+      - name: Run smaller suite of test cases
+        run : |
+          pytest tests/test_smaller.py 
+      # - name: Run larger suite of test cases
+      #   run : |
+      #     pytest tests/test_larger.py 
+      # - name: Verify that the C++ executable compiles and runs
+      #   run : |
+      #     git clone https://github.com/ThrunGroup/BanditPAM
+      #     cd BanditPAM
+      #     mkdir build
+      #     cd build
+      #     cmake ..
+      #     make
+      #     src/BanditPAM -f ../data/MNIST_1k.csv -k 5

README.md

@@ -17,7 +17,7 @@ Mo Tiwari, Martin Jinye Zhang, James Mayclin, Sebastian Thrun, Chris Piech, Ilan
 ```
 
 # Requirements
-# TL;DR run `pip3 install banditpam` and jump to the [examples](https://github.com/ThrunGroup/BanditPAM#example-1-synthetic-data-from-a-gaussian-mixture-model). 
+# TL;DR run `pip install banditpam` and jump to the [examples](https://github.com/ThrunGroup/BanditPAM#example-1-synthetic-data-from-a-gaussian-mixture-model). 
 
 If you have any difficulties, please see the [platform-specific guides](https://github.com/ThrunGroup/BanditPAM#platform-specific-installation-guides) and file a Github issue if you have additional trouble.
 
@@ -41,7 +41,7 @@ OR through the source code via
 ```python
 /BanditPAM/: git submodule update --init --recursive
 /BanditPAM/: cd headers/carma
-/BanditPAM/: mkdir build && cd build && cmake .. && make && sudo make install
+/BanditPAM/: mkdir build && cd build && cmake -DCARMA_INSTALL_LIB=ON .. && sudo cmake --build . --config Release --target install
 /BanditPAM/: cd ../../..
 /BanditPAM/: pip install -r requirements.txt
 /BanditPAM/: sudo pip install .
@@ -140,13 +140,14 @@ Building this repository requires four external requirements:
 * [OpenMP](https://www.openmp.org/resources/openmp-compilers-tools/) >= 2.5 (OpenMP is supported by default on most Linux platforms, and can be downloaded through [homebrew](https://brew.sh/) on MacOS)
 * [CARMA](https://github.com/RUrlus/carma) >= 0.6.2
 
-If installing these requirements from source, one can generally use the following procedure to install each requirement from the library's root folder (with CARMA used as an example here):
+If installing these requirements from source, one can generally use the following procedure to install each requirement from the library's root folder (with `armadillo` used as an example here):
 ```
-/BanditPAM$ cd headers/carma
-/BanditPAM/headers/carma$ mkdir build && cd build
-/BanditPAM/headers/carma/build$ cmake .. && make && sudo make install
+/armadillo$ mkdir build && cd build
+/armadillo/build$ cmake .. && make && sudo make install
 ```
 
+Note that `CARMA` has different installation instructions; see the [quickstart](https://github.com/ThrunGroup/BanditPAM#install-the-repo-and-its-dependencies).
+
 ####  Platform-specific installation guides
 Further installation information for [MacOS](docs/install_mac.md), [Linux](docs/install_linux.md), and [Windows](docs/install_windows.md) is available in the [docs folder](docs). Ensure all the requirements above are installed and then run:
 

docs/Doxyfile

@@ -1223,7 +1223,7 @@ HTML_FILE_EXTENSION    = .html
 # of the possible markers and block names see the documentation.
 # This tag requires that the tag GENERATE_HTML is set to YES.
 
-HTML_HEADER            =
+HTML_HEADER            = headerFile
 
 # The HTML_FOOTER tag can be used to specify a user-defined HTML footer for each
 # generated HTML page. If the tag is left blank doxygen will generate a standard
@@ -1268,7 +1268,7 @@ HTML_EXTRA_STYLESHEET  =
 # files will be copied as-is; there are no commands or markers available.
 # This tag requires that the tag GENERATE_HTML is set to YES.
 
-HTML_EXTRA_FILES       =
+HTML_EXTRA_FILES       = banditpam_favicon.ico
 
 # The HTML_COLORSTYLE_HUE tag controls the color of the HTML output. Doxygen
 # will adjust the colors in the style sheet and background images according to

docs/headerFile

@@ -0,0 +1,74 @@
+<!-- HTML header for doxygen 1.9.3-->
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "https://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head>
+<meta http-equiv="Content-Type" content="text/xhtml;charset=UTF-8"/>
+<meta http-equiv="X-UA-Compatible" content="IE=11"/>
+<meta name="generator" content="Doxygen $doxygenversion"/>
+<meta name="viewport" content="width=device-width, initial-scale=1"/>
+<link rel="shortcut icon" href="$relpath^banditpam_favicon.ico" type="image/x-icon" />
+<!--BEGIN PROJECT_NAME--><title>$projectname: $title</title><!--END PROJECT_NAME-->
+<!--BEGIN !PROJECT_NAME--><title>$title</title><!--END !PROJECT_NAME-->
+<link href="$relpath^tabs.css" rel="stylesheet" type="text/css"/>
+<!--BEGIN DISABLE_INDEX-->
+  <!--BEGIN FULL_SIDEBAR-->
+<script type="text/javascript">var page_layout=1;</script>
+  <!--END FULL_SIDEBAR-->
+<!--END DISABLE_INDEX-->
+<script type="text/javascript" src="$relpath^jquery.js"></script>
+<script type="text/javascript" src="$relpath^dynsections.js"></script>
+$treeview
+$search
+$mathjax
+<link href="$relpath^$stylesheet" rel="stylesheet" type="text/css" />
+$extrastylesheet
+</head>
+<body>
+<!--BEGIN DISABLE_INDEX-->
+  <!--BEGIN FULL_SIDEBAR-->
+<div id="side-nav" class="ui-resizable side-nav-resizable"><!-- do not remove this div, it is closed by doxygen! -->
+  <!--END FULL_SIDEBAR-->
+<!--END DISABLE_INDEX-->
+
+<div id="top"><!-- do not remove this div, it is closed by doxygen! -->
+
+<!--BEGIN TITLEAREA-->
+<div id="titlearea">
+<table cellspacing="0" cellpadding="0">
+ <tbody>
+ <tr id="projectrow">
+  <!--BEGIN PROJECT_LOGO-->
+  <td id="projectlogo"><img alt="Logo" src="$relpath^$projectlogo"/></td>
+  <!--END PROJECT_LOGO-->
+  <!--BEGIN PROJECT_NAME-->
+  <td id="projectalign">
+   <div id="projectname">$projectname<!--BEGIN PROJECT_NUMBER--><span id="projectnumber">&#160;$projectnumber</span><!--END PROJECT_NUMBER-->
+   </div>
+   <!--BEGIN PROJECT_BRIEF--><div id="projectbrief">$projectbrief</div><!--END PROJECT_BRIEF-->
+  </td>
+  <!--END PROJECT_NAME-->
+  <!--BEGIN !PROJECT_NAME-->
+   <!--BEGIN PROJECT_BRIEF-->
+    <td>
+    <div id="projectbrief">$projectbrief</div>
+    </td>
+   <!--END PROJECT_BRIEF-->
+  <!--END !PROJECT_NAME-->
+  <!--BEGIN DISABLE_INDEX-->
+   <!--BEGIN SEARCHENGINE-->
+     <!--BEGIN !FULL_SIDEBAR-->
+    <td>$searchbox</td>
+     <!--END !FULL_SIDEBAR-->
+   <!--END SEARCHENGINE-->
+  <!--END DISABLE_INDEX-->
+ </tr>
+  <!--BEGIN SEARCHENGINE-->
+   <!--BEGIN FULL_SIDEBAR-->
+   <tr><td colspan="2">$searchbox</td></tr>
+   <!--END FULL_SIDEBAR-->
+  <!--END SEARCHENGINE-->
+ </tbody>
+</table>
+</div>
+<!--END TITLEAREA-->
+<!-- end header part -->