Merge pull request #17 from NOEL-MNI/update_packages

Update packages no longer available from PyPi and resolve dependencies

.github/workflows/docker-image-build.yml

@@ -7,10 +7,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out the repo
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: Build Docker image
-        uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
+        uses: docker/build-push-action@v5
         with:
           context: .
           push: false

.github/workflows/docker-image-release.yml

@@ -10,22 +10,22 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out the repo
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: Log in to Docker Hub
-        uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
 
       - name: Extract metadata (tags, labels) for Docker
         id: meta
-        uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38
+        uses: docker/metadata-action@v5
         with:
           images: noelmni/deep-fcd
 
       - name: Build and push Docker image
-        uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
+        uses: docker/build-push-action@v5
         with:
           context: .
           push: true

.github/workflows/python-build.yml

@@ -12,7 +12,7 @@ jobs:
         python-version: ["3.7", "3.8"]
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v4
         with:

Dockerfile

@@ -1,4 +1,4 @@
-FROM nvidia/cuda:10.0-cudnn7-devel-ubuntu18.04
+FROM noelmni/cuda:10.0-cudnn7-devel-ubuntu18.04
 LABEL maintainer="Ravnoor Singh Gill <ravnoor@gmail.com>" \
         org.opencontainers.image.title="deepFCD" \
         org.opencontainers.image.description="Automated Detection of Focal Cortical Dysplasia using Deep Learning" \
@@ -35,17 +35,17 @@ USER user
 ENV HOME=/home/user
 RUN chmod 777 /home/user
 
-RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-py37_4.12.0-Linux-x86_64.sh \
-    && /bin/bash Miniconda3-py37_4.12.0-Linux-x86_64.sh -b -p /home/user/conda \
-    && rm -f Miniconda3-py37_4.12.0-Linux-x86_64.sh
+RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-py38_23.5.2-0-Linux-x86_64.sh \
+    && /bin/bash Miniconda3-py38_23.5.2-0-Linux-x86_64.sh -b -p /home/user/conda \
+    && rm Miniconda3-py38_23.5.2-0-Linux-x86_64.sh
 
-RUN conda update -n base -c defaults conda
+# RUN conda update -n base -c defaults conda
 
 RUN git clone --depth 1 https://github.com/NOEL-MNI/deepMask.git \
     && rm -rf deepMask/.git
 
 RUN eval "$(conda shell.bash hook)" \
-    && conda create -n preprocess python=3.7 \
+    && conda create -n preprocess python=3.8 \
     && conda activate preprocess \
     && python -m pip install -r deepMask/app/requirements.txt \
     && conda deactivate

Makefile

@@ -9,6 +9,8 @@ TMPDIR	:= /host/hamlet/local_raid/data/ravnoor/sandbox
 BRAIN_MASKING := 1
 PREPROCESS		:= 1
 
+.PHONY: all clean
+
 build:
 	docker build -t $(ACCOUNT)/$(SERVICE):$(TAG) .
 

README.md

@@ -5,7 +5,7 @@
 
 <p align="center">
       <a href="https://www.python.org/">
-        <img src="https://img.shields.io/badge/Python-3.7-ff69b4.svg" /></a>
+        <img src="https://img.shields.io/badge/Python-3.7+-ff69b4.svg" /></a>
       <a href= "https://keras.io/">
         <img src="https://img.shields.io/badge/Keras-2.2.4-2BAF2B.svg" /></a>
       <a href= "https://github.com/Theano/Theano">
@@ -42,12 +42,12 @@ OR
 ## Pre-requisites
 ```bash
 0. Anaconda Python Environment
-1. Python == 3.7.x
+1. Python == 3.8
 2. Keras == 2.2.4
 3. Theano == 1.0.4
-4. ANTsPy == 0.3.5 (for MRI preprocessing)
-4. ANTsPyNet == 0.2.0 (for deepMask)
-5. PyTorch == 1.11.0 (for deepMask)
+4. ANTsPy == 0.4.2 (for MRI preprocessing)
+4. ANTsPyNet == 0.2.3 (for deepMask)
+5. PyTorch == 1.8.2 LTS (for deepMask)
 6. h5py == 2.10.0
 + app/requirements.txt
 + app/deepMask/app/requirements.txt
@@ -65,14 +65,14 @@ wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/
 bash ~/miniconda.sh -b -p $HOME/miniconda
 
 # create and activate a Conda environment for preprocessing
-conda create -n preprocess python=3.7
+conda create -n preprocess python=3.8
 conda activate preprocess
 # install dependencies using pip
 python -m pip install -r app/deepMask/app/requirements.txt
 conda deactivate
 
 # create and activate a Conda environment for deepFCD
-conda create -n deepFCD python=3.7
+conda create -n deepFCD python=3.8
 conda activate deepFCD
 # install dependencies using pip
 python -m pip install -r app/requirements.txt

app/requirements.txt

@@ -1,4 +1,4 @@
-antspyx==0.3.5 --only-binary=antspyx
+antspyx==0.4.2 --only-binary=antspyx
 git+https://github.com/ravnoor/atlasreader@master#egg=atlasreader
 Theano==1.0.4
 keras==2.2.4
@@ -15,5 +15,5 @@ scikit-learn==1.0.2
 scipy==1.7.3
 setproctitle==1.2.3
 tabulate==0.9.0
-tqdm==4.64.0
+tqdm==4.65.0
 xlrd==2.0.1

app/utils/base.py

@@ -21,12 +21,26 @@
 
 
 def print_data_shape(X):
+    """ Print shape of the training data
+
+    Args:
+        X (_type_): numpy array with the 3D patches for T1w and FLAIR
+    """    
     print("====> # 3D training patches:", X.shape[0], "\n")
     print("====> # patch size:", (X.shape[2], X.shape[3], X.shape[4]), "\n")
     print("====> # modalities:", (X.shape[1]), "\n")
 
 
 def partition_leave_one_site_out(datafile=None, test_site=None):
+    """Partition the data based on leave-one-site-out (LoSo).
+
+    Args:
+        datafile (_type_, optional): CSV file containing all patient IDs associated with their respective site label. Defaults to None.
+        test_site (_type_, optional): The site label to be used as held-out. Defaults to None.
+
+    Returns:
+        _type_: _description_
+    """    
     data = pd.read_excel(datafile)
     ids = data["index"]
     groups = data["testing_dataset"].values

app/utils/create_hdf5_patch_dataset.py

@@ -1,30 +1,29 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 import os
+try:
+    import h5py
+except ImportError:
+    raise ImportError('install h5py first: `pip install h5py --upgrade`')
+
+import numpy as np
 
 os.environ["KERAS_BACKEND"] = "theano"
 os.environ["THEANO_FLAGS"] = "mode=FAST_RUN,device=cpu,floatX=float32"
 print(os.environ["THEANO_FLAGS"])
 
 import time
-
-# from utils.base import *
-from utils.h5data import *
-from utils.metrics import *
-
-# from keras.utils import to_categorical
-
-# import utils.h5data as h5d
+from h5data import load_training_data, create_dataset
 
 # set configuration parameters
 options = {}
-options["n_patches"] = 1000
+options["n_patches"] = 1500
 options["seed"] = 666
 options["modalities"] = ["T1", "FLAIR"]
 options["x_names"] = ["_t1.nii.gz", "_flair.nii.gz"]
 options["y_names"] = ["_lesion.nii.gz"]
 options["submask_names"] = ["subcorticalMask_final_negative.nii.gz"]
-options["patch_size"] = (18, 18, 18)
+options["patch_size"] = (16, 16, 16)
 
 options["thr"] = 0.1
 options["min_th"] = options["thr"]
@@ -39,11 +38,12 @@
 seed = options["seed"]
 print("seed: {}".format(seed))
 # Select an experiment name to store net weights and segmentation masks
-options["experiment"] = "noel_FCDdata_"
+options["experiment"] = "noel_FCDdata"
 
 options["model_dir"] = "./weights"  # weights/noel_dropoutMC_model_{1,2}.h5
 options["train_folder"] = "/host/hamlet/local_raid/data/ravnoor/01_Projects/55_Bayesian_DeepLesion_LoSo/data/"
-options["data_folder"] = "/host/hamlet/local_raid/data/ravnoorX/data/noel_hdf5"
+# options["data_folder"] = "/host/hamlet/local_raid/data/ravnoorX/data/noel_hdf5"
+options["data_folder"] = "/tmp/noel_hdf5"
 
 list_of_train_scans = os.listdir(options["train_folder"] + "brain")
 include_train = list(set(list_of_train_scans))
@@ -93,8 +93,11 @@
     + str(options["n_patches"])
     + "_patchsize_"
     + str(options["patch_size"][0])
-    + "_iso.h5"
+    + "_iso_fix.h5"
 )
+
+print(np.histogram(y, bins=2))
+
 datapath = os.path.join(options["data_folder"], h5_fname)
 print("\nhdf5 dataset is being created: {}".format(datapath))
 
@@ -103,3 +106,17 @@
 end = time.time()
 diff = end - start
 print("time elapsed: ~ {} minutes".format(diff // 60))
+
+# validate the newly created dataset
+print("\nhdf5 dataset is being loaded: {}".format(datapath))
+
+# sample hdf5 dataset available from https://doi.org/10.5281/zenodo.3239446
+with h5py.File(datapath, "r") as f:
+    X = f['data'][:].astype('f')
+    y = f['labels'][:].astype('i')
+
+# output the shape of the patches and labels
+print(X.shape, y.shape)
+
+# should output equal number of positive and negative examples (0/1)
+print(np.histogram(y, bins=2))