covidpredict_pub.py

# -*- coding: utf-8 -*-
"""CovidPredict_Pub.ipynb

Automatically generated by Colaboratory.

"""

# Covid predict program

import os
import random
import tensorflow as tf
from tensorflow.keras.optimizers import RMSprop
from shutil import copyfile

# import data from google drive
from google.colab import drive
drive.mount('/content/drive')

import zipfile

local_zip = '/content/drive/My Drive/CTImages.zip'
zip_ref   = zipfile.ZipFile(local_zip, 'r')
zip_ref.extractall('/tmp')
zip_ref.close()

train_covid_dir = '/tmp/CTImages/Covid/'
train_non_dir = '/tmp/CTImages/Non/'

print(len(os.listdir(train_covid_dir)))
print(len(os.listdir(train_non_dir)))


"""### Training and validation folders"""

try:
  os.mkdir('/tmp/Covid-v-Non')
  os.mkdir('/tmp/Covid-v-Non/training')
  os.mkdir('/tmp/Covid-v-Non/validation')
  os.mkdir('/tmp/Covid-v-Non/training/Covid')
  os.mkdir('/tmp/Covid-v-Non/training/Non')
  os.mkdir('/tmp/Covid-v-Non/validation/Covid')
  os.mkdir('/tmp/Covid-v-Non/validation/Non')
except OSError:
  pass

def split_data(SOURCE, TRAINING, TESTING, SPLIT_SIZE):
    files = []
    for filename in os.listdir(SOURCE):
        file = SOURCE + filename
        if os.path.getsize(file) > 0:
            files.append(filename)
        else:
            print(filename + " is zero length, so ignoring.")

    training_length = int(len(files) * SPLIT_SIZE)
    testing_length = int(len(files) - training_length)
    shuffled_set = random.sample(files, len(files))
    training_set = shuffled_set[0:training_length]
    testing_set = shuffled_set[-testing_length:]

    for filename in training_set:
        this_file = SOURCE + filename
        destination = TRAINING + filename
        copyfile(this_file, destination)

    for filename in testing_set:
        this_file = SOURCE + filename
        destination = TESTING + filename
        copyfile(this_file, destination)

COVID_SOURCE_DIR = "/tmp/CTImages/Covid/"       # train_covid_dir
TRAINING_COVID_DIR = "/tmp/Covid-v-Non/training/Covid/"
VALIDATION_COVID_DIR = "/tmp/Covid-v-Non/validation/Covid/"

NON_SOURCE_DIR = "/tmp/CTImages/Non/"           # train_non_dir
TRAINING_NON_DIR = "/tmp/Covid-v-Non/training/Non/"
VALIDATION_NON_DIR = "/tmp/Covid-v-Non/validation/Non/"

split_size = .73
split_data(COVID_SOURCE_DIR, TRAINING_COVID_DIR, VALIDATION_COVID_DIR, split_size)
split_data(NON_SOURCE_DIR, TRAINING_NON_DIR, VALIDATION_NON_DIR, split_size)

print(len(os.listdir('/tmp/Covid-v-Non/training/Covid/')))
print(len(os.listdir('/tmp/Covid-v-Non/training/Non/')))

print(len(os.listdir('/tmp/Covid-v-Non/validation/Covid/')))
print(len(os.listdir('/tmp/Covid-v-Non/validation/Non/')))

"""*texte en italique*# Download Model : DenseNet or NAS"""

from keras.applications.densenet import DenseNet121

IMG_SIZE = 224 # All images will be resized to 150x150
IMG_SHAPE = (IMG_SIZE, IMG_SIZE, 3)

# raises ValueError
try:
  pre_trained_model = DenseNet121(input_shape=IMG_SHAPE, include_top=False)
except ValueError as e:
    print(e)

pre_trained_model.summary()

from keras.layers import Input
from keras.optimizers import RMSprop
from keras import layers
from keras import Model

last_layer = pre_trained_model.get_layer('relu')
#last_layer = pre_trained_model.get_layer('block_6_project_BN')
print('last layer output shape: ', last_layer.output_shape)
last_output = last_layer.output


x = layers.Flatten()(last_output)
x = layers.Dense(512, activation='relu')(x)
x = layers.Dropout(0.2)(x)                  
x = layers.Dense  (1, activation='sigmoid')(x)           

model = Model( pre_trained_model.input, x) 

model.summary()

"""### *Prepare trainig*"""

model.compile(loss='binary_crossentropy',
              optimizer=RMSprop(lr=1e-4),
              metrics=['accuracy'])

#model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy'])

train_dir = '/tmp/Covid-v-Non/training'
validation_dir = '/tmp/Covid-v-Non/validation'

from tensorflow.keras.preprocessing.image import ImageDataGenerator

train_datagen = ImageDataGenerator(
      rescale=1./255,
      rotation_range=40,
      width_shift_range=0.2,
      height_shift_range=0.2,
      shear_range=0.2,
      zoom_range=0.2,
      horizontal_flip=True,
      fill_mode='nearest')

test_datagen = ImageDataGenerator(rescale=1./255)


# Flow training images in batches of 20 using train_datagen generator
train_generator = train_datagen.flow_from_directory(
        train_dir,  # This is the source directory for training images
        target_size=(IMG_SIZE, IMG_SIZE),  # All images will be resized to 150x150
        batch_size=20,
        # Since we use binary_crossentropy loss, we need binary labels
        class_mode='binary')

# Flow validation images in batches of 20 using test_datagen generator
validation_generator = test_datagen.flow_from_directory(
        validation_dir,
        target_size=(IMG_SIZE, IMG_SIZE),
        batch_size=20,
        class_mode='binary')

# ---------------------- 
class myCallback(tf.keras.callbacks.Callback):
  def on_epoch_end(self, epoch, logs={}):
    if(logs.get('accuracy')>0.99):
      print("\nReached 95% accuracy so cancelling training!")
      self.model.stop_training = True

callbacks = myCallback()
# ----------------------


history = model.fit(
      train_generator,
      steps_per_epoch=20,  # 500\20   images = batch_size * steps
      epochs=100,
      validation_data=validation_generator,
      validation_steps=10,  #50\10    images = batch_size * steps
      verbose=2,
      callbacks=[callbacks])

"""### Evaluating Accuracy and Loss for the Model"""

import matplotlib.pyplot as plt

acc      = history.history[     'accuracy' ]
val_acc  = history.history[ 'val_accuracy' ]
loss     = history.history[    'loss' ]
val_loss = history.history['val_loss' ]

epochs   = range(len(acc)) # Get number of epochs


plt.plot  ( epochs,     acc )
plt.plot  ( epochs, val_acc )
plt.title ('Training and validation accuracy')
plt.figure()


plt.plot  ( epochs,     loss )
plt.plot  ( epochs, val_loss )
plt.title ('Training and validation loss'   )