dataEngineering.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "COVID_Xray.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "wdmGI6DK--tx",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "import matplotlib.image as mpimg\n",
        "from PIL import Image\n",
        "import glob"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CbHR5EE8_IPI",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 122
        },
        "outputId": "fb6616ac-8f31-4bbc-d42f-5f3466a950bd"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1kt6AyYI_xiv",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df_patients = pd.read_csv('/content/drive/My Drive/COVID-CTset/CSV/Patient_details.csv')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Dq0yVrqhAcjB",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "outputId": "4c33dc04-b25a-4c64-8102-ec802b5418ad"
      },
      "source": [
        "df_patients.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Patient ID</th>\n",
              "      <th>SEX</th>\n",
              "      <th>AGE</th>\n",
              "      <th>Date</th>\n",
              "      <th>COVID-19 Infection</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>patient1</td>\n",
              "      <td>F</td>\n",
              "      <td>053Y</td>\n",
              "      <td>20200407</td>\n",
              "      <td>Positive</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>patient3</td>\n",
              "      <td>M</td>\n",
              "      <td>066Y</td>\n",
              "      <td>20200418</td>\n",
              "      <td>Positive</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>patient5</td>\n",
              "      <td>M</td>\n",
              "      <td>035Y</td>\n",
              "      <td>20200328</td>\n",
              "      <td>Positive</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>patient7</td>\n",
              "      <td>F</td>\n",
              "      <td>060Y</td>\n",
              "      <td>20200324</td>\n",
              "      <td>Positive</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>patient9</td>\n",
              "      <td>F</td>\n",
              "      <td>034Y</td>\n",
              "      <td>20200325</td>\n",
              "      <td>Positive</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "  Patient ID SEX   AGE      Date COVID-19 Infection\n",
              "0   patient1   F  053Y  20200407           Positive\n",
              "1   patient3   M  066Y  20200418           Positive\n",
              "2   patient5   M  035Y  20200328           Positive\n",
              "3   patient7   F  060Y  20200324           Positive\n",
              "4   patient9   F  034Y  20200325           Positive"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rbNBwAAPBPbh",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 437
        },
        "outputId": "b614a77f-3a00-4a2e-eee2-6367f383d276"
      },
      "source": [
        "!unzip -uq \"/content/drive/My Drive/COVID-CTset/COVID-CTset.zip\" -d \"/content/drive/My Drive/COVID_XRAY/Images_test\""
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "file #1:  bad zipfile offset (lseek):  0\n",
            "file #2:  bad zipfile offset (local header sig):  42\n",
            "file #3:  bad zipfile offset (local header sig):  91\n",
            "file #4:  bad zipfile offset (local header sig):  149\n",
            "file #5:  bad zipfile offset (local header sig):  212\n",
            "file #6:  bad zipfile offset (lseek):  368640\n",
            "file #7:  bad zipfile offset (lseek):  737280\n",
            "file #8:  bad zipfile offset (lseek):  1114112\n",
            "file #9:  bad zipfile offset (lseek):  1482752\n",
            "file #10:  bad zipfile offset (lseek):  1859584\n",
            "file #11:  bad zipfile offset (lseek):  2228224\n",
            "file #12:  bad zipfile offset (lseek):  2596864\n",
            "file #13:  bad zipfile offset (lseek):  2965504\n",
            "file #14:  bad zipfile offset (lseek):  3334144\n",
            "file #15:  bad zipfile offset (lseek):  3710976\n",
            "file #16:  bad zipfile offset (lseek):  4087808\n",
            "file #17:  bad zipfile offset (lseek):  4464640\n",
            "file #18:  bad zipfile offset (lseek):  4841472\n",
            "file #19:  bad zipfile offset (lseek):  5218304\n",
            "file #20:  bad zipfile offset (lseek):  5595136\n",
            "file #21:  bad zipfile offset (lseek):  5971968\n",
            "file #22:  bad zipfile offset (lseek):  6348800\n",
            "file #23:  bad zipfile offset (lseek):  6725632\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MWQxmWgzMkcw",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "names = df_patients[\"Patient ID\"] + \"_\""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dxeZv2SgA2wW",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 88
        },
        "outputId": "2f915d60-9a19-4cb6-d010-be15fa186c1a"
      },
      "source": [
        "image_list = []\n",
        "\n",
        "for filename in glob.glob('/content/drive/My Drive/COVID_XRAY/Images/*.tif'): #assuming gif\n",
        "\n",
        "    for i, name in enumerate(names):\n",
        "      try:\n",
        "        if name in filename[-40:]:\n",
        "          im = Image.open(filename)\n",
        "          image_list.append([np.array(im), df_patients[\"COVID-19 Infection\"][i]])\n",
        "      except:\n",
        "        print(\"Deu Ruim\")\n",
        "\n",
        "\n",
        "df = pd.DataFrame(image_list, columns = [\"Images\",\"Infection\" ])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/PIL/TiffImagePlugin.py:788: UserWarning: Corrupt EXIF data.  Expecting to read 2 bytes but only got 0. \n",
            "  warnings.warn(str(msg))\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Deu Ruim\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fyr1-uyLqrPa",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "outputId": "765540e6-9578-4dfa-cd11-624277d53d08"
      },
      "source": [
        "df.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Images</th>\n",
              "      <th>Infection</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>[[43, 7, 21, 35, 7, 9, 0, 40, 48, 57, 2, 0, 11...</td>\n",
              "      <td>Negative</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>[[19, 27, 71, 7, 73, 0, 77, 25, 59, 0, 48, 45,...</td>\n",
              "      <td>Negative</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>[[14, 34, 82, 48, 0, 0, 142, 101, 0, 22, 10, 6...</td>\n",
              "      <td>Negative</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>[[35, 42, 78, 28, 14, 0, 71, 77, 22, 28, 0, 7,...</td>\n",
              "      <td>Negative</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>[[44, 29, 30, 25, 12, 14, 27, 31, 29, 34, 26, ...</td>\n",
              "      <td>Negative</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                              Images Infection\n",
              "0  [[43, 7, 21, 35, 7, 9, 0, 40, 48, 57, 2, 0, 11...  Negative\n",
              "1  [[19, 27, 71, 7, 73, 0, 77, 25, 59, 0, 48, 45,...  Negative\n",
              "2  [[14, 34, 82, 48, 0, 0, 142, 101, 0, 22, 10, 6...  Negative\n",
              "3  [[35, 42, 78, 28, 14, 0, 71, 77, 22, 28, 0, 7,...  Negative\n",
              "4  [[44, 29, 30, 25, 12, 14, 27, 31, 29, 34, 26, ...  Negative"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5kXn83nsg4o_",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "f3c77b39-94c4-45ba-fa11-3c2dfaa73dfc"
      },
      "source": [
        "df[\"Images\"][0].shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(512, 512)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Pl4WeE7EbYnT",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "X = []\n",
        "for d in df[\"Images\"]:\n",
        "  X.append(d)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ef55Pr1ug_Te",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "c15ed1d4-c100-44ab-e5bb-0dfe5e7f56af"
      },
      "source": [
        "X = np.array(X)\n",
        "X.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(5745, 512, 512)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XHjLGYSrhQAg",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "np.save('X.npy', X)\n",
        "!cp X.npy '/content/drive/My Drive/COVID_XRAY/'"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_X1MnueuLsx-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df.to_csv(\"covid_xray2.csv\")\n",
        "!cp covid_xray.csv \"/content/drive/My Drive/COVID_XRAY/\""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mpcu4CkMV0zK",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "l = []\n",
        "for d in df[\"Infection\"]:\n",
        "  if d == \"Positive\":\n",
        "    l.append(True)\n",
        "  else:\n",
        "    l.append(False)\n",
        "\n",
        "df[\"Infection Bool\"] = l"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0m5HY5uWV8mI",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "outputId": "9ad2ae28-fe60-49a9-c4af-dcbe7c25b57e"
      },
      "source": [
        "df.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Images</th>\n",
              "      <th>Infection</th>\n",
              "      <th>Infection Bool</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>[[43, 7, 21, 35, 7, 9, 0, 40, 48, 57, 2, 0, 11...</td>\n",
              "      <td>Negative</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>[[19, 27, 71, 7, 73, 0, 77, 25, 59, 0, 48, 45,...</td>\n",
              "      <td>Negative</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>[[14, 34, 82, 48, 0, 0, 142, 101, 0, 22, 10, 6...</td>\n",
              "      <td>Negative</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>[[35, 42, 78, 28, 14, 0, 71, 77, 22, 28, 0, 7,...</td>\n",
              "      <td>Negative</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>[[44, 29, 30, 25, 12, 14, 27, 31, 29, 34, 26, ...</td>\n",
              "      <td>Negative</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                              Images Infection  Infection Bool\n",
              "0  [[43, 7, 21, 35, 7, 9, 0, 40, 48, 57, 2, 0, 11...  Negative           False\n",
              "1  [[19, 27, 71, 7, 73, 0, 77, 25, 59, 0, 48, 45,...  Negative           False\n",
              "2  [[14, 34, 82, 48, 0, 0, 142, 101, 0, 22, 10, 6...  Negative           False\n",
              "3  [[35, 42, 78, 28, 14, 0, 71, 77, 22, 28, 0, 7,...  Negative           False\n",
              "4  [[44, 29, 30, 25, 12, 14, 27, 31, 29, 34, 26, ...  Negative           False"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fmAm07QkiyEk",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df['Infection Bool'].to_csv('y.csv')\n",
        "!cp y.csv \"/content/drive/My Drive/COVID_XRAY/\""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mQqqvK76zEi5",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "X = np.load('/content/drive/My Drive/COVID_XRAY/X.npy')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-ySlGSqlzcnP",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "outputId": "30df0da4-d7cc-4049-a31b-4c2bf5a2e537"
      },
      "source": [
        "y = pd.read_csv('/content/drive/My Drive/COVID_XRAY/y.csv')\n",
        "y.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Unnamed: 0</th>\n",
              "      <th>Infection Bool</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>3</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>4</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   Unnamed: 0  Infection Bool\n",
              "0           0           False\n",
              "1           1           False\n",
              "2           2           False\n",
              "3           3           False\n",
              "4           4           False"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vClghmb_xmpv",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}