OHE Input SO Model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "9c1d77e7",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-07-16 12:11:52.878524: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
      "To enable the following instructions: AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2023-07-16 12:11:53.550658: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Num GPUs Available:  1\n",
      "/mnt/e/ML/cafa-5-protein-function-prediction\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-07-16 12:11:54.962736: I tensorflow/compiler/xla/stream_executor/cuda/cuda_gpu_executor.cc:982] could not open file to read NUMA node: /sys/bus/pci/devices/0000:2b:00.0/numa_node\n",
      "Your kernel may have been built without NUMA support.\n",
      "2023-07-16 12:11:55.048745: I tensorflow/compiler/xla/stream_executor/cuda/cuda_gpu_executor.cc:982] could not open file to read NUMA node: /sys/bus/pci/devices/0000:2b:00.0/numa_node\n",
      "Your kernel may have been built without NUMA support.\n",
      "2023-07-16 12:11:55.049060: I tensorflow/compiler/xla/stream_executor/cuda/cuda_gpu_executor.cc:982] could not open file to read NUMA node: /sys/bus/pci/devices/0000:2b:00.0/numa_node\n",
      "Your kernel may have been built without NUMA support.\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "from dotenv import load_dotenv\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import pandas as pd\n",
    "import datetime\n",
    "\n",
    "physical_devices = tf.config.list_physical_devices('GPU')\n",
    "print(\"Num GPUs Available: \", len(physical_devices))\n",
    "# try:\n",
    "#   tf.config.experimental.set_memory_growth(physical_devices[0], True)\n",
    "# except:\n",
    "#   # Invalid device or cannot modify virtual devices once initialized.\n",
    "#   pass\n",
    "\n",
    "load_dotenv(override=True)\n",
    "\n",
    "DATA_PATH = os.getenv('DATA_PATH')\n",
    "print(DATA_PATH)\n",
    "\n",
    "# Choose subontology (CCO, MFO or BPO)\n",
    "SO = 'BPO'"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "e3544f8a",
   "metadata": {},
   "source": [
    "## Reading fasta, obo and tsv files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "c8b515f2",
   "metadata": {},
   "outputs": [],
   "source": [
    "from Bio import SeqIO\n",
    "\n",
    "sequences = [rec.seq for rec in SeqIO.parse(os.path.join(DATA_PATH, \"Train/train_sequences.fasta\"),\"fasta\")]\n",
    "ids = [rec.id for rec in SeqIO.parse(os.path.join(DATA_PATH, \"Train/train_sequences.fasta\"),\"fasta\")]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "2898414e",
   "metadata": {},
   "outputs": [],
   "source": [
    "import networkx\n",
    "import obonet\n",
    "\n",
    "# Read the taxrank ontology\n",
    "url = os.path.join(DATA_PATH, \"Train/go-basic.obo\")\n",
    "graph = obonet.read_obo(url)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "9f4bf949",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(3497732, 3)\n"
     ]
    }
   ],
   "source": [
    "df = pd.read_csv(os.path.join(DATA_PATH, \"Train/train_terms.tsv\"), sep='\\t')\n",
    "\n",
    "dfSO = df.loc[df[\"aspect\"]==SO]\n",
    "uniqueTerms = dfSO[\"term\"].unique()\n",
    "termsArr = list(dfSO[\"term\"].to_numpy())\n",
    "\n",
    "uniqueTermsDict={}\n",
    "for i,el in enumerate(uniqueTerms):\n",
    "    uniqueTermsDict[el] = i\n",
    "    \n",
    "print(dfSO.shape)\n",
    "df=dfSO\n",
    "\n",
    "df.set_index(\"EntryID\", inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "e4dec705",
   "metadata": {},
   "outputs": [],
   "source": [
    "testID = df.index.to_list()[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "7fff067f",
   "metadata": {},
   "outputs": [],
   "source": [
    "dfGo = pd.read_csv(os.path.join(DATA_PATH, \"Train/train_terms.tsv\"), sep='\\t')\n",
    "\n",
    "dfGo = dfGo.loc[dfGo[\"aspect\"]==SO]\n",
    "\n",
    "dfGo.set_index(\"term\", inplace=True)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "5338b7f8",
   "metadata": {},
   "source": [
    "## GO analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "1058ef1d",
   "metadata": {},
   "outputs": [],
   "source": [
    "item_counts = df[\"term\"].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "eb35584f",
   "metadata": {},
   "outputs": [],
   "source": [
    "id_to_name = {id_: data.get('name') for id_, data in graph.nodes(data=True)}\n",
    "name_to_id = {data['name']: id_ for id_, data in graph.nodes(data=True) if 'name' in data}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "7651a3ec",
   "metadata": {},
   "outputs": [],
   "source": [
    "relevantNodes=[]\n",
    "for layer, nodes in enumerate(networkx.topological_generations(graph)):\n",
    "    if layer<10:\n",
    "        relevantNodes = relevantNodes + nodes"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "3c099fc2",
   "metadata": {},
   "source": [
    "## Label encoding"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "cae9a3b0",
   "metadata": {},
   "source": [
    "The task is a multilabel classification: The output has several possible targets (Gene Ontologies) but each can only be 1 (existing) or 0 (non existing)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "c8e63ed9",
   "metadata": {},
   "source": [
    "Extract label weights from IA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "c7e3c316",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Not found GOs: 0 (set to 0)\n"
     ]
    }
   ],
   "source": [
    "dfIa = pd.read_csv(os.path.join(DATA_PATH, \"IA.txt\"), sep='\\t', header=None)\n",
    "\n",
    "dfIa.set_index(0, inplace=True)\n",
    "\n",
    "labelWeights=[]\n",
    "allIndices = dfIa.index.tolist()\n",
    "\n",
    "\n",
    "\n",
    "notFound=0\n",
    "for go in item_counts.index.to_list():\n",
    "    if go in allIndices:\n",
    "        labelWeights.append(dfIa.loc[go].to_numpy()[0])\n",
    "    else:\n",
    "        notFound += 1\n",
    "        labelWeights.append(0)\n",
    "\n",
    "print(\"Not found GOs: {} (set to 0)\".format(notFound))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "caeee590",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading presaved data\n"
     ]
    }
   ],
   "source": [
    "topGOs=item_counts.index.to_list()\n",
    "\n",
    "threshold=0\n",
    "labelWeights=np.array(labelWeights)\n",
    "selection = labelWeights>threshold\n",
    "topGOs=np.array(topGOs)[selection]\n",
    "\n",
    "if os.path.exists(os.path.join(DATA_PATH, \"GODataSizes_\"+SO+\".npy\")):\n",
    "    print(\"Loading presaved data\")\n",
    "    GODataSizes = np.load(os.path.join(DATA_PATH, \"GODataSizes_\"+SO+\".npy\"))\n",
    "else:\n",
    "    GODataSizes= [dfGo.loc[g].size for g in topGOs]\n",
    "    np.save(os.path.join(DATA_PATH, \"GODataSizes_\"+SO), GODataSizes)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "3e87f301",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8241\n"
     ]
    }
   ],
   "source": [
    "#At least 10 samples\n",
    "print(np.count_nonzero(np.array(GODataSizes)>20))\n",
    "GODataSizes= np.array(GODataSizes)\n",
    "GOsWithSufficientData = topGOs[GODataSizes>20]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "61e2ef95",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7966\n"
     ]
    }
   ],
   "source": [
    "#Reduce to relevant GOs (see Data Exploration.ipynb)\n",
    "\n",
    "finalGOSet =[]\n",
    "for g in GOsWithSufficientData:\n",
    "    if g in relevantNodes:\n",
    "        finalGOSet.append(g)\n",
    "\n",
    "print(len(finalGOSet))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "99f6a6ca",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['GO:0008152' 'GO:0034655' 'GO:0072523' 'GO:0044270' 'GO:0006753'\n",
      " 'GO:1901292' 'GO:0044237' 'GO:1901360' 'GO:0008150' 'GO:1901564'\n",
      " 'GO:1901565' 'GO:0009117' 'GO:0006139' 'GO:0044281' 'GO:0046496'\n",
      " 'GO:0019362' 'GO:0046483' 'GO:0055086' 'GO:0044248' 'GO:0019439'\n",
      " 'GO:0019637' 'GO:0006807' 'GO:0019677' 'GO:1901361' 'GO:0006163'\n",
      " 'GO:0046700' 'GO:0009987' 'GO:0006725' 'GO:0006796' 'GO:0034641'\n",
      " 'GO:0072521' 'GO:0071704' 'GO:0019364' 'GO:1901575' 'GO:0072526'\n",
      " 'GO:0046434' 'GO:0009166' 'GO:0072524' 'GO:0006195' 'GO:0009056'\n",
      " 'GO:0044238' 'GO:0006793' 'GO:0019674']\n",
      "[[0 0 0 ... 0 0 0]]\n",
      "7966\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/manuel/miniconda3/envs/tf/lib/python3.9/site-packages/sklearn/preprocessing/_label.py:895: UserWarning: unknown class(es) ['GO:0006139', 'GO:0006195', 'GO:0006725', 'GO:0006753', 'GO:0006793', 'GO:0006796', 'GO:0006807', 'GO:0008150', 'GO:0008152', 'GO:0009056', 'GO:0009987', 'GO:0019364', 'GO:0019439', 'GO:0019637', 'GO:0034641', 'GO:0044237', 'GO:0044238', 'GO:0044248', 'GO:0044270', 'GO:0044281', 'GO:0046483', 'GO:0046700', 'GO:0055086', 'GO:0071704', 'GO:1901292', 'GO:1901360', 'GO:1901361', 'GO:1901564', 'GO:1901575'] will be ignored\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "from sklearn.preprocessing import MultiLabelBinarizer\n",
    "import pickle\n",
    "\n",
    "\n",
    "mlb = MultiLabelBinarizer()\n",
    "mlb.fit([finalGOSet])\n",
    "\n",
    "dftest=df.loc[testID]\n",
    "indices = dftest[\"term\"].to_numpy()\n",
    "print(indices)\n",
    "print(mlb.transform([indices]))\n",
    "print(len(mlb.classes_))\n",
    "\n",
    "with open(os.path.join(DATA_PATH,'MLB_'+SO+'.pkl'), 'wb') as f:\n",
    "    pickle.dump(mlb, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "fb189ff8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Not found GOs: 0 (set to 0)\n"
     ]
    }
   ],
   "source": [
    "labelWeightsCorr=[]\n",
    "\n",
    "notFound=0\n",
    "for go in mlb.classes_:\n",
    "    if go in allIndices:\n",
    "        labelWeightsCorr.append(dfIa.loc[go].to_numpy()[0])\n",
    "    else:\n",
    "        notFound += 1\n",
    "        labelWeightsCorr.append(0)\n",
    "\n",
    "print(\"Not found GOs: {} (set to 0)\".format(notFound))\n",
    "labelWeightsCorr=np.array(labelWeightsCorr)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "92019f8c",
   "metadata": {},
   "source": [
    "## Amino acids encoding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "6e42462e",
   "metadata": {},
   "outputs": [],
   "source": [
    "aa_dict = {'A': 1, 'B':24, 'C': 2, 'D': 3, 'E': 4, 'F': 5, 'G': 6, 'H': 7, 'I': 8, 'K': 9, 'L': 10, 'M': 11, 'N': 12, 'O': 21, 'P': 13, 'Q': 14, 'R': 15, 'S': 16, 'T': 17, 'U': 22, 'V': 18, 'W': 19, 'Y': 20, 'X':30, 'Z':23}"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "bfd823ba",
   "metadata": {},
   "source": [
    "## Build Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "431f9df8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The max. length of the sequences is 35375\n"
     ]
    }
   ],
   "source": [
    "seqLengths = [len(seq) for seq in sequences]\n",
    "maxLen = max(seqLengths)\n",
    "print(\"The max. length of the sequences is {}\".format(maxLen))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "e581e06c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['A0A009IHW8' 'A0A021WW32' 'A0A023FFD0' ... 'X5L1L5' 'X5L565' 'X5M5N0']\n"
     ]
    }
   ],
   "source": [
    "\n",
    "dfAll=pd.read_csv(os.path.join(DATA_PATH, \"Train/train_terms.tsv\"), sep='\\t')\n",
    "\n",
    "soEntries = dfAll.loc[dfAll[\"aspect\"]==SO]\n",
    "soEntryIds = soEntries[\"EntryID\"].unique()\n",
    "\n",
    "# print(len(seqEntries))\n",
    "print(soEntryIds)\n",
    "\n",
    "# SoSequences = []\n",
    "# for entry in soEntryIds:\n",
    "#     SoSequences.append(sequences[ids.index(entry)])\n",
    "\n",
    "# print(len(SoSequences))\n",
    "dfAll.set_index(\"EntryID\", inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "44f722e8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "99572\n"
     ]
    }
   ],
   "source": [
    "import warnings\n",
    "from itertools import product\n",
    "from tqdm import tqdm\n",
    "import os\n",
    "\n",
    "TRAIN_VAL_SPLIT = 0.7\n",
    "TRUNC = 2000\n",
    "\n",
    "\n",
    "#Use numpy vectorize to speed up the mapping (hopefully)\n",
    "mapping = lambda x: aa_dict[x]\n",
    "vectMapping = np.vectorize(mapping)\n",
    "\n",
    "# Shuffle the data\n",
    "import random\n",
    "random.seed(516213)\n",
    "c = list(zip(sequences, ids))\n",
    "random.shuffle(c)\n",
    "sequencesShuffle, idsShuffle = zip(*c)\n",
    "\n",
    "\n",
    "#Train Validation Split\n",
    "split = int(np.floor(len(sequencesShuffle)*TRAIN_VAL_SPLIT))\n",
    "print(split)\n",
    "trainSeq = sequencesShuffle[0:split]\n",
    "valSeq = sequencesShuffle[split+1:]\n",
    "trainIds = idsShuffle[0:split]\n",
    "valIds = idsShuffle[split+1:]\n",
    "\n",
    "\n",
    "def generator():\n",
    "  for i,seq in enumerate(trainSeq):\n",
    "      entryId = trainIds[i]\n",
    "      if entryId in soEntryIds:\n",
    "        labelData = df.loc[entryId]\n",
    "        # indices = labelData[\"termToken\"].to_numpy()\n",
    "        indices = labelData[\"term\"].to_numpy()\n",
    "      else: \n",
    "        indices=[]\n",
    "\n",
    "      with warnings.catch_warnings():\n",
    "          #supress the warnings for unknown classes\n",
    "          warnings.simplefilter(\"ignore\")\n",
    "          y = mlb.transform([indices])\n",
    "\n",
    "\n",
    "      intSeq = [vectMapping(s) for s in seq]\n",
    "\n",
    "      if(len(intSeq)>TRUNC):\n",
    "        intSeq = intSeq[0:TRUNC]\n",
    "      elif len(intSeq)<TRUNC:\n",
    "        intSeq = np.pad(intSeq, (0,TRUNC-len(intSeq)))\n",
    "\n",
    "      \n",
    "         \n",
    "      yield (tf.one_hot(intSeq, len(aa_dict)),y[0])\n",
    "\n",
    "\n",
    "def generatorVal():\n",
    "  for i,seq in enumerate(valSeq):\n",
    "      entryId = valIds[i]\n",
    "      if entryId in soEntryIds:\n",
    "        labelData = df.loc[entryId]\n",
    "        # indices = labelData[\"termToken\"].to_numpy()\n",
    "        indices = labelData[\"term\"].to_numpy()\n",
    "      else:\n",
    "        indices=[]\n",
    "\n",
    "      with warnings.catch_warnings():\n",
    "          #supress the warnings for unknown classes\n",
    "          warnings.simplefilter(\"ignore\")\n",
    "          y = mlb.transform([indices])\n",
    "\n",
    "      intSeq = [vectMapping(s) for s in seq]\n",
    "\n",
    "      if(len(intSeq)>TRUNC):\n",
    "        intSeq = intSeq[0:TRUNC]\n",
    "      elif len(intSeq)<TRUNC:\n",
    "        intSeq = np.pad(intSeq, (0,TRUNC-len(intSeq)))\n",
    "         \n",
    "      yield (tf.one_hot(intSeq, len(aa_dict)),y[0])\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "2727338a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The first sample: \n",
      "(2000, 25)\n",
      "[[0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      "  0.]]\n",
      "The first sample has 0 classes\n"
     ]
    }
   ],
   "source": [
    "g = generator()\n",
    "test = next(g)\n",
    "print(\"The first sample: \\n{}\\n{}\".format(test[0].shape, test[0][0:1]))\n",
    "print(\"The first sample has {} classes\".format(np.count_nonzero(test[1])))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "10c4a51a",
   "metadata": {},
   "source": [
    "## Tensorflow Classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "63f0d2e0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(<tf.Tensor: shape=(2000, 25), dtype=int32, numpy=\n",
      "array([[0, 0, 0, ..., 0, 0, 0],\n",
      "       [0, 0, 0, ..., 0, 0, 0],\n",
      "       [0, 0, 0, ..., 0, 0, 0],\n",
      "       ...,\n",
      "       [1, 0, 0, ..., 0, 0, 0],\n",
      "       [1, 0, 0, ..., 0, 0, 0],\n",
      "       [1, 0, 0, ..., 0, 0, 0]], dtype=int32)>, <tf.Tensor: shape=(7966,), dtype=int32, numpy=array([0, 0, 0, ..., 0, 0, 0], dtype=int32)>)]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-07-16 12:24:38.578462: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'Placeholder/_0' with dtype int32\n",
      "\t [[{{node Placeholder/_0}}]]\n"
     ]
    }
   ],
   "source": [
    "\n",
    "\n",
    "dataset = tf.data.Dataset.from_generator(generator, output_signature=(\n",
    "         tf.TensorSpec(shape=(TRUNC,len(aa_dict)), dtype=tf.int32),\n",
    "         tf.TensorSpec(shape=(len(mlb.classes_),), dtype=tf.int32)))\n",
    "print(list(dataset.take(1)))\n",
    "\n",
    "datasetVal = tf.data.Dataset.from_generator(generatorVal, output_signature=(\n",
    "         tf.TensorSpec(shape=(TRUNC,len(aa_dict)), dtype=tf.int32),\n",
    "         tf.TensorSpec(shape=(len(mlb.classes_),), dtype=tf.int32)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "f0a554f0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The tensorboard extension is already loaded. To reload it, use:\n",
      "  %reload_ext tensorboard\n"
     ]
    }
   ],
   "source": [
    "%load_ext tensorboard\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "98752d7e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"OHEConvModel\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " input_7 (InputLayer)        [(None, 2000, 25)]        0         \n",
      "                                                                 \n",
      " conv1d_6 (Conv1D)           (None, 1994, 16)          2816      \n",
      "                                                                 \n",
      " leaky_re_lu_17 (LeakyReLU)  (None, 1994, 16)          0         \n",
      "                                                                 \n",
      " batch_normalization_14 (Bat  (None, 1994, 16)         64        \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " conv1d_7 (Conv1D)           (None, 1988, 16)          1808      \n",
      "                                                                 \n",
      " leaky_re_lu_18 (LeakyReLU)  (None, 1988, 16)          0         \n",
      "                                                                 \n",
      " batch_normalization_15 (Bat  (None, 1988, 16)         64        \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " conv1d_8 (Conv1D)           (None, 1982, 16)          1808      \n",
      "                                                                 \n",
      " leaky_re_lu_19 (LeakyReLU)  (None, 1982, 16)          0         \n",
      "                                                                 \n",
      " batch_normalization_16 (Bat  (None, 1982, 16)         64        \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " conv1d_9 (Conv1D)           (None, 1976, 32)          3616      \n",
      "                                                                 \n",
      " leaky_re_lu_20 (LeakyReLU)  (None, 1976, 32)          0         \n",
      "                                                                 \n",
      " batch_normalization_17 (Bat  (None, 1976, 32)         128       \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " conv1d_10 (Conv1D)          (None, 985, 32)           7200      \n",
      "                                                                 \n",
      " leaky_re_lu_21 (LeakyReLU)  (None, 985, 32)           0         \n",
      "                                                                 \n",
      " batch_normalization_18 (Bat  (None, 985, 32)          128       \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " conv1d_11 (Conv1D)          (None, 490, 32)           7200      \n",
      "                                                                 \n",
      " leaky_re_lu_22 (LeakyReLU)  (None, 490, 32)           0         \n",
      "                                                                 \n",
      " batch_normalization_19 (Bat  (None, 490, 32)          128       \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " flatten_1 (Flatten)         (None, 15680)             0         \n",
      "                                                                 \n",
      " dense_4 (Dense)             (None, 64)                1003584   \n",
      "                                                                 \n",
      " leaky_re_lu_23 (LeakyReLU)  (None, 64)                0         \n",
      "                                                                 \n",
      " dense_5 (Dense)             (None, 64)                4160      \n",
      "                                                                 \n",
      " leaky_re_lu_24 (LeakyReLU)  (None, 64)                0         \n",
      "                                                                 \n",
      " dense_6 (Dense)             (None, 64)                4160      \n",
      "                                                                 \n",
      " leaky_re_lu_25 (LeakyReLU)  (None, 64)                0         \n",
      "                                                                 \n",
      " dense_7 (Dense)             (None, 7966)              517790    \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 1,554,718\n",
      "Trainable params: 1,554,430\n",
      "Non-trainable params: 288\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.keras import layers\n",
    "\n",
    "VOCAB_SIZE=len(aa_dict)\n",
    "EMBED_DIM=10\n",
    "\n",
    "def createModel():\n",
    "    inputs = tf.keras.Input(shape=(TRUNC,VOCAB_SIZE))\n",
    "    # x=layers.Embedding(VOCAB_SIZE, EMBED_DIM, name=\"embedding\")(inputs)\n",
    "    # x = layers.BatchNormalization()(tf.expand_dims(inputs,2))\n",
    "\n",
    "\n",
    "    x=layers.Conv1D(16, 7)(inputs)\n",
    "    x=layers.LeakyReLU()(x)\n",
    "    x = layers.BatchNormalization()(x)\n",
    "\n",
    "    x=layers.Conv1D(16, 7)(x)\n",
    "    x=layers.LeakyReLU()(x)\n",
    "    x = layers.BatchNormalization()(x)\n",
    "    \n",
    "    x=layers.Conv1D(16, 7)(x)\n",
    "    x=layers.LeakyReLU()(x)\n",
    "    x = layers.BatchNormalization()(x)\n",
    "\n",
    "    x=layers.Conv1D(32, 7)(x)\n",
    "    x=layers.LeakyReLU()(x)\n",
    "    x = layers.BatchNormalization()(x)\n",
    "\n",
    "    x=layers.Conv1D(32, 7,strides=2)(x)\n",
    "    x=layers.LeakyReLU()(x)\n",
    "    x = layers.BatchNormalization()(x)\n",
    "\n",
    "    x=layers.Conv1D(32, 7,strides=2)(x)\n",
    "    x=layers.LeakyReLU()(x)\n",
    "    x = layers.BatchNormalization()(x)\n",
    "    \n",
    "    \n",
    "    \n",
    "\n",
    "    # x=layers.Conv1D(32, 5, activation=tf.keras.activations.relu)(x)\n",
    "    # x=layers.Conv1D(32, 5, activation=tf.keras.activations.relu)(x)\n",
    "    # x=layers.Conv1D(32, 5, activation=tf.keras.activations.relu)(x)\n",
    "    x=layers.Flatten()(x)\n",
    "    x=layers.Dense(64)(x)\n",
    "    x=layers.LeakyReLU()(x)\n",
    "    x=layers.Dense(64)(x)\n",
    "    x=layers.LeakyReLU()(x)\n",
    "    x=layers.Dense(64)(x)\n",
    "    x=layers.LeakyReLU()(x)\n",
    "    outputs=layers.Dense(len(mlb.classes_), activation=tf.keras.activations.sigmoid)(x)\n",
    "\n",
    "    return tf.keras.Model(inputs=inputs, outputs=outputs, name=\"OHEConvModel\")\n",
    "\n",
    "model = createModel()\n",
    "\n",
    "model.summary()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "5a3ddc20",
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "#Learning rate schedule\n",
    "initial_learning_rate = 0.001\n",
    "decaySteps=5000\n",
    "lr_schedule = tf.keras.optimizers.schedules.CosineDecayRestarts(initial_learning_rate, first_decay_steps=decaySteps,\n",
    "                                                                t_mul=2.0, m_mul=0.7)\n",
    "# lr_schedule = tf.keras.optimizers.schedules.CosineDecay(\n",
    "#     initial_learning_rate, decay_steps=decaySteps, alpha=0.01)\n",
    "# lr_schedule = tf.keras.optimizers.schedules.ExponentialDecay(\n",
    "#     initial_learning_rate,decay_steps=decaySteps,decay_rate=0.9,staircase=False)\n",
    "step = np.linspace(0,decaySteps*3)\n",
    "lr = lr_schedule(step)\n",
    "# plt.figure(figsize = (8,6))\n",
    "# # plt.yscale(\"log\")\n",
    "# plt.plot(step, lr)\n",
    "# plt.ylim([0,max(plt.ylim())])\n",
    "# plt.xlabel('step')\n",
    "# _ = plt.ylabel('Learning Rate')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "d2f43e7c",
   "metadata": {},
   "outputs": [],
   "source": [
    "from utils import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "9113c4a7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Start of epoch 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-07-16 12:29:01.517732: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'Placeholder/_0' with dtype int32\n",
      "\t [[{{node Placeholder/_0}}]]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/Step 0, Loss: 0.00017, Accuracy: 0.51286, F1: 0.0016, Prec: 0.3162, Rec: 0.0008 lr: 0.00010\n",
      "Epoch 1/Step 20, Loss: 0.00026, Accuracy: 0.52397, F1: 0.0020, Prec: 0.3301, Rec: 0.0010 lr: 0.00010\n",
      "Epoch 1/Step 40, Loss: -0.00029, Accuracy: 0.54711, F1: 0.0021, Prec: 0.3359, Rec: 0.0010 lr: 0.00010\n",
      "Epoch 1/Step 60, Loss: -0.00125, Accuracy: 0.59413, F1: 0.0023, Prec: 0.3340, Rec: 0.0012 lr: 0.00010\n",
      "Epoch 1/Step 80, Loss: -0.00358, Accuracy: 0.66002, F1: 0.0033, Prec: 0.3231, Rec: 0.0017 lr: 0.00010\n",
      "Epoch 1/Step 100, Loss: -0.00620, Accuracy: 0.71719, F1: 0.0058, Prec: 0.3118, Rec: 0.0030 lr: 0.00010\n",
      "Epoch 1/Step 120, Loss: -0.00912, Accuracy: 0.75998, F1: 0.0093, Prec: 0.2998, Rec: 0.0048 lr: 0.00010\n",
      "Epoch 1/Step 140, Loss: -0.02516, Accuracy: 0.79201, F1: 0.0127, Prec: 0.2836, Rec: 0.0068 lr: 0.00010\n",
      "Epoch 1/Step 160, Loss: -0.01625, Accuracy: 0.81671, F1: 0.0157, Prec: 0.2666, Rec: 0.0086 lr: 0.00010\n",
      "Epoch 1/Step 180, Loss: -0.02964, Accuracy: 0.83620, F1: 0.0182, Prec: 0.2502, Rec: 0.0102 lr: 0.00010\n",
      "Epoch 1/Step 200, Loss: -0.01396, Accuracy: 0.85191, F1: 0.0207, Prec: 0.2354, Rec: 0.0119 lr: 0.00010\n",
      "Epoch 1/Step 220, Loss: -0.02076, Accuracy: 0.86487, F1: 0.0229, Prec: 0.2232, Rec: 0.0135 lr: 0.00010\n",
      "Epoch 1/Step 240, Loss: -0.03270, Accuracy: 0.87573, F1: 0.0248, Prec: 0.2118, Rec: 0.0151 lr: 0.00010\n",
      "Epoch 1/Step 260, Loss: -0.03089, Accuracy: 0.88494, F1: 0.0263, Prec: 0.2021, Rec: 0.0163 lr: 0.00010\n",
      "Epoch 1/Step 280, Loss: -0.02472, Accuracy: 0.89285, F1: 0.0281, Prec: 0.1941, Rec: 0.0177 lr: 0.00010\n",
      "Epoch 1/Step 300, Loss: -0.03293, Accuracy: 0.89973, F1: 0.0294, Prec: 0.1869, Rec: 0.0188 lr: 0.00010\n",
      "Epoch 1/Step 320, Loss: -0.02729, Accuracy: 0.90577, F1: 0.0305, Prec: 0.1799, Rec: 0.0198 lr: 0.00010\n",
      "Epoch 1/Step 340, Loss: -0.03017, Accuracy: 0.91111, F1: 0.0319, Prec: 0.1745, Rec: 0.0210 lr: 0.00010\n",
      "Epoch 1/Step 360, Loss: -0.03374, Accuracy: 0.91586, F1: 0.0328, Prec: 0.1689, Rec: 0.0219 lr: 0.00010\n",
      "Epoch 1/Step 380, Loss: -0.03643, Accuracy: 0.92013, F1: 0.0336, Prec: 0.1640, Rec: 0.0227 lr: 0.00010\n",
      "Epoch 1/Step 400, Loss: -0.01239, Accuracy: 0.92397, F1: 0.0343, Prec: 0.1589, Rec: 0.0235 lr: 0.00010\n",
      "Epoch 1/Step 420, Loss: -0.03183, Accuracy: 0.92746, F1: 0.0349, Prec: 0.1544, Rec: 0.0242 lr: 0.00010\n",
      "Epoch 1/Step 440, Loss: -0.02170, Accuracy: 0.93062, F1: 0.0354, Prec: 0.1503, Rec: 0.0248 lr: 0.00010\n",
      "Epoch 1/Step 460, Loss: -0.02576, Accuracy: 0.93351, F1: 0.0358, Prec: 0.1464, Rec: 0.0254 lr: 0.00010\n",
      "Epoch 1/Step 480, Loss: -0.03709, Accuracy: 0.93617, F1: 0.0363, Prec: 0.1428, Rec: 0.0260 lr: 0.00010\n",
      "Epoch 1/Step 500, Loss: -0.02815, Accuracy: 0.93862, F1: 0.0366, Prec: 0.1392, Rec: 0.0265 lr: 0.00010\n",
      "Epoch 1/Step 520, Loss: -0.02554, Accuracy: 0.94087, F1: 0.0369, Prec: 0.1362, Rec: 0.0270 lr: 0.00010\n",
      "Epoch 1/Step 540, Loss: -0.03810, Accuracy: 0.94296, F1: 0.0373, Prec: 0.1332, Rec: 0.0275 lr: 0.00010\n",
      "Epoch 1/Step 560, Loss: -0.04441, Accuracy: 0.94491, F1: 0.0377, Prec: 0.1306, Rec: 0.0280 lr: 0.00010\n",
      "Epoch 1/Step 580, Loss: -0.05557, Accuracy: 0.94673, F1: 0.0379, Prec: 0.1278, Rec: 0.0285 lr: 0.00010\n",
      "Epoch 1/Step 600, Loss: -0.02281, Accuracy: 0.94843, F1: 0.0381, Prec: 0.1250, Rec: 0.0290 lr: 0.00010\n",
      "Epoch 1/Step 620, Loss: -0.04259, Accuracy: 0.95003, F1: 0.0383, Prec: 0.1224, Rec: 0.0296 lr: 0.00010\n",
      "Epoch 1/Step 640, Loss: -0.05446, Accuracy: 0.95153, F1: 0.0385, Prec: 0.1199, Rec: 0.0303 lr: 0.00010\n",
      "Epoch 1/Step 660, Loss: -0.03654, Accuracy: 0.95294, F1: 0.0388, Prec: 0.1177, Rec: 0.0309 lr: 0.00010\n",
      "Epoch 1/Step 680, Loss: -0.03308, Accuracy: 0.95427, F1: 0.0389, Prec: 0.1155, Rec: 0.0313 lr: 0.00010\n",
      "Epoch 1/Step 700, Loss: -0.01388, Accuracy: 0.95551, F1: 0.0391, Prec: 0.1135, Rec: 0.0318 lr: 0.00010\n",
      "Epoch 1/Step 720, Loss: -0.05456, Accuracy: 0.95670, F1: 0.0393, Prec: 0.1116, Rec: 0.0323 lr: 0.00010\n",
      "Epoch 1/Step 740, Loss: -0.02952, Accuracy: 0.95781, F1: 0.0394, Prec: 0.1098, Rec: 0.0327 lr: 0.00010\n",
      "Epoch 1/Step 760, Loss: -0.03895, Accuracy: 0.95887, F1: 0.0396, Prec: 0.1081, Rec: 0.0332 lr: 0.00010\n",
      "Epoch 1/Step 780, Loss: -0.04008, Accuracy: 0.95988, F1: 0.0398, Prec: 0.1066, Rec: 0.0336 lr: 0.00010\n",
      "Epoch 1/Step 800, Loss: -0.04473, Accuracy: 0.96083, F1: 0.0400, Prec: 0.1052, Rec: 0.0340 lr: 0.00010\n",
      "Epoch 1/Step 820, Loss: -0.03289, Accuracy: 0.96174, F1: 0.0402, Prec: 0.1037, Rec: 0.0344 lr: 0.00010\n",
      "Epoch 1/Step 840, Loss: -0.02889, Accuracy: 0.96260, F1: 0.0402, Prec: 0.1023, Rec: 0.0347 lr: 0.00010\n",
      "Epoch 1/Step 860, Loss: -0.03365, Accuracy: 0.96343, F1: 0.0403, Prec: 0.1008, Rec: 0.0350 lr: 0.00010\n",
      "Epoch 1/Step 880, Loss: -0.04739, Accuracy: 0.96421, F1: 0.0405, Prec: 0.0996, Rec: 0.0354 lr: 0.00010\n",
      "Epoch 1/Step 900, Loss: -0.05254, Accuracy: 0.96497, F1: 0.0406, Prec: 0.0984, Rec: 0.0356 lr: 0.00010\n",
      "Epoch 1/Step 920, Loss: -0.04920, Accuracy: 0.96569, F1: 0.0406, Prec: 0.0972, Rec: 0.0359 lr: 0.00010\n",
      "Epoch 1/Step 940, Loss: -0.02679, Accuracy: 0.96638, F1: 0.0407, Prec: 0.0960, Rec: 0.0362 lr: 0.00010\n",
      "Epoch 1/Step 960, Loss: -0.04295, Accuracy: 0.96704, F1: 0.0407, Prec: 0.0949, Rec: 0.0364 lr: 0.00010\n",
      "Epoch 1/Step 980, Loss: -0.06252, Accuracy: 0.96768, F1: 0.0408, Prec: 0.0938, Rec: 0.0368 lr: 0.00010\n",
      "Epoch 1/Step 1000, Loss: -0.03940, Accuracy: 0.96829, F1: 0.0409, Prec: 0.0928, Rec: 0.0371 lr: 0.00010\n",
      "Epoch 1/Step 1020, Loss: -0.03179, Accuracy: 0.96887, F1: 0.0411, Prec: 0.0919, Rec: 0.0373 lr: 0.00010\n",
      "Epoch 1/Step 1040, Loss: -0.04602, Accuracy: 0.96943, F1: 0.0411, Prec: 0.0910, Rec: 0.0375 lr: 0.00010\n",
      "Epoch 1/Step 1060, Loss: -0.04863, Accuracy: 0.96998, F1: 0.0412, Prec: 0.0900, Rec: 0.0377 lr: 0.00010\n",
      "Epoch 1/Step 1080, Loss: -0.05224, Accuracy: 0.97050, F1: 0.0413, Prec: 0.0892, Rec: 0.0381 lr: 0.00010\n",
      "Epoch 1/Step 1100, Loss: -0.06200, Accuracy: 0.97100, F1: 0.0415, Prec: 0.0884, Rec: 0.0385 lr: 0.00010\n",
      "Epoch 1/Step 1120, Loss: -0.03325, Accuracy: 0.97149, F1: 0.0415, Prec: 0.0876, Rec: 0.0387 lr: 0.00010\n",
      "Epoch 1/Step 1140, Loss: -0.03289, Accuracy: 0.97196, F1: 0.0416, Prec: 0.0868, Rec: 0.0389 lr: 0.00010\n",
      "Epoch 1/Step 1160, Loss: -0.03344, Accuracy: 0.97241, F1: 0.0417, Prec: 0.0862, Rec: 0.0391 lr: 0.00010\n",
      "Epoch 1/Step 1180, Loss: -0.04222, Accuracy: 0.97285, F1: 0.0420, Prec: 0.0856, Rec: 0.0395 lr: 0.00010\n",
      "Epoch 1/Step 1200, Loss: -0.02654, Accuracy: 0.97327, F1: 0.0420, Prec: 0.0849, Rec: 0.0397 lr: 0.00010\n",
      "Epoch 1/Step 1220, Loss: -0.02881, Accuracy: 0.97368, F1: 0.0420, Prec: 0.0842, Rec: 0.0398 lr: 0.00010\n",
      "Epoch 1/Step 1240, Loss: -0.06370, Accuracy: 0.97407, F1: 0.0421, Prec: 0.0834, Rec: 0.0401 lr: 0.00010\n",
      "Epoch 1/Step 1260, Loss: -0.05686, Accuracy: 0.97446, F1: 0.0422, Prec: 0.0829, Rec: 0.0403 lr: 0.00010\n",
      "Epoch 1/Step 1280, Loss: -0.05713, Accuracy: 0.97483, F1: 0.0422, Prec: 0.0823, Rec: 0.0404 lr: 0.00010\n",
      "Epoch 1/Step 1300, Loss: -0.04160, Accuracy: 0.97519, F1: 0.0423, Prec: 0.0817, Rec: 0.0407 lr: 0.00010\n",
      "Epoch 1/Step 1320, Loss: -0.03985, Accuracy: 0.97554, F1: 0.0424, Prec: 0.0812, Rec: 0.0408 lr: 0.00010\n",
      "Epoch 1/Step 1340, Loss: -0.02773, Accuracy: 0.97587, F1: 0.0425, Prec: 0.0807, Rec: 0.0411 lr: 0.00010\n",
      "Epoch 1/Step 1360, Loss: -0.03865, Accuracy: 0.97620, F1: 0.0426, Prec: 0.0802, Rec: 0.0412 lr: 0.00010\n",
      "Epoch 1/Step 1380, Loss: -0.04518, Accuracy: 0.97652, F1: 0.0427, Prec: 0.0797, Rec: 0.0415 lr: 0.00010\n",
      "Epoch 1/Step 1400, Loss: -0.05634, Accuracy: 0.97683, F1: 0.0427, Prec: 0.0791, Rec: 0.0416 lr: 0.00010\n",
      "Epoch 1/Step 1420, Loss: -0.03773, Accuracy: 0.97713, F1: 0.0428, Prec: 0.0787, Rec: 0.0418 lr: 0.00010\n",
      "Epoch 1/Step 1440, Loss: -0.03896, Accuracy: 0.97743, F1: 0.0429, Prec: 0.0782, Rec: 0.0421 lr: 0.00010\n",
      "Epoch 1/Step 1460, Loss: -0.03314, Accuracy: 0.97771, F1: 0.0429, Prec: 0.0776, Rec: 0.0421 lr: 0.00010\n",
      "Epoch 1/Step 1480, Loss: -0.06043, Accuracy: 0.97799, F1: 0.0429, Prec: 0.0771, Rec: 0.0422 lr: 0.00010\n",
      "Epoch 1/Step 1500, Loss: -0.04726, Accuracy: 0.97826, F1: 0.0429, Prec: 0.0766, Rec: 0.0425 lr: 0.00010\n",
      "Epoch 1/Step 1520, Loss: -0.06476, Accuracy: 0.97852, F1: 0.0430, Prec: 0.0762, Rec: 0.0426 lr: 0.00010\n",
      "Epoch 1/Step 1540, Loss: -0.01310, Accuracy: 0.97878, F1: 0.0430, Prec: 0.0757, Rec: 0.0428 lr: 0.00010\n",
      "Epoch finished. Start validation\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-07-16 12:37:08.031069: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'Placeholder/_0' with dtype int32\n",
      "\t [[{{node Placeholder/_0}}]]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation acc: 0.9984\n",
      "Validation f1: 0.0466\n",
      "Validation precision: 0.0426\n",
      "Validation recall: 0.0547\n",
      "\n",
      "Start of epoch 2\n",
      "Epoch 2/Step 0, Loss: -0.03157, Accuracy: 0.99830, F1: 0.0240, Prec: 0.0439, Rec: 0.0166 lr: 0.00010\n",
      "Epoch 2/Step 20, Loss: -0.03431, Accuracy: 0.99826, F1: 0.0479, Prec: 0.0487, Rec: 0.0499 lr: 0.00010\n",
      "Epoch 2/Step 40, Loss: -0.04947, Accuracy: 0.99828, F1: 0.0492, Prec: 0.0483, Rec: 0.0533 lr: 0.00010\n",
      "Epoch 2/Step 60, Loss: -0.04972, Accuracy: 0.99829, F1: 0.0488, Prec: 0.0473, Rec: 0.0537 lr: 0.00010\n",
      "Epoch 2/Step 80, Loss: -0.02605, Accuracy: 0.99832, F1: 0.0474, Prec: 0.0453, Rec: 0.0530 lr: 0.00010\n",
      "Epoch 2/Step 100, Loss: -0.03417, Accuracy: 0.99833, F1: 0.0471, Prec: 0.0445, Rec: 0.0532 lr: 0.00010\n",
      "Epoch 2/Step 120, Loss: -0.02723, Accuracy: 0.99832, F1: 0.0468, Prec: 0.0438, Rec: 0.0538 lr: 0.00010\n",
      "Epoch 2/Step 140, Loss: -0.05141, Accuracy: 0.99831, F1: 0.0468, Prec: 0.0438, Rec: 0.0539 lr: 0.00010\n",
      "Epoch 2/Step 160, Loss: -0.04211, Accuracy: 0.99830, F1: 0.0471, Prec: 0.0437, Rec: 0.0543 lr: 0.00010\n",
      "Epoch 2/Step 180, Loss: -0.04027, Accuracy: 0.99830, F1: 0.0468, Prec: 0.0434, Rec: 0.0540 lr: 0.00010\n",
      "Epoch 2/Step 200, Loss: -0.02551, Accuracy: 0.99829, F1: 0.0464, Prec: 0.0428, Rec: 0.0540 lr: 0.00010\n",
      "Epoch 2/Step 220, Loss: -0.03433, Accuracy: 0.99829, F1: 0.0466, Prec: 0.0431, Rec: 0.0540 lr: 0.00010\n",
      "Epoch 2/Step 240, Loss: -0.06595, Accuracy: 0.99828, F1: 0.0465, Prec: 0.0430, Rec: 0.0542 lr: 0.00010\n",
      "Epoch 2/Step 260, Loss: -0.04657, Accuracy: 0.99828, F1: 0.0464, Prec: 0.0428, Rec: 0.0540 lr: 0.00010\n",
      "Epoch 2/Step 280, Loss: -0.03622, Accuracy: 0.99828, F1: 0.0471, Prec: 0.0434, Rec: 0.0549 lr: 0.00010\n",
      "Epoch 2/Step 300, Loss: -0.05385, Accuracy: 0.99828, F1: 0.0473, Prec: 0.0437, Rec: 0.0549 lr: 0.00010\n",
      "Epoch 2/Step 320, Loss: -0.04843, Accuracy: 0.99828, F1: 0.0473, Prec: 0.0436, Rec: 0.0550 lr: 0.00010\n",
      "Epoch 2/Step 340, Loss: -0.05063, Accuracy: 0.99828, F1: 0.0477, Prec: 0.0440, Rec: 0.0554 lr: 0.00010\n",
      "Epoch 2/Step 360, Loss: -0.04836, Accuracy: 0.99829, F1: 0.0478, Prec: 0.0440, Rec: 0.0556 lr: 0.00010\n",
      "Epoch 2/Step 380, Loss: -0.05870, Accuracy: 0.99829, F1: 0.0479, Prec: 0.0441, Rec: 0.0556 lr: 0.00010\n",
      "Epoch 2/Step 400, Loss: -0.01302, Accuracy: 0.99830, F1: 0.0477, Prec: 0.0439, Rec: 0.0557 lr: 0.00010\n",
      "Epoch 2/Step 420, Loss: -0.04884, Accuracy: 0.99830, F1: 0.0476, Prec: 0.0438, Rec: 0.0556 lr: 0.00010\n",
      "Epoch 2/Step 440, Loss: -0.03045, Accuracy: 0.99831, F1: 0.0476, Prec: 0.0437, Rec: 0.0556 lr: 0.00010\n",
      "Epoch 2/Step 460, Loss: -0.04213, Accuracy: 0.99831, F1: 0.0474, Prec: 0.0436, Rec: 0.0554 lr: 0.00010\n",
      "Epoch 2/Step 480, Loss: -0.04796, Accuracy: 0.99831, F1: 0.0475, Prec: 0.0436, Rec: 0.0555 lr: 0.00010\n",
      "Epoch 2/Step 500, Loss: -0.04005, Accuracy: 0.99831, F1: 0.0472, Prec: 0.0434, Rec: 0.0552 lr: 0.00010\n",
      "Epoch 2/Step 520, Loss: -0.03078, Accuracy: 0.99831, F1: 0.0473, Prec: 0.0434, Rec: 0.0553 lr: 0.00010\n",
      "Epoch 2/Step 540, Loss: -0.04130, Accuracy: 0.99831, F1: 0.0472, Prec: 0.0433, Rec: 0.0554 lr: 0.00010\n",
      "Epoch 2/Step 560, Loss: -0.05131, Accuracy: 0.99831, F1: 0.0473, Prec: 0.0434, Rec: 0.0553 lr: 0.00010\n",
      "Epoch 2/Step 580, Loss: -0.06755, Accuracy: 0.99831, F1: 0.0473, Prec: 0.0434, Rec: 0.0554 lr: 0.00010\n",
      "Epoch 2/Step 600, Loss: -0.02894, Accuracy: 0.99831, F1: 0.0472, Prec: 0.0432, Rec: 0.0554 lr: 0.00010\n",
      "Epoch 2/Step 620, Loss: -0.05732, Accuracy: 0.99831, F1: 0.0473, Prec: 0.0434, Rec: 0.0555 lr: 0.00010\n",
      "Epoch 2/Step 640, Loss: -0.05890, Accuracy: 0.99831, F1: 0.0473, Prec: 0.0433, Rec: 0.0555 lr: 0.00010\n",
      "Epoch 2/Step 660, Loss: -0.04903, Accuracy: 0.99831, F1: 0.0474, Prec: 0.0434, Rec: 0.0555 lr: 0.00010\n",
      "Epoch 2/Step 680, Loss: -0.03946, Accuracy: 0.99831, F1: 0.0473, Prec: 0.0434, Rec: 0.0555 lr: 0.00010\n",
      "Epoch 2/Step 700, Loss: -0.01549, Accuracy: 0.99831, F1: 0.0473, Prec: 0.0433, Rec: 0.0555 lr: 0.00010\n",
      "Epoch 2/Step 720, Loss: -0.06189, Accuracy: 0.99831, F1: 0.0474, Prec: 0.0434, Rec: 0.0556 lr: 0.00010\n",
      "Epoch 2/Step 740, Loss: -0.03169, Accuracy: 0.99831, F1: 0.0473, Prec: 0.0434, Rec: 0.0556 lr: 0.00010\n",
      "Epoch 2/Step 760, Loss: -0.04296, Accuracy: 0.99831, F1: 0.0474, Prec: 0.0434, Rec: 0.0556 lr: 0.00010\n",
      "Epoch 2/Step 780, Loss: -0.04617, Accuracy: 0.99831, F1: 0.0474, Prec: 0.0435, Rec: 0.0557 lr: 0.00010\n",
      "Epoch 2/Step 800, Loss: -0.04273, Accuracy: 0.99831, F1: 0.0474, Prec: 0.0434, Rec: 0.0557 lr: 0.00010\n",
      "Epoch 2/Step 820, Loss: -0.03445, Accuracy: 0.99831, F1: 0.0474, Prec: 0.0434, Rec: 0.0557 lr: 0.00010\n",
      "Epoch 2/Step 840, Loss: -0.03052, Accuracy: 0.99831, F1: 0.0473, Prec: 0.0433, Rec: 0.0557 lr: 0.00010\n",
      "Epoch 2/Step 860, Loss: -0.03919, Accuracy: 0.99831, F1: 0.0472, Prec: 0.0432, Rec: 0.0557 lr: 0.00010\n",
      "Epoch 2/Step 880, Loss: -0.04912, Accuracy: 0.99831, F1: 0.0473, Prec: 0.0433, Rec: 0.0557 lr: 0.00010\n",
      "Epoch 2/Step 900, Loss: -0.06721, Accuracy: 0.99831, F1: 0.0473, Prec: 0.0433, Rec: 0.0556 lr: 0.00010\n",
      "Epoch 2/Step 920, Loss: -0.05378, Accuracy: 0.99831, F1: 0.0472, Prec: 0.0432, Rec: 0.0556 lr: 0.00010\n",
      "Epoch 2/Step 940, Loss: -0.02529, Accuracy: 0.99831, F1: 0.0472, Prec: 0.0432, Rec: 0.0556 lr: 0.00010\n",
      "Epoch 2/Step 960, Loss: -0.04644, Accuracy: 0.99831, F1: 0.0471, Prec: 0.0431, Rec: 0.0555 lr: 0.00010\n",
      "Epoch 2/Step 980, Loss: -0.06781, Accuracy: 0.99831, F1: 0.0471, Prec: 0.0431, Rec: 0.0556 lr: 0.00010\n",
      "Epoch 2/Step 1000, Loss: -0.04717, Accuracy: 0.99831, F1: 0.0471, Prec: 0.0431, Rec: 0.0556 lr: 0.00010\n",
      "Epoch 2/Step 1020, Loss: -0.03364, Accuracy: 0.99832, F1: 0.0471, Prec: 0.0431, Rec: 0.0557 lr: 0.00010\n",
      "Epoch 2/Step 1040, Loss: -0.04480, Accuracy: 0.99832, F1: 0.0471, Prec: 0.0430, Rec: 0.0556 lr: 0.00010\n",
      "Epoch 2/Step 1060, Loss: -0.05892, Accuracy: 0.99832, F1: 0.0470, Prec: 0.0429, Rec: 0.0557 lr: 0.00010\n",
      "Epoch 2/Step 1080, Loss: -0.05951, Accuracy: 0.99832, F1: 0.0471, Prec: 0.0429, Rec: 0.0558 lr: 0.00010\n",
      "Epoch 2/Step 1100, Loss: -0.06836, Accuracy: 0.99832, F1: 0.0471, Prec: 0.0429, Rec: 0.0559 lr: 0.00010\n",
      "Epoch 2/Step 1120, Loss: -0.03294, Accuracy: 0.99832, F1: 0.0471, Prec: 0.0429, Rec: 0.0560 lr: 0.00010\n",
      "Epoch 2/Step 1140, Loss: -0.03731, Accuracy: 0.99832, F1: 0.0471, Prec: 0.0428, Rec: 0.0560 lr: 0.00010\n",
      "Epoch 2/Step 1160, Loss: -0.03484, Accuracy: 0.99832, F1: 0.0471, Prec: 0.0428, Rec: 0.0560 lr: 0.00010\n",
      "Epoch 2/Step 1180, Loss: -0.04002, Accuracy: 0.99833, F1: 0.0472, Prec: 0.0429, Rec: 0.0562 lr: 0.00010\n",
      "Epoch 2/Step 1200, Loss: -0.02670, Accuracy: 0.99833, F1: 0.0472, Prec: 0.0428, Rec: 0.0563 lr: 0.00010\n",
      "Epoch 2/Step 1220, Loss: -0.03164, Accuracy: 0.99833, F1: 0.0470, Prec: 0.0427, Rec: 0.0561 lr: 0.00010\n",
      "Epoch 2/Step 1240, Loss: -0.07194, Accuracy: 0.99833, F1: 0.0470, Prec: 0.0425, Rec: 0.0562 lr: 0.00010\n",
      "Epoch 2/Step 1260, Loss: -0.06399, Accuracy: 0.99833, F1: 0.0470, Prec: 0.0426, Rec: 0.0563 lr: 0.00010\n",
      "Epoch 2/Step 1280, Loss: -0.06570, Accuracy: 0.99833, F1: 0.0470, Prec: 0.0426, Rec: 0.0563 lr: 0.00010\n",
      "Epoch 2/Step 1300, Loss: -0.05087, Accuracy: 0.99833, F1: 0.0470, Prec: 0.0426, Rec: 0.0563 lr: 0.00010\n",
      "Epoch 2/Step 1320, Loss: -0.04446, Accuracy: 0.99833, F1: 0.0471, Prec: 0.0426, Rec: 0.0564 lr: 0.00010\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[40], line 76\u001b[0m\n\u001b[1;32m     73\u001b[0m \u001b[39mprint\u001b[39m(\u001b[39m\"\u001b[39m\u001b[39m\\n\u001b[39;00m\u001b[39mStart of epoch \u001b[39m\u001b[39m%d\u001b[39;00m\u001b[39m\"\u001b[39m \u001b[39m%\u001b[39m (epoch\u001b[39m+\u001b[39m\u001b[39m1\u001b[39m,))\n\u001b[1;32m     75\u001b[0m \u001b[39m# Iterate over the batches of the dataset.\u001b[39;00m\n\u001b[0;32m---> 76\u001b[0m \u001b[39mfor\u001b[39;00m step, (x_batch_train, y_batch_train) \u001b[39min\u001b[39;00m \u001b[39menumerate\u001b[39m(batchedDataset):\n\u001b[1;32m     78\u001b[0m     loss_value \u001b[39m=\u001b[39mtrainStep(x_batch_train,y_batch_train)\n\u001b[1;32m     80\u001b[0m     \u001b[39m# Log \u001b[39;00m\n",
      "File \u001b[0;32m~/miniconda3/envs/tf/lib/python3.9/site-packages/tensorflow/python/data/ops/iterator_ops.py:797\u001b[0m, in \u001b[0;36mOwnedIterator.__next__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    795\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39m__next__\u001b[39m(\u001b[39mself\u001b[39m):\n\u001b[1;32m    796\u001b[0m   \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m--> 797\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_next_internal()\n\u001b[1;32m    798\u001b[0m   \u001b[39mexcept\u001b[39;00m errors\u001b[39m.\u001b[39mOutOfRangeError:\n\u001b[1;32m    799\u001b[0m     \u001b[39mraise\u001b[39;00m \u001b[39mStopIteration\u001b[39;00m\n",
      "File \u001b[0;32m~/miniconda3/envs/tf/lib/python3.9/site-packages/tensorflow/python/data/ops/iterator_ops.py:780\u001b[0m, in \u001b[0;36mOwnedIterator._next_internal\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    777\u001b[0m \u001b[39m# TODO(b/77291417): This runs in sync mode as iterators use an error status\u001b[39;00m\n\u001b[1;32m    778\u001b[0m \u001b[39m# to communicate that there is no more data to iterate over.\u001b[39;00m\n\u001b[1;32m    779\u001b[0m \u001b[39mwith\u001b[39;00m context\u001b[39m.\u001b[39mexecution_mode(context\u001b[39m.\u001b[39mSYNC):\n\u001b[0;32m--> 780\u001b[0m   ret \u001b[39m=\u001b[39m gen_dataset_ops\u001b[39m.\u001b[39;49miterator_get_next(\n\u001b[1;32m    781\u001b[0m       \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_iterator_resource,\n\u001b[1;32m    782\u001b[0m       output_types\u001b[39m=\u001b[39;49m\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_flat_output_types,\n\u001b[1;32m    783\u001b[0m       output_shapes\u001b[39m=\u001b[39;49m\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_flat_output_shapes)\n\u001b[1;32m    785\u001b[0m   \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m    786\u001b[0m     \u001b[39m# Fast path for the case `self._structure` is not a nested structure.\u001b[39;00m\n\u001b[1;32m    787\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_element_spec\u001b[39m.\u001b[39m_from_compatible_tensor_list(ret)  \u001b[39m# pylint: disable=protected-access\u001b[39;00m\n",
      "File \u001b[0;32m~/miniconda3/envs/tf/lib/python3.9/site-packages/tensorflow/python/ops/gen_dataset_ops.py:3011\u001b[0m, in \u001b[0;36miterator_get_next\u001b[0;34m(iterator, output_types, output_shapes, name)\u001b[0m\n\u001b[1;32m   3009\u001b[0m \u001b[39mif\u001b[39;00m tld\u001b[39m.\u001b[39mis_eager:\n\u001b[1;32m   3010\u001b[0m   \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m-> 3011\u001b[0m     _result \u001b[39m=\u001b[39m pywrap_tfe\u001b[39m.\u001b[39;49mTFE_Py_FastPathExecute(\n\u001b[1;32m   3012\u001b[0m       _ctx, \u001b[39m\"\u001b[39;49m\u001b[39mIteratorGetNext\u001b[39;49m\u001b[39m\"\u001b[39;49m, name, iterator, \u001b[39m\"\u001b[39;49m\u001b[39moutput_types\u001b[39;49m\u001b[39m\"\u001b[39;49m, output_types,\n\u001b[1;32m   3013\u001b[0m       \u001b[39m\"\u001b[39;49m\u001b[39moutput_shapes\u001b[39;49m\u001b[39m\"\u001b[39;49m, output_shapes)\n\u001b[1;32m   3014\u001b[0m     \u001b[39mreturn\u001b[39;00m _result\n\u001b[1;32m   3015\u001b[0m   \u001b[39mexcept\u001b[39;00m _core\u001b[39m.\u001b[39m_NotOkStatusException \u001b[39mas\u001b[39;00m e:\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "\n",
    "BATCH_SIZE=64\n",
    "LOG_INTERVAL=20\n",
    "epochs = 15\n",
    "saveModel=False\n",
    "\n",
    "\n",
    "log_dir = \"./logs/\"+model.name+\"/\" + datetime.datetime.now().strftime(\"%Y%m%d-%H%M%S\")+\"_\"+SO\n",
    "tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir, histogram_freq=1,\n",
    "                                                      write_graph=True, update_freq=5)\n",
    "\n",
    "\n",
    "summary_writer = tf.summary.create_file_writer(log_dir)\n",
    "\n",
    "# Instantiate an optimizer .\n",
    "# optimizer = tf.keras.optimizers.SGD(learning_rate=lr_schedule)\n",
    "# optimizer = tf.keras.optimizers.SGD(learning_rate=1e-3)\n",
    "optimizer = tf.keras.optimizers.Adam(learning_rate=1e-4)\n",
    "\n",
    "# Instantiate a loss function.\n",
    "# loss_fn = tf.keras.losses.BinaryCrossentropy()\n",
    "# loss_fn = WeightedBinaryCE(np.ones(len(mlb.classes_)))\n",
    "# loss_fn = WeightedBinaryCE(labelWeightsCorr)\n",
    "# loss_fn = WeightedComboLoss(labelWeightsCorr, alpha=0.5, beta=0.5, labelSmoothing=0.05)\n",
    "loss_fn = WeightedComboLoss(labelWeightsCorr, alpha=0.5, beta=0.5)\n",
    "\n",
    "train_acc_metric = WeightedAccuracy(classWeights=labelWeightsCorr)\n",
    "train_f1_metric = WeightedF1(classWeights=labelWeightsCorr, threshold=0.5)\n",
    "train_prec = WeightedPrecision(classWeights=labelWeightsCorr)\n",
    "train_rec = WeightedRecall(classWeights=labelWeightsCorr)\n",
    "\n",
    "val_acc_metric = WeightedAccuracy(classWeights=labelWeightsCorr)\n",
    "val_f1_metric = WeightedF1(classWeights=labelWeightsCorr, threshold=0.5)\n",
    "val_prec = WeightedPrecision(classWeights=labelWeightsCorr)\n",
    "val_rec = WeightedRecall(classWeights=labelWeightsCorr)\n",
    "\n",
    "batchedDataset = dataset.batch(BATCH_SIZE, drop_remainder=False)\n",
    "batchedDatasetVal = datasetVal.batch(BATCH_SIZE, drop_remainder=False)\n",
    "\n",
    "# batchedDataset = batchedDataset.cache(os.path.join(DATA_PATH, \"datasetCache\"+SO))\n",
    "# batchedDatasetVal = batchedDatasetVal.cache(os.path.join(DATA_PATH, \"datasetCacheVal\"+SO))\n",
    "\n",
    "@tf.function()\n",
    "def trainStep(x_batch_train, y_batch_train):\n",
    "    with tf.GradientTape() as tape:\n",
    "\n",
    "        probs = model(x_batch_train, training=True) \n",
    "        loss_value = loss_fn(y_batch_train, probs)\n",
    "\n",
    "    grads = tape.gradient(loss_value, model.trainable_weights)\n",
    "    #Gradient clipping\n",
    "    # grads = [tf.clip_by_norm(g, 2.0) for g in grads]\n",
    "\n",
    "    train_acc_metric.update_state(y_batch_train, probs)\n",
    "    train_f1_metric.update_state(y_batch_train, probs)\n",
    "    train_prec.update_state(y_batch_train, probs)\n",
    "    train_rec.update_state(y_batch_train, probs)\n",
    "\n",
    "    optimizer.apply_gradients(zip(grads, model.trainable_weights)) \n",
    "    return loss_value\n",
    "\n",
    "@tf.function()\n",
    "def valStep(x_batch_val, y_batch_val):\n",
    "    valProbs = model(x_batch_val, training=False)\n",
    "    # Update val metrics\n",
    "    val_acc_metric.update_state(y_batch_val, valProbs)\n",
    "    val_f1_metric.update_state(y_batch_val, valProbs)\n",
    "    val_prec.update_state(y_batch_val, valProbs)\n",
    "    val_rec.update_state(y_batch_val, valProbs)\n",
    "\n",
    "maxStep=0\n",
    "\n",
    "for epoch in range(epochs):\n",
    "    print(\"\\nStart of epoch %d\" % (epoch+1,))\n",
    "\n",
    "    # Iterate over the batches of the dataset.\n",
    "    for step, (x_batch_train, y_batch_train) in enumerate(batchedDataset):\n",
    "\n",
    "        loss_value =trainStep(x_batch_train,y_batch_train)\n",
    "\n",
    "        # Log \n",
    "        if step % LOG_INTERVAL == 0:\n",
    "            template = 'Epoch {}/Step {}, Loss: {:.5f}, Accuracy: {:.5f}, F1: {:.4f}, Prec: {:.4f}, Rec: {:.4f} lr: {:.5f}'\n",
    "            print(template.format(epoch+1, step,loss_value.numpy(), \n",
    "                                    train_acc_metric.result(),train_f1_metric.result(),\n",
    "                                    train_prec.result(), train_rec.result(), optimizer.learning_rate.numpy()))\n",
    "            \n",
    "            with summary_writer.as_default():\n",
    "                tf.summary.scalar('loss', loss_value, step=maxStep*epoch+step)\n",
    "                tf.summary.scalar('accuracy', train_acc_metric.result(), step=maxStep*epoch+step)\n",
    "                tf.summary.scalar('f1', train_f1_metric.result(), step=maxStep*epoch+step)\n",
    "                tf.summary.scalar('prec', train_prec.result(), step=maxStep*epoch+step)\n",
    "                tf.summary.scalar('rec', train_rec.result(), step=maxStep*epoch+step)\n",
    "                tf.summary.scalar('learning rate', optimizer.learning_rate.numpy(), step=maxStep*epoch+step)\n",
    "                summary_writer.flush()\n",
    "\n",
    "    \n",
    "    train_acc_metric.reset_states()\n",
    "    train_f1_metric.reset_states()\n",
    "    train_prec.reset_states()\n",
    "    train_rec.reset_states()\n",
    "\n",
    "    maxStep=step\n",
    "\n",
    "    print(\"Epoch finished. Start validation\")\n",
    "    for x_batch_val, y_batch_val in batchedDatasetVal:\n",
    "        valStep(x_batch_val, y_batch_val)\n",
    "    val_acc = val_acc_metric.result()\n",
    "    val_acc_metric.reset_states()\n",
    "    val_f1 = val_f1_metric.result()\n",
    "    val_f1_metric.reset_states()\n",
    "    val_precision = val_prec.result()\n",
    "    val_prec.reset_states()\n",
    "    val_recall = val_rec.result()\n",
    "    val_rec.reset_states()\n",
    "    print(\"Validation acc: %.4f\" % (float(val_acc),))\n",
    "    print(\"Validation f1: %.4f\" % (float(val_f1),))\n",
    "    print(\"Validation precision: %.4f\" % (float(val_precision),))\n",
    "    print(\"Validation recall: %.4f\" % (float(val_recall),))\n",
    "    with summary_writer.as_default():\n",
    "        tf.summary.scalar('valAcc', float(val_acc), step=epoch)\n",
    "        tf.summary.scalar('valF1', float(val_f1), step=epoch)\n",
    "        tf.summary.scalar('valPrecision', float(val_precision), step=epoch)\n",
    "        tf.summary.scalar('valRecall', float(val_recall), step=epoch)\n",
    "        summary_writer.flush()\n",
    "    if saveModel:\n",
    "      model.save(os.path.join(DATA_PATH, \"model_\"+SO+\"_epoch_{}_valF1Score{:.3f}\".format(epoch, float(val_f1))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "374f7234",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# model.save(os.path.join(DATA_PATH, \"model_\"+SO+\"_epoch_{}_valf1Score{:.3f}\".format(epoch, float(val_f1))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cefc1f6a",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "probs= model.predict(tf.expand_dims(list(datasetVal.take(32))[10][0], 0))\n",
    "prediction= [1 if p > 0.5 else 0 for p in probs[0]]\n",
    "probabilities= probs[probs>0.5]\n",
    "# classes = np.argwhere(prediction)\n",
    "print(mlb.inverse_transform(np.array([prediction])))\n",
    "print(probabilities)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}