zz_fragement_analysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "\n",
    "from torch.optim.lr_scheduler import ReduceLROnPlateau\n",
    "from torch.utils.data import DataLoader\n",
    "from torch.utils.data import Dataset\n",
    "from torch.utils.tensorboard import SummaryWriter\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import hashlib\n",
    "import glob\n",
    "import time\n",
    "import re\n",
    "import os\n",
    "\n",
    "from tqdm import tqdm\n",
    "from datetime import datetime\n",
    "from sklearn.metrics import f1_score, recall_score, precision_score, accuracy_score\n",
    "\n",
    "class Net(nn.Module):\n",
    "    def __init__(self, sequenceSize, embeddingDim, vocabularySize, filterWidth, filterNumber):\n",
    "        super(Net, self).__init__()\n",
    "        self.sequenceSize   = sequenceSize\n",
    "        self.embeddingDim   = embeddingDim\n",
    "        self.vocabularySize = vocabularySize\n",
    "        self.filterWidth    = filterWidth\n",
    "        self.filterNumber   = filterNumber \n",
    "        \n",
    "        self.embedding = nn.Embedding(self.vocabularySize, self.embeddingDim)\n",
    "        self.conv = nn.Sequential(\n",
    "                            nn.Conv2d(1, self.filterNumber, (self.filterWidth, self.embeddingDim)),\n",
    "                            nn.BatchNorm2d(self.filterNumber),\n",
    "                            nn.ReLU()\n",
    "                        )\n",
    "        \n",
    "        self.fc = nn.Sequential(\n",
    "                        nn.Linear(self.filterNumber , 512),\n",
    "                        nn.BatchNorm1d(512),\n",
    "                        nn.ReLU(),\n",
    "            \n",
    "                        nn.Linear(512, 256),\n",
    "                        nn.BatchNorm1d(256),\n",
    "                        nn.ReLU(),\n",
    "                        \n",
    "                        nn.Linear(256, 1),\n",
    "                        nn.Sigmoid()\n",
    "                    )\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.embedding(x)\n",
    "        #print(x.size())\n",
    "        \n",
    "        x = self.conv(x)\n",
    "        #print(x.size())\n",
    "        \n",
    "        x = x.max(dim=2)[0]\n",
    "        #print(x.size())\n",
    "\n",
    "        x = x.view(-1,  self.filterNumber)\n",
    "        x = self.fc(x)\n",
    "        return x\n",
    "\n",
    "class SampleDataset(Dataset):\n",
    "    def __init__(self, filePathList, labels, sequenceSize, featureName):\n",
    "        self.filePathList = filePathList\n",
    "        self.labels = labels\n",
    "        self.sequenceSize = sequenceSize\n",
    "        self.featureName = featureName\n",
    "        \n",
    "    def __len__(self):\n",
    "        return len(self.filePathList)\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        df = pd.read_parquet(self.filePathList[idx])\n",
    "        seed = int(round(time.time()%1, 6) * 1000000)\n",
    "        x = np.concatenate(df.iloc[np.random.RandomState(seed).permutation(len(df))][self.featureName].values)\n",
    "\n",
    "        if len(x) > self.sequenceSize:\n",
    "            x = x[:self.sequenceSize]\n",
    "        else:\n",
    "            x = np.concatenate((x, np.zeros([self.sequenceSize - len(x)])))\n",
    "            \n",
    "        sample = torch.from_numpy(x)\n",
    "        return (sample.long(), self.labels[idx], self.filePathList[idx])\n",
    "\n",
    "def computerMetrics(epoch, tlabels, tpredicted, tloss, \n",
    "                           vlabels, vpredicted, vloss):\n",
    "    \n",
    "    message = '[{:04d}] '.format(epoch)\n",
    "\n",
    "    tf1score   = f1_score(tlabels, tpredicted)\n",
    "    message  += 'TF1: {:2.4f}, '.format(tf1score*100)\n",
    "    message  += 'Tloss: {:2.8f}, '.format(tloss)\n",
    "    \n",
    "    vf1score   = f1_score(vlabels, vpredicted)\n",
    "    message  += 'VF1: {:2.4f}, '.format(vf1score*100)\n",
    "    message  += 'VLoss: {:2.8f},'.format(vloss)    \n",
    "\n",
    "    return epoch, tf1score, tloss, vf1score, vloss, message\n",
    "\n",
    "def train(model, optimizer, dataLoader, device):\n",
    "    running_loss  = 0.0  \n",
    "    label_lst     = list()\n",
    "    predicted_lst = list()\n",
    "\n",
    "    model.train()\n",
    "    for inputs, labels, _ in dataLoader:\n",
    "        \n",
    "        #\n",
    "        inputs = inputs.unsqueeze(1).to(device)\n",
    "        labels = labels.to(device)\n",
    "\n",
    "        #\n",
    "        optimizer.zero_grad()\n",
    "\n",
    "        #\n",
    "        outputs = model(inputs)\n",
    "        predicted = (outputs > 0.5).squeeze().long()\n",
    "        loss = F.binary_cross_entropy(outputs.squeeze(), labels.float())\n",
    "\n",
    "        #\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        #\n",
    "        label_lst.append(labels.cpu().numpy())\n",
    "        predicted_lst.append(predicted.cpu().numpy())        \n",
    "        running_loss += loss.item() \n",
    "\n",
    "    labels    = np.concatenate(label_lst)\n",
    "    predicted = np.concatenate(predicted_lst)\n",
    "    loss      = running_loss / len(predicted)\n",
    "    \n",
    "    return labels, predicted, loss\n",
    "\n",
    "def assess(model, dataLoader, device):\n",
    "    running_loss  = 0.0  \n",
    "    label_lst     = list()\n",
    "    predicted_lst = list()\n",
    "    proba_lst     = list()\n",
    "    path_lst      = list()\n",
    "\n",
    "    with torch.no_grad():\n",
    "        model.eval()\n",
    "        for inputs, labels, paths in dataLoader:\n",
    "            #\n",
    "            inputs = inputs.unsqueeze(1).to(device)\n",
    "            labels = labels.to(device)\n",
    "\n",
    "            #\n",
    "            outputs = model(inputs)\n",
    "            predicted = (outputs > 0.5).squeeze().long()\n",
    "            loss = F.binary_cross_entropy(outputs.squeeze(), labels.float())\n",
    "\n",
    "            #\n",
    "            #if len(inputs) > 1:\n",
    "            label_lst.append(labels.cpu().numpy())\n",
    "            predicted_lst.append(predicted.cpu().numpy())\n",
    "            proba_lst.append(outputs.squeeze().cpu().numpy())\n",
    "            path_lst.append(paths)\n",
    "            running_loss += loss.item() \n",
    "    \n",
    "    labels    = np.concatenate(label_lst)\n",
    "    predicted = np.concatenate(predicted_lst)\n",
    "    proba     = np.concatenate(proba_lst)\n",
    "    paths     = np.concatenate(path_lst)\n",
    "    loss      = running_loss / len(predicted)\n",
    "    \n",
    "    return labels, predicted, loss, proba, paths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#\n",
    "ws = 'ws066'\n",
    "featureName  = 'functionMethodCallsArgs'\n",
    "\n",
    "#\n",
    "lr             = 1e-3\n",
    "batchSize      = 32\n",
    "weightDecay    = 9e-6\n",
    "sequenceSize   = 20000\n",
    "embeddingDim   = 128\n",
    "filterWidth    = 5\n",
    "filterNumber   = 1024\n",
    "vocabularySize = 10000\n",
    "\n",
    "#\n",
    "gpuDevice   = \"cuda:2\"\n",
    "epochNum    = 100\n",
    "numWorkers  = 16"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7000\n",
      "1000\n",
      "2000\n"
     ]
    }
   ],
   "source": [
    "# prepare dataset\n",
    "trainPercentage = 0.7\n",
    "validPercentage = 0.8\n",
    "testPercentage  = 1.0\n",
    "\n",
    "malware_rootDir = 'output/zoomalware/'\n",
    "benign_rootDir  = 'output/zoobenign/'\n",
    "\n",
    "malware_df  = pd.DataFrame([(fileName, 1) for fileName in glob.glob(malware_rootDir + '*')], columns=['filePath', 'label']).sample(5000, random_state=54)\n",
    "benign_df   = pd.DataFrame([(fileName, 0) for fileName in glob.glob(benign_rootDir + '*')], columns=['filePath', 'label'])\n",
    "benign_df   = benign_df.sample(len(malware_df), random_state=54)\n",
    "dataset_df  = pd.concat([malware_df, benign_df])\n",
    "datasetSize = len(dataset_df)\n",
    "dataset_df.label.value_counts()\n",
    "\n",
    "rand_idx = np.random.RandomState(seed=54).permutation(datasetSize)\n",
    "train_df = dataset_df.iloc[rand_idx[:int(trainPercentage * datasetSize)]]\n",
    "valid_df = dataset_df.iloc[rand_idx[int(trainPercentage * datasetSize):int(validPercentage * datasetSize)]]\n",
    "test_df  = dataset_df.iloc[rand_idx[int(validPercentage * datasetSize):]]\n",
    "\n",
    "print(len(train_df))\n",
    "print(len(valid_df))\n",
    "print(len(test_df))\n",
    "\n",
    "trainDataset = SampleDataset(train_df.filePath.values, train_df.label.values, sequenceSize, featureName)\n",
    "trainLoader = DataLoader(trainDataset, batch_size=batchSize, shuffle=True, num_workers=numWorkers)\n",
    "\n",
    "validDataset = SampleDataset(valid_df.filePath.values, valid_df.label.values, sequenceSize, featureName)\n",
    "validLoader = DataLoader(validDataset, batch_size=batchSize, shuffle=False, num_workers=numWorkers)\n",
    "\n",
    "testDataset  = SampleDataset(test_df.filePath.values, test_df.label.values, sequenceSize, featureName)\n",
    "testLoader  = DataLoader(testDataset,  batch_size=batchSize, shuffle=False, num_workers=numWorkers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "model  = Net(sequenceSize, embeddingDim, vocabularySize, filterWidth, filterNumber)\n",
    "device = torch.device(gpuDevice)\n",
    "model  = model.to(device)\n",
    "\n",
    "optimizer = optim.Adam(model.parameters(), lr=lr, weight_decay=weightDecay)\n",
    "scheduler = ReduceLROnPlateau(optimizer, 'min', verbose=True, patience=5, factor=0.7)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " ws=ws066_filterWidth=5_filterNumber=1024_feature=functionMethodCallsArgs_seqSize=20000_embDim=128_vocabSize=10000_batchSize=32_lr=0.001_Decay=9e-06\n",
      "----------\n",
      "[0000] TF1: 69.4254, Tloss: 0.01730343, VF1: 65.2432, VLoss: 0.01791538,\n",
      "[0001] TF1: 73.3731, Tloss: 0.01596766, VF1: 62.4204, VLoss: 0.01907901,\n",
      "[0002] TF1: 75.3534, Tloss: 0.01530542, VF1: 72.1480, VLoss: 0.01715359,\n",
      "[0003] TF1: 76.9137, Tloss: 0.01476104, VF1: 49.4815, VLoss: 0.02355767,\n",
      "[0004] TF1: 76.7639, Tloss: 0.01452832, VF1: 74.3636, VLoss: 0.01734054,\n",
      "[0005] TF1: 78.0812, Tloss: 0.01403550, VF1: 47.4627, VLoss: 0.02271966,\n",
      "[0006] TF1: 77.9542, Tloss: 0.01386800, VF1: 66.8311, VLoss: 0.01808810,\n",
      "[0007] TF1: 78.2502, Tloss: 0.01369737, VF1: 61.5385, VLoss: 0.01973335,\n",
      "[0008] TF1: 78.9924, Tloss: 0.01353167, VF1: 74.1453, VLoss: 0.01697017,\n",
      "[0009] TF1: 78.9755, Tloss: 0.01327195, VF1: 72.4479, VLoss: 0.01688258,\n",
      "[0010] TF1: 79.5527, Tloss: 0.01322969, VF1: 74.7720, VLoss: 0.01629944,\n",
      "[0011] TF1: 80.3019, Tloss: 0.01285264, VF1: 65.8507, VLoss: 0.01807402,\n",
      "[0012] TF1: 80.4239, Tloss: 0.01270982, VF1: 72.8435, VLoss: 0.01654163,\n",
      "[0013] TF1: 80.9639, Tloss: 0.01257036, VF1: 71.5393, VLoss: 0.01710867,\n",
      "[0014] TF1: 80.6461, Tloss: 0.01243926, VF1: 60.5195, VLoss: 0.02004558,\n",
      "[0015] TF1: 80.8659, Tloss: 0.01250350, VF1: 70.7800, VLoss: 0.01793604,\n",
      "[0016] TF1: 81.0877, Tloss: 0.01217720, VF1: 75.2204, VLoss: 0.01703129,\n",
      "[0017] TF1: 81.0222, Tloss: 0.01208813, VF1: 72.6862, VLoss: 0.01737049,\n",
      "[0018] TF1: 81.6119, Tloss: 0.01222617, VF1: 74.2331, VLoss: 0.01758656,\n",
      "[0019] TF1: 81.4028, Tloss: 0.01202067, VF1: 75.0507, VLoss: 0.01670138,\n",
      "[0020] TF1: 82.4699, Tloss: 0.01180923, VF1: 74.9562, VLoss: 0.01783377,\n",
      "[0021] TF1: 82.3636, Tloss: 0.01153889, VF1: 39.0169, VLoss: 0.03337226,\n",
      "[0022] TF1: 82.1034, Tloss: 0.01183137, VF1: 45.5927, VLoss: 0.02647997,\n",
      "[0023] TF1: 82.9210, Tloss: 0.01141802, VF1: 74.0402, VLoss: 0.01696676,\n",
      "[0024] TF1: 83.2231, Tloss: 0.01123918, VF1: 75.6219, VLoss: 0.01838215,\n",
      "[0025] TF1: 82.1584, Tloss: 0.01169461, VF1: 75.2535, VLoss: 0.01759894,\n",
      "[0026] TF1: 83.0488, Tloss: 0.01108597, VF1: 74.5763, VLoss: 0.01670911,\n",
      "[0027] TF1: 82.9493, Tloss: 0.01147952, VF1: 18.4783, VLoss: 0.03535114,\n",
      "[0028] TF1: 83.9051, Tloss: 0.01096892, VF1: 74.7437, VLoss: 0.01906808,\n",
      "[0029] TF1: 84.1745, Tloss: 0.01087875, VF1: 74.3993, VLoss: 0.02326761,\n",
      "[0030] TF1: 83.4530, Tloss: 0.01096379, VF1: 72.0088, VLoss: 0.01859740,\n",
      "[0031] TF1: 83.5198, Tloss: 0.01097199, VF1: 74.1902, VLoss: 0.01934244,\n",
      "[0032] TF1: 84.0983, Tloss: 0.01074076, VF1: 51.8519, VLoss: 0.02716310,\n",
      "[0033] TF1: 83.8218, Tloss: 0.01074670, VF1: 73.6950, VLoss: 0.01769828,\n",
      "[0034] TF1: 83.9665, Tloss: 0.01060781, VF1: 70.8333, VLoss: 0.01861947,\n",
      "[0035] TF1: 83.5333, Tloss: 0.01069479, VF1: 72.0430, VLoss: 0.04408143,\n",
      "[0036] TF1: 84.2419, Tloss: 0.01059818, VF1: 71.6900, VLoss: 0.01807013,\n",
      "[0037] TF1: 83.9384, Tloss: 0.01051234, VF1: 72.2394, VLoss: 0.01823802,\n",
      "[0038] TF1: 84.7468, Tloss: 0.01025097, VF1: 75.3860, VLoss: 0.01976496,\n",
      "[0039] TF1: 84.4272, Tloss: 0.01025644, VF1: 74.5472, VLoss: 0.01954183,\n",
      "[0040] TF1: 85.3669, Tloss: 0.00992016, VF1: 73.0159, VLoss: 0.01886580,\n",
      "[0041] TF1: 84.6188, Tloss: 0.01022508, VF1: 73.1137, VLoss: 0.02238677,\n",
      "[0042] TF1: 84.5867, Tloss: 0.01035876, VF1: 72.9339, VLoss: 0.01771868,\n",
      "[0043] TF1: 85.4331, Tloss: 0.00991013, VF1: 75.0000, VLoss: 0.02125465,\n",
      "[0044] TF1: 85.1521, Tloss: 0.00994562, VF1: 75.5309, VLoss: 0.02413302,\n",
      "[0045] TF1: 85.0316, Tloss: 0.01027344, VF1: 71.1332, VLoss: 0.03364076,\n",
      "[0046] TF1: 85.4848, Tloss: 0.00996597, VF1: 71.9124, VLoss: 0.01991703,\n",
      "[0047] TF1: 85.3305, Tloss: 0.00981043, VF1: 72.4479, VLoss: 0.01858258,\n",
      "[0048] TF1: 85.8252, Tloss: 0.00964712, VF1: 74.6237, VLoss: 0.01904437,\n",
      "[0049] TF1: 85.7781, Tloss: 0.00978984, VF1: 64.6541, VLoss: 0.02354617,\n",
      "[0050] TF1: 86.0722, Tloss: 0.00963651, VF1: 69.9415, VLoss: 0.02081961,\n",
      "[0051] TF1: 85.7058, Tloss: 0.00962824, VF1: 63.0491, VLoss: 0.02296148,\n",
      "[0052] TF1: 85.1582, Tloss: 0.00973074, VF1: 70.4741, VLoss: 0.02058629,\n",
      "[0053] TF1: 86.2054, Tloss: 0.00942340, VF1: 72.6714, VLoss: 0.01775050,\n",
      "[0054] TF1: 85.9228, Tloss: 0.00949990, VF1: 73.1092, VLoss: 0.02651422,\n",
      "[0055] TF1: 85.6633, Tloss: 0.00953259, VF1: 76.1026, VLoss: 0.01987417,\n",
      "[0056] TF1: 86.4554, Tloss: 0.00929196, VF1: 76.8670, VLoss: 0.02112172,\n",
      "[0057] TF1: 86.1251, Tloss: 0.00914346, VF1: 74.4581, VLoss: 0.02053379,\n",
      "[0058] TF1: 86.1849, Tloss: 0.00943567, VF1: 72.4464, VLoss: 0.02297871,\n",
      "[0059] TF1: 86.6172, Tloss: 0.00918486, VF1: 71.2299, VLoss: 0.01900313,\n",
      "[0060] TF1: 86.0913, Tloss: 0.00924207, VF1: 75.4067, VLoss: 0.01967850,\n",
      "[0061] TF1: 86.3198, Tloss: 0.00933900, VF1: 76.5143, VLoss: 0.01937368,\n",
      "[0062] TF1: 86.9062, Tloss: 0.00917101, VF1: 75.3327, VLoss: 0.01950562,\n",
      "[0063] TF1: 86.3905, Tloss: 0.00914764, VF1: 68.3721, VLoss: 0.01863278,\n",
      "Epoch    63: reducing learning rate of group 0 to 7.0000e-04.\n",
      "[0064] TF1: 87.4092, Tloss: 0.00851060, VF1: 66.8241, VLoss: 0.02573716,\n",
      "[0065] TF1: 87.7193, Tloss: 0.00847756, VF1: 74.8454, VLoss: 0.02042821,\n",
      "[0066] TF1: 87.5977, Tloss: 0.00845416, VF1: 72.0183, VLoss: 0.02248489,\n",
      "[0067] TF1: 87.7126, Tloss: 0.00832548, VF1: 68.9412, VLoss: 0.02083934,\n",
      "[0068] TF1: 88.2806, Tloss: 0.00824372, VF1: 69.4845, VLoss: 0.02046738,\n",
      "[0069] TF1: 87.8819, Tloss: 0.00836601, VF1: 74.4770, VLoss: 0.02186126,\n",
      "[0070] TF1: 88.1778, Tloss: 0.00819773, VF1: 72.7884, VLoss: 0.02309014,\n",
      "[0071] TF1: 88.3865, Tloss: 0.00802190, VF1: 75.0000, VLoss: 0.02236679,\n",
      "[0072] TF1: 88.3304, Tloss: 0.00803326, VF1: 75.2988, VLoss: 0.02085100,\n",
      "[0073] TF1: 88.1850, Tloss: 0.00805316, VF1: 72.9316, VLoss: 0.02646850,\n",
      "[0074] TF1: 88.6467, Tloss: 0.00782251, VF1: 75.8427, VLoss: 0.02333844,\n",
      "[0075] TF1: 88.6982, Tloss: 0.00784125, VF1: 75.1301, VLoss: 0.02370398,\n",
      "[0076] TF1: 88.7812, Tloss: 0.00786007, VF1: 70.5215, VLoss: 0.02375598,\n",
      "[0077] TF1: 88.3324, Tloss: 0.00785306, VF1: 71.0911, VLoss: 0.02410820,\n",
      "[0078] TF1: 88.4444, Tloss: 0.00780663, VF1: 76.5455, VLoss: 0.02666233,\n",
      "[0079] TF1: 88.6723, Tloss: 0.00776094, VF1: 68.9576, VLoss: 0.02552918,\n",
      "[0080] TF1: 89.0035, Tloss: 0.00765024, VF1: 74.0331, VLoss: 0.02201656,\n",
      "[0081] TF1: 88.8530, Tloss: 0.00778305, VF1: 74.7271, VLoss: 0.04064941,\n",
      "[0082] TF1: 88.6568, Tloss: 0.00773293, VF1: 67.3759, VLoss: 0.02431420,\n",
      "[0083] TF1: 88.8856, Tloss: 0.00773344, VF1: 74.9734, VLoss: 0.02028313,\n",
      "[0084] TF1: 89.0433, Tloss: 0.00749976, VF1: 75.4639, VLoss: 0.02496933,\n",
      "[0085] TF1: 89.3748, Tloss: 0.00753189, VF1: 74.6554, VLoss: 0.02197432,\n",
      "[0086] TF1: 88.5718, Tloss: 0.00763668, VF1: 67.6504, VLoss: 0.02866505,\n",
      "[0087] TF1: 88.9280, Tloss: 0.00759200, VF1: 75.1553, VLoss: 0.02077064,\n",
      "[0088] TF1: 88.9547, Tloss: 0.00765337, VF1: 74.8162, VLoss: 0.03311955,\n",
      "[0089] TF1: 88.7061, Tloss: 0.00757111, VF1: 67.9335, VLoss: 0.02723317,\n",
      "[0090] TF1: 89.3392, Tloss: 0.00743723, VF1: 73.6515, VLoss: 0.02289468,\n",
      "[0091] TF1: 89.6968, Tloss: 0.00713676, VF1: 73.5471, VLoss: 0.02744629,\n",
      "[0092] TF1: 89.8031, Tloss: 0.00723224, VF1: 72.6937, VLoss: 0.03452096,\n",
      "[0093] TF1: 89.8380, Tloss: 0.00724438, VF1: 68.4770, VLoss: 0.02778970,\n",
      "[0094] TF1: 89.1566, Tloss: 0.00746422, VF1: 73.1707, VLoss: 0.02062305,\n",
      "[0095] TF1: 89.6602, Tloss: 0.00728343, VF1: 77.3663, VLoss: 0.02299136,\n",
      "[0096] TF1: 90.2790, Tloss: 0.00697086, VF1: 74.2493, VLoss: 0.02769651,\n",
      "[0097] TF1: 89.1658, Tloss: 0.00729428, VF1: 73.9837, VLoss: 0.02467929,\n",
      "[0098] TF1: 90.0544, Tloss: 0.00704694, VF1: 61.5190, VLoss: 0.02880005,\n",
      "[0099] TF1: 90.5421, Tloss: 0.00683139, VF1: 68.1395, VLoss: 0.02606802,\n",
      "----------\n"
     ]
    }
   ],
   "source": [
    "modelID = f' ws={ws}_filterWidth={filterWidth}_filterNumber={filterNumber}_feature={featureName}_seqSize={sequenceSize}_embDim={embeddingDim}_vocabSize={vocabularySize}_batchSize={batchSize}_lr={lr}_Decay={weightDecay}'\n",
    "print(modelID)\n",
    "trainComment = f' ws={ws} Train filterWidth={filterWidth} filterNumber={filterNumber} feature={featureName} seqSize={sequenceSize} embDim={embeddingDim} vocabSize={vocabularySize} batchSize={batchSize} lr={lr} Decay={weightDecay}'\n",
    "validComment = f' ws={ws} Valid filterWidth={filterWidth} filterNumber={filterNumber} feature={featureName} seqSize={sequenceSize} embDim={embeddingDim} vocabSize={vocabularySize} batchSize={batchSize} lr={lr} Decay={weightDecay}'\n",
    "\n",
    "tb = SummaryWriter(comment=trainComment)\n",
    "vb = SummaryWriter(comment=validComment)\n",
    "outputlogFilePath = f'./traces/{ws}/logs'\n",
    "outputtracesPath  = f'./traces/{ws}'\n",
    "os.mkdir(outputtracesPath)\n",
    "\n",
    "result_lst = list()\n",
    "\n",
    "print('----------')\n",
    "for epoch in range(epochNum):\n",
    "\n",
    "    tlabel, tpredicted, tloss = train(model, optimizer, trainLoader, device)\n",
    "    vlabel, vpredicted, vloss, vproba, _ = assess(model, validLoader, device)\n",
    "\n",
    "    metrics = computerMetrics(epoch, tlabel, tpredicted, tloss,\n",
    "                                     vlabel, vpredicted, vloss)\n",
    "    epoch, tf1score, tloss, vf1score, vloss, message = metrics\n",
    "\n",
    "    with open(outputlogFilePath, 'a') as writer:\n",
    "        writer.write(message + '\\n')\n",
    "    print(message)\n",
    "\n",
    "    tb.add_scalar(\"Loss\",  tloss,    epoch)\n",
    "    tb.add_scalar(\"F1\",    tf1score, epoch)\n",
    "    vb.add_scalar(\"Loss\",  vloss,    epoch)\n",
    "    vb.add_scalar(\"F1\",    vf1score, epoch)\n",
    "\n",
    "    modelOutputPath = f'{outputtracesPath}/model_{epoch:03d}.pth'\n",
    "    torch.save(model.state_dict(), modelOutputPath)\n",
    "    result_lst.append((epoch, modelOutputPath, vlabel, vpredicted, vf1score, vloss, tf1score, tloss))\n",
    "    \n",
    "    scheduler.step(tloss)\n",
    "    \n",
    "df = pd.DataFrame(result_lst, \n",
    "                   columns=['epoch', 'path', 'labels', 'predicted', 'vf1score', 'vloss', 'tf1score', 'tloss'])\n",
    "df.to_parquet(f'{outputtracesPath}/results.parquet')\n",
    "\n",
    "print('----------')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(result_lst, \n",
    "                   columns=['epoch', 'path', 'labels', 'predicted', 'vf1score', 'vloss', 'tf1score', 'tloss'])\n",
    "df.to_parquet(f'{outputtracesPath}/results.parquet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.sort_values(by=['vloss', 'tloss', 'vf1score'], inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>epoch</th>\n",
       "      <th>path</th>\n",
       "      <th>labels</th>\n",
       "      <th>predicted</th>\n",
       "      <th>vf1score</th>\n",
       "      <th>vloss</th>\n",
       "      <th>tf1score</th>\n",
       "      <th>tloss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>19</td>\n",
       "      <td>./traces/ws039/model_019.pth</td>\n",
       "      <td>[1, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 1, 1, 1, 0, ...</td>\n",
       "      <td>[1, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 1, 1, 1, 0, ...</td>\n",
       "      <td>0.965803</td>\n",
       "      <td>0.002939</td>\n",
       "      <td>0.996954</td>\n",
       "      <td>0.000704</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>22</td>\n",
       "      <td>./traces/ws039/model_022.pth</td>\n",
       "      <td>[1, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 1, 1, 1, 0, ...</td>\n",
       "      <td>[1, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 1, 1, 1, 0, ...</td>\n",
       "      <td>0.961094</td>\n",
       "      <td>0.003038</td>\n",
       "      <td>0.996954</td>\n",
       "      <td>0.000630</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>20</td>\n",
       "      <td>./traces/ws039/model_020.pth</td>\n",
       "      <td>[1, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 1, 1, 1, 0, ...</td>\n",
       "      <td>[1, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 1, 1, 1, 0, ...</td>\n",
       "      <td>0.959916</td>\n",
       "      <td>0.003471</td>\n",
       "      <td>0.994924</td>\n",
       "      <td>0.000791</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>24</td>\n",
       "      <td>./traces/ws039/model_024.pth</td>\n",
       "      <td>[1, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 1, 1, 1, 0, ...</td>\n",
       "      <td>[1, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 1, 1, 1, 0, ...</td>\n",
       "      <td>0.959752</td>\n",
       "      <td>0.003632</td>\n",
       "      <td>0.990844</td>\n",
       "      <td>0.000970</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>23</td>\n",
       "      <td>./traces/ws039/model_023.pth</td>\n",
       "      <td>[1, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 1, 1, 1, 0, ...</td>\n",
       "      <td>[1, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 1, 1, 1, 0, ...</td>\n",
       "      <td>0.959184</td>\n",
       "      <td>0.003704</td>\n",
       "      <td>0.993902</td>\n",
       "      <td>0.000875</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    epoch                          path  \\\n",
       "19     19  ./traces/ws039/model_019.pth   \n",
       "22     22  ./traces/ws039/model_022.pth   \n",
       "20     20  ./traces/ws039/model_020.pth   \n",
       "24     24  ./traces/ws039/model_024.pth   \n",
       "23     23  ./traces/ws039/model_023.pth   \n",
       "\n",
       "                                               labels  \\\n",
       "19  [1, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 1, 1, 1, 0, ...   \n",
       "22  [1, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 1, 1, 1, 0, ...   \n",
       "20  [1, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 1, 1, 1, 0, ...   \n",
       "24  [1, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 1, 1, 1, 0, ...   \n",
       "23  [1, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 1, 1, 1, 0, ...   \n",
       "\n",
       "                                            predicted  vf1score     vloss  \\\n",
       "19  [1, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 1, 1, 1, 0, ...  0.965803  0.002939   \n",
       "22  [1, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 1, 1, 1, 0, ...  0.961094  0.003038   \n",
       "20  [1, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 1, 1, 1, 0, ...  0.959916  0.003471   \n",
       "24  [1, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 1, 1, 1, 0, ...  0.959752  0.003632   \n",
       "23  [1, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 1, 1, 1, 0, ...  0.959184  0.003704   \n",
       "\n",
       "    tf1score     tloss  \n",
       "19  0.996954  0.000704  \n",
       "22  0.996954  0.000630  \n",
       "20  0.994924  0.000791  \n",
       "24  0.990844  0.000970  \n",
       "23  0.993902  0.000875  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "numberModels    = 6\n",
    "numberFragments = 1\n",
    "\n",
    "probaUpperBorn = 0.8\n",
    "probaLowerBorn = 0.2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ModelPath=./traces/ws039/model_019.pth Fragment=00\n",
      "score=0.9649484536082474\n",
      "ModelPath=./traces/ws039/model_022.pth Fragment=00\n",
      "score=0.956887486855941\n",
      "ModelPath=./traces/ws039/model_020.pth Fragment=00\n",
      "score=0.9548793284365162\n",
      "ModelPath=./traces/ws039/model_024.pth Fragment=00\n",
      "score=0.959917780061665\n",
      "ModelPath=./traces/ws039/model_023.pth Fragment=00\n",
      "score=0.9543147208121827\n",
      "ModelPath=./traces/ws039/model_028.pth Fragment=00\n",
      "score=0.9481641468682505\n"
     ]
    }
   ],
   "source": [
    "modelPathList   = df.path.iloc[:numberModels].values\n",
    "vmodelResultList = []\n",
    "\n",
    "for modelPath in modelPathList:\n",
    "    for fragment in range(numberFragments):\n",
    "        print(f'ModelPath={modelPath} Fragment={fragment:02d}')\n",
    "        mdl = Net(sequenceSize, embeddingDim, vocabularySize, filterWidth, filterNumber).to(device)\n",
    "        mdl.load_state_dict(torch.load(modelPath))\n",
    "        mdl.eval()\n",
    "        modelResult = assess(mdl, validLoader, device)\n",
    "        print(f'score={f1_score(modelResult[0], modelResult[1])}')\n",
    "        vmodelResultList.append((modelPath,) + modelResult)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "vresult_df = pd.DataFrame(vmodelResultList, columns=['name', 'Truth', 'Predicted', 'loss', 'Proba', 'Path'])\n",
    "vresults   = np.vstack(vresult_df.Proba.values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1000\n",
      "0.9624217118997912\n"
     ]
    }
   ],
   "source": [
    "truth       = vresult_df.Truth.iloc[0]\n",
    "paths       = vresult_df.Path.iloc[0]\n",
    "result_mean = vresults.mean(axis=0)\n",
    "result_std  = vresults.std(axis=0)\n",
    "predicted   = (result_mean > 0.5).astype('int')\n",
    "print(len(truth))\n",
    "print(f1_score(truth, predicted))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "940\n",
      "0.984478935698448\n"
     ]
    }
   ],
   "source": [
    "vtruth       = truth[(result_mean >= probaUpperBorn) | (result_mean <= probaLowerBorn)]\n",
    "vpaths       = paths[(result_mean >= probaUpperBorn) | (result_mean <= probaLowerBorn)]\n",
    "vresult_prob = result_mean[(result_mean >= probaUpperBorn) | (result_mean <= probaLowerBorn)]\n",
    "vresult_std  = result_std[(result_mean >= probaUpperBorn) | (result_mean <= probaLowerBorn)]\n",
    "\n",
    "vpredicted    = (vresult_prob > 0.5).astype('int')\n",
    "print(len(vtruth))\n",
    "print(f1_score(vtruth, vpredicted))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "60\n",
      "0.6071428571428571\n"
     ]
    }
   ],
   "source": [
    "etruth       = truth[(result_mean < probaUpperBorn) & (result_mean > probaLowerBorn)]\n",
    "epaths       = paths[(result_mean < probaUpperBorn) & (result_mean > probaLowerBorn)]\n",
    "eresult_prob = result_mean[(result_mean < probaUpperBorn) & (result_mean > probaLowerBorn)]\n",
    "eresult_std  = result_std[(result_mean < probaUpperBorn) & (result_mean > probaLowerBorn)]\n",
    "epredicted    = (eresult_prob > 0.5).astype('int')\n",
    "print(len(etruth))\n",
    "print(f1_score(etruth, epredicted))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ModelPath=./traces/ws039/model_019.pth Fragment=00\n",
      "score=0.6551724137931035\n",
      "ModelPath=./traces/ws039/model_022.pth Fragment=00\n",
      "score=0.509090909090909\n",
      "ModelPath=./traces/ws039/model_020.pth Fragment=00\n",
      "score=0.4313725490196078\n",
      "ModelPath=./traces/ws039/model_024.pth Fragment=00\n",
      "score=0.6567164179104477\n",
      "ModelPath=./traces/ws039/model_023.pth Fragment=00\n",
      "score=0.6575342465753424\n",
      "ModelPath=./traces/ws039/model_028.pth Fragment=00\n",
      "score=0.46153846153846156\n"
     ]
    }
   ],
   "source": [
    "errorDataset  = SampleDataset(epaths, etruth, sequenceSize, featureName)\n",
    "errorLoader  = DataLoader(errorDataset,  batch_size=batchSize, shuffle=False, num_workers=numWorkers)\n",
    "\n",
    "modelPathList   = df.path.iloc[:numberModels].values\n",
    "emodelResultList = []\n",
    "\n",
    "for modelPath in modelPathList:\n",
    "    for fragment in range(numberFragments):\n",
    "        print(f'ModelPath={modelPath} Fragment={fragment:02d}')\n",
    "        mdl = Net(sequenceSize, embeddingDim, vocabularySize, filterWidth, filterNumber).to(device)\n",
    "        mdl.load_state_dict(torch.load(modelPath))\n",
    "        mdl.eval()\n",
    "        modelResult = assess(mdl, errorLoader, device)\n",
    "        print(f'score={f1_score(modelResult[0], modelResult[1])}')\n",
    "        emodelResultList.append((modelPath,) + modelResult)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "60\n",
      "0.6666666666666666\n"
     ]
    }
   ],
   "source": [
    "eresult_df = pd.DataFrame(emodelResultList, columns=['name', 'Truth', 'Predicted', 'loss', 'Proba', 'Path'])\n",
    "eresults   = np.vstack(eresult_df.Proba.values)\n",
    "\n",
    "eresult_mean = eresults.mean(axis=0)\n",
    "eresult_std  = eresults.std(axis=0)\n",
    "epredicted   = (eresult_mean > 0.5).astype('int')\n",
    "print(len(etruth))\n",
    "print(f1_score(etruth, epredicted))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9655891553701773\n"
     ]
    }
   ],
   "source": [
    "ftruth     = np.concatenate([vtruth, etruth])\n",
    "fpredicted = np.concatenate([vpredicted, epredicted])\n",
    "print(f1_score(ftruth, fpredicted))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ModelPath=./traces/ws039/model_019.pth Fragment=00\n",
      "score=0.9575384615384614\n",
      "ModelPath=./traces/ws039/model_022.pth Fragment=00\n",
      "score=0.9596523898199876\n",
      "ModelPath=./traces/ws039/model_020.pth Fragment=00\n",
      "score=0.9551050864320357\n",
      "ModelPath=./traces/ws039/model_024.pth Fragment=00\n",
      "score=0.9538612164973688\n",
      "ModelPath=./traces/ws039/model_023.pth Fragment=00\n",
      "score=0.95767131594906\n",
      "ModelPath=./traces/ws039/model_028.pth Fragment=00\n",
      "score=0.9441533546325878\n"
     ]
    }
   ],
   "source": [
    "modelPathList   = df.path.iloc[:numberModels].values\n",
    "tmodelResultList = []\n",
    "\n",
    "for modelPath in modelPathList:\n",
    "    for fragment in range(numberFragments):\n",
    "        print(f'ModelPath={modelPath} Fragment={fragment:02d}')\n",
    "        mdl = Net(sequenceSize, embeddingDim, vocabularySize, filterWidth, filterNumber).to(device)\n",
    "        mdl.load_state_dict(torch.load(modelPath))\n",
    "        mdl.eval()\n",
    "        modelResult = assess(mdl, testLoader, device)\n",
    "        print(f'score={f1_score(modelResult[0], modelResult[1])}')\n",
    "        tmodelResultList.append((modelPath,) + modelResult)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "tresult_df = pd.DataFrame(tmodelResultList, columns=['name', 'Truth', 'Predicted', 'loss', 'Proba', 'Path'])\n",
    "tresults   = np.vstack(tresult_df.Proba.values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>Truth</th>\n",
       "      <th>Predicted</th>\n",
       "      <th>loss</th>\n",
       "      <th>Proba</th>\n",
       "      <th>Path</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>./traces/ws039/model_019.pth</td>\n",
       "      <td>[1, 1, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, 1, 0, 1, ...</td>\n",
       "      <td>[1, 1, 0, 1, 0, 1, 1, 0, 0, 1, 0, 0, 1, 0, 1, ...</td>\n",
       "      <td>0.004127</td>\n",
       "      <td>[0.9849166, 0.99896944, 0.00015135479, 0.97967...</td>\n",
       "      <td>[output/maldozer/6f917ddde266b6081f3eea7ce978f...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>./traces/ws039/model_022.pth</td>\n",
       "      <td>[1, 1, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, 1, 0, 1, ...</td>\n",
       "      <td>[1, 1, 0, 1, 0, 1, 1, 1, 0, 1, 0, 0, 1, 0, 1, ...</td>\n",
       "      <td>0.003980</td>\n",
       "      <td>[0.98783445, 0.95781535, 0.0010909116, 0.91214...</td>\n",
       "      <td>[output/maldozer/6f917ddde266b6081f3eea7ce978f...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>./traces/ws039/model_020.pth</td>\n",
       "      <td>[1, 1, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, 1, 0, 1, ...</td>\n",
       "      <td>[1, 1, 0, 1, 0, 1, 1, 0, 0, 1, 0, 0, 1, 0, 1, ...</td>\n",
       "      <td>0.004190</td>\n",
       "      <td>[0.9791101, 0.9973315, 0.001930463, 0.9634221,...</td>\n",
       "      <td>[output/maldozer/6f917ddde266b6081f3eea7ce978f...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>./traces/ws039/model_024.pth</td>\n",
       "      <td>[1, 1, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, 1, 0, 1, ...</td>\n",
       "      <td>[1, 1, 0, 1, 0, 1, 1, 0, 0, 1, 0, 0, 1, 0, 1, ...</td>\n",
       "      <td>0.004395</td>\n",
       "      <td>[0.9846147, 0.9877671, 0.0067612054, 0.9864375...</td>\n",
       "      <td>[output/maldozer/6f917ddde266b6081f3eea7ce978f...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>./traces/ws039/model_023.pth</td>\n",
       "      <td>[1, 1, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, 1, 0, 1, ...</td>\n",
       "      <td>[1, 1, 0, 1, 0, 1, 1, 0, 0, 1, 0, 0, 1, 0, 1, ...</td>\n",
       "      <td>0.004062</td>\n",
       "      <td>[0.9944318, 0.9982674, 0.0014715098, 0.9790892...</td>\n",
       "      <td>[output/maldozer/6f917ddde266b6081f3eea7ce978f...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                           name  \\\n",
       "0  ./traces/ws039/model_019.pth   \n",
       "1  ./traces/ws039/model_022.pth   \n",
       "2  ./traces/ws039/model_020.pth   \n",
       "3  ./traces/ws039/model_024.pth   \n",
       "4  ./traces/ws039/model_023.pth   \n",
       "\n",
       "                                               Truth  \\\n",
       "0  [1, 1, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, 1, 0, 1, ...   \n",
       "1  [1, 1, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, 1, 0, 1, ...   \n",
       "2  [1, 1, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, 1, 0, 1, ...   \n",
       "3  [1, 1, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, 1, 0, 1, ...   \n",
       "4  [1, 1, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, 1, 0, 1, ...   \n",
       "\n",
       "                                           Predicted      loss  \\\n",
       "0  [1, 1, 0, 1, 0, 1, 1, 0, 0, 1, 0, 0, 1, 0, 1, ...  0.004127   \n",
       "1  [1, 1, 0, 1, 0, 1, 1, 1, 0, 1, 0, 0, 1, 0, 1, ...  0.003980   \n",
       "2  [1, 1, 0, 1, 0, 1, 1, 0, 0, 1, 0, 0, 1, 0, 1, ...  0.004190   \n",
       "3  [1, 1, 0, 1, 0, 1, 1, 0, 0, 1, 0, 0, 1, 0, 1, ...  0.004395   \n",
       "4  [1, 1, 0, 1, 0, 1, 1, 0, 0, 1, 0, 0, 1, 0, 1, ...  0.004062   \n",
       "\n",
       "                                               Proba  \\\n",
       "0  [0.9849166, 0.99896944, 0.00015135479, 0.97967...   \n",
       "1  [0.98783445, 0.95781535, 0.0010909116, 0.91214...   \n",
       "2  [0.9791101, 0.9973315, 0.001930463, 0.9634221,...   \n",
       "3  [0.9846147, 0.9877671, 0.0067612054, 0.9864375...   \n",
       "4  [0.9944318, 0.9982674, 0.0014715098, 0.9790892...   \n",
       "\n",
       "                                                Path  \n",
       "0  [output/maldozer/6f917ddde266b6081f3eea7ce978f...  \n",
       "1  [output/maldozer/6f917ddde266b6081f3eea7ce978f...  \n",
       "2  [output/maldozer/6f917ddde266b6081f3eea7ce978f...  \n",
       "3  [output/maldozer/6f917ddde266b6081f3eea7ce978f...  \n",
       "4  [output/maldozer/6f917ddde266b6081f3eea7ce978f...  "
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tresult_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8000\n",
      "0.9641089108910891\n"
     ]
    }
   ],
   "source": [
    "truth       = tresult_df.Truth.iloc[0]\n",
    "paths       = tresult_df.Path.iloc[0]\n",
    "result_mean = tresults.mean(axis=0)\n",
    "result_std  = tresults.std(axis=0)\n",
    "predicted   = (result_mean > 0.5).astype('int')\n",
    "print(len(truth))\n",
    "print(f1_score(truth, predicted))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7453\n",
      "0.9823396627273935\n"
     ]
    }
   ],
   "source": [
    "vtruth       = truth[(result_mean >= probaUpperBorn) | (result_mean <= probaLowerBorn)]\n",
    "vpaths       = paths[(result_mean >= probaUpperBorn) | (result_mean <= probaLowerBorn)]\n",
    "vresult_prob = result_mean[(result_mean >= probaUpperBorn) | (result_mean <= probaLowerBorn)]\n",
    "vresult_std  = result_std[(result_mean >= probaUpperBorn) | (result_mean <= probaLowerBorn)]\n",
    "\n",
    "vpredicted    = (vresult_prob > 0.5).astype('int')\n",
    "print(len(vtruth))\n",
    "print(f1_score(vtruth, vpredicted))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "547\n",
      "0.7140255009107468\n"
     ]
    }
   ],
   "source": [
    "etruth       = truth[(result_mean < probaUpperBorn) & (result_mean > probaLowerBorn)]\n",
    "epaths       = paths[(result_mean < probaUpperBorn) & (result_mean > probaLowerBorn)]\n",
    "eresult_prob = result_mean[(result_mean < probaUpperBorn) & (result_mean > probaLowerBorn)]\n",
    "eresult_std  = result_std[(result_mean < probaUpperBorn) & (result_mean > probaLowerBorn)]\n",
    "epredicted    = (eresult_prob > 0.5).astype('int')\n",
    "print(len(etruth))\n",
    "print(f1_score(etruth, epredicted))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ModelPath=./traces/ws039/model_019.pth Fragment=00\n",
      "score=0.6804123711340206\n",
      "ModelPath=./traces/ws039/model_022.pth Fragment=00\n",
      "score=0.6252354048964219\n",
      "ModelPath=./traces/ws039/model_020.pth Fragment=00\n",
      "score=0.6303939962476549\n",
      "ModelPath=./traces/ws039/model_024.pth Fragment=00\n",
      "score=0.6677215189873419\n",
      "ModelPath=./traces/ws039/model_023.pth Fragment=00\n",
      "score=0.6924219910846954\n",
      "ModelPath=./traces/ws039/model_028.pth Fragment=00\n",
      "score=0.4367245657568239\n"
     ]
    }
   ],
   "source": [
    "errorDataset  = SampleDataset(epaths, etruth, sequenceSize, featureName)\n",
    "errorLoader  = DataLoader(errorDataset,  batch_size=batchSize, shuffle=False, num_workers=numWorkers)\n",
    "\n",
    "modelPathList   = df.path.iloc[:numberModels].values\n",
    "emodelResultList = []\n",
    "\n",
    "for modelPath in modelPathList:\n",
    "    for fragment in range(numberFragments):\n",
    "        print(f'ModelPath={modelPath} Fragment={fragment:02d}')\n",
    "        mdl = Net(sequenceSize, embeddingDim, vocabularySize, filterWidth, filterNumber).to(device)\n",
    "        mdl.load_state_dict(torch.load(modelPath))\n",
    "        mdl.eval()\n",
    "        modelResult = assess(mdl, errorLoader, device)\n",
    "        print(f'score={f1_score(modelResult[0], modelResult[1])}')\n",
    "        emodelResultList.append((modelPath,) + modelResult)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "547\n",
      "0.7137809187279152\n"
     ]
    }
   ],
   "source": [
    "eresult_df = pd.DataFrame(emodelResultList, columns=['name', 'Truth', 'Predicted', 'loss', 'Proba', 'Path'])\n",
    "eresults   = np.vstack(eresult_df.Proba.values)\n",
    "\n",
    "eresult_mean = eresults.mean(axis=0)\n",
    "eresult_std  = eresults.std(axis=0)\n",
    "epredicted   = (eresult_mean > 0.5).astype('int')\n",
    "print(len(etruth))\n",
    "print(f1_score(etruth, epredicted))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9635667531184389\n"
     ]
    }
   ],
   "source": [
    "ftruth     = np.concatenate([vtruth, etruth])\n",
    "fpredicted = np.concatenate([vpredicted, epredicted])\n",
    "print(f1_score(ftruth, fpredicted))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "pysciws",
   "language": "python",
   "name": "sciws"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}