3.entropy.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import time\n",
    "import torch\n",
    "import pickle as pkl\n",
    "import torch.nn as nn\n",
    "from sklearn import metrics\n",
    "from torch.utils.data import TensorDataset\n",
    "from torch.utils.data import DataLoader\n",
    "from datetime import timedelta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#numpy随机数置为1\n",
    "np.random.seed(1)\n",
    "#为CPU设置种子用于生成随机数，以使得结果是确定的\n",
    "torch.manual_seed(1)\n",
    "#GPU随机数种子置为1\n",
    "torch.cuda.manual_seed_all(1)\n",
    "torch.backends.cudnn.deterministic = True  # 保证每次结果一样"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 加载数据并转为数字"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "word-to-index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "word_to_index=pkl.load(open(\"./data/vocab.pkl\",\"rb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "pad_size=32\n",
    "#读取数据并转成索引\n",
    "def read_text(path):\n",
    "    contents,labels=[],[]\n",
    "    with open(path,\"r\",encoding=\"UTF-8\") as file:\n",
    "        for line in file:\n",
    "            #print(line.split(\"\\t\"))\n",
    "            #获取数据以及数据的标签\n",
    "            sentence,label=line.split(\"\\t\")[0],line.split(\"\\t\")[1]\n",
    "            #将文本数据按字分词\n",
    "            sentence_list=[]\n",
    "            for i in sentence:\n",
    "                sentence_list.append(i)\n",
    "            #pad_size=32，不够的补齐，超出的去掉\n",
    "            if len(sentence_list)<pad_size:\n",
    "                sentence_list.extend([word_to_index[\"<PAD>\"]] * (pad_size - len(sentence_list)))\n",
    "            else:\n",
    "                sentence_list=sentence_list[:pad_size]\n",
    "                #print(len(sentence_list),sentence_list)\n",
    "            #都转成固定的pad_size大小后，然后将其转为语料库的index\n",
    "            res_list=[]\n",
    "            for word in sentence_list:\n",
    "                if word == 4761:\n",
    "                    res_list.append(4761)\n",
    "                    continue\n",
    "                try: # 能在语料库中找到这个字的添加其索引\n",
    "                    res_list.append(word_to_index[word])\n",
    "                except: # 语料库里没有的字换成<UNK>的索引\n",
    "                    res_list.append(word_to_index[\"<UNK>\"])\n",
    "            #print(res_list,label)\n",
    "            contents.append(res_list)\n",
    "            labels.append(int(label))\n",
    "    return contents,labels"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "因为是主动学习，这里我们采用验证集的数据作为模型训练数据，训练集的数据作为模型的未标注数据，测试集数据还作为测试集数据进行模型效果评估"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "unlabeled_data,unlabeled_label=read_text(\"./data/train.txt\")\n",
    "train_data,train_label=read_text(\"./data/dev.txt\")\n",
    "test_data,test_label=read_text(\"./data/test.txt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10000 180000 10000\n"
     ]
    }
   ],
   "source": [
    "print(len(train_data),len(unlabeled_data),len(test_data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'list'> [173, 714, 3, 186, 1844, 889, 0, 2641, 80, 2061, 416, 478, 382, 5, 308, 15, 1264, 1344, 4761, 4761, 4761, 4761, 4761, 4761, 4761, 4761, 4761, 4761, 4761, 4761, 4761, 4761]\n"
     ]
    }
   ],
   "source": [
    "print(type(train_data),train_data[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 定义模型"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "index-to-vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding_pretrained = torch.tensor(np.load( './data/embedding_SougouNews.npz')[\"embeddings\"].astype('float32'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([4762, 300])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "embedding_pretrained.size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "num_class=len([x.strip() for x in open('./data/class.txt').readlines()])\n",
    "num_class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "class LSTM(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.embedding=nn.Embedding.from_pretrained(embedding_pretrained, freeze=False)\n",
    "        self.lstm=nn.LSTM(input_size=embedding_pretrained.size(1),hidden_size=128 ,num_layers=2,bidirectional=True, batch_first=True, dropout=0.2)\n",
    "        self.fc=nn.Linear(128*2,num_class)\n",
    "        \n",
    "    def forward(self,x):\n",
    "        out = self.embedding(x)  # out:[batch_size, seq_len, embedding]=[128, 32, 300],这也是LSTM层(batch_first=True)的输入格式\n",
    "        out, _ = self.lstm(out)#out接收output，_接收元组(h_n,c_n)\n",
    "        out = self.fc(out[:, -1, :])  # 句子最后时刻的 hidden state\n",
    "        return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "lstm=LSTM()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### loss、optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer=torch.optim.Adam(lstm.parameters(),lr=0.001)\n",
    "lstm_loss_fn=nn.functional.cross_entropy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### run"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def batch_data(data,label,batch_size=64,shuffle=True):\n",
    "    tensor_data, tensor_label = map(torch.tensor, (data, label))\n",
    "    dataset = TensorDataset(tensor_data, tensor_label)\n",
    "    dataloader=DataLoader(dataset,num_workers=4, batch_size=batch_size, shuffle=shuffle)\n",
    "    return dataloader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "device=torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_time_dif(start_time):\n",
    "    \"\"\"获取已使用时间\"\"\"\n",
    "    end_time = time.time()\n",
    "    time_dif = end_time - start_time\n",
    "    return timedelta(seconds=int(round(time_dif)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(model,data,label,num_epochs):\n",
    "    model.train()\n",
    "    train_iter=batch_data(data,label)\n",
    "    \n",
    "    for trains,labels in train_iter:\n",
    "        trains=trains.to(device)\n",
    "        labels=labels.to(device)\n",
    "        outputs= model(trains)\n",
    "        #print(outputs.shape)#torch.Size([batch_size, 10])\n",
    "        #获得一个batch数据的loss\n",
    "        loss=lstm_loss_fn(outputs, labels)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        model.zero_grad()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test(model,data,label):\n",
    "    model.eval()\n",
    "    test_loss=0\n",
    "    test_iter=batch_data(data,label)\n",
    "    accuracy=0\n",
    "    with torch.no_grad():\n",
    "        for trains,labels in test_iter:\n",
    "            trains=trains.to(device)\n",
    "            labels=labels.to(device)\n",
    "            outputs= model(trains)\n",
    "            loss = lstm_loss_fn(outputs, labels)\n",
    "            test_loss+=loss.item()\n",
    "            \n",
    "            #计算测试集的准确率,计算准确率时不能在gpu上计算，得转到cpu上\n",
    "            true = labels.data.cpu()\n",
    "            predic = torch.max(outputs.data, 1)[1].cpu()\n",
    "            train_acc = metrics.accuracy_score(true, predic)\n",
    "            accuracy+=train_acc\n",
    "        print(\"平均accuracy:\",accuracy/len(test_iter),\"平均loss:\",test_loss/len(test_iter))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "from math import log\n",
    "\n",
    "#定义信息熵计算\n",
    "def InformationEntropy(probability_list):\n",
    "    shannonEnt = 0.0\n",
    "    for val in probability_list:\n",
    "        shannonEnt -= val * log(val, 2)  # log base 2\n",
    "    return shannonEnt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "softmax = torch.nn.Softmax(dim=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def active_learning(model,train_data,train_label,unlabeled_data,unlabeled_label):\n",
    "    model.eval()\n",
    "    batch_size=64\n",
    "    with torch.no_grad():\n",
    "        unlabeled_iter=batch_data(unlabeled_data,unlabeled_label,batch_size,False)\n",
    "        res_dict={}\n",
    "        res_list=[]\n",
    "        index=0\n",
    "        for trains,labels in unlabeled_iter:\n",
    "            trains=trains.to(device)\n",
    "            labels=labels.to(device)\n",
    "            outputs = model(trains)#print(outputs.shape)#torch.Size([batch_size, 10])\n",
    "            #print(\"len(outputs):\",len(outputs))\n",
    "            for i in range(len(outputs)):\n",
    "                res_dict[index+i]=InformationEntropy(softmax(outputs[i]).cpu().detach().numpy().tolist())\n",
    "            index=index+batch_size\n",
    "        # 按v值降序降序,排序后变成元组(k,v)组成的list：[('key1', 3), ('key2', 2), ('key3', 1)]\n",
    "        res_dict=sorted(res_dict.items(),key=lambda x:x[1],reverse = True)\n",
    "        #print(res_dict)\n",
    "        #每次主动学习取出1000个值加入到标记数据集中，并从未标记数据集中删除\n",
    "        num=0\n",
    "        for con in res_dict:\n",
    "            if num >= 1000:\n",
    "                break\n",
    "            res_list.append(con[0])\n",
    "            num+=1\n",
    "        #往标注数据集中添加元素\n",
    "        for i in res_list:\n",
    "            train_data.append(unlabeled_data[i])\n",
    "            train_label.append(unlabeled_label[i])\n",
    "        #从无标注数据集中删除元素(先删角标大的)\n",
    "        for i in sorted(res_list, reverse=True):\n",
    "            del(unlabeled_data[i])\n",
    "            del(unlabeled_label[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def run(model,train_data,train_label,test_data,test_label,unlabeled_data,unlabeled_label,num_epochs):\n",
    "    model.to(device)\n",
    "    start_time=time.time()\n",
    "    \n",
    "    #run\n",
    "    for epoch in range(num_epochs):\n",
    "        #训练\n",
    "        print(\"第{}次train_data和train_label数量:\".format(epoch),len(train_data),len(train_label))\n",
    "        print('Epoch [{}/{}]'.format(epoch + 1, num_epochs),\"用时：\",get_time_dif(start_time))\n",
    "        train(model,train_data,train_label,num_epochs)\n",
    "        #使用测试数据测试模型效果\n",
    "        test(model,test_data,test_label)\n",
    "        #active learning\n",
    "        active_learning(model,train_data,train_label,unlabeled_data,unlabeled_label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "第0次train_data和train_label数量: 10000 10000\n",
      "Epoch [1/10] 用时： 0:00:00\n",
      "平均accuracy: 0.3443471337579618 平均loss: 1.6754197451719053\n",
      "第1次train_data和train_label数量: 11000 11000\n",
      "Epoch [2/10] 用时： 0:01:15\n",
      "平均accuracy: 0.46785429936305734 平均loss: 1.3338653156711797\n",
      "第2次train_data和train_label数量: 12000 12000\n",
      "Epoch [3/10] 用时： 0:02:25\n",
      "平均accuracy: 0.690187101910828 平均loss: 0.8854692407474396\n",
      "第3次train_data和train_label数量: 13000 13000\n",
      "Epoch [4/10] 用时： 0:03:35\n",
      "平均accuracy: 0.8050358280254777 平均loss: 0.6399551533210049\n",
      "第4次train_data和train_label数量: 14000 14000\n",
      "Epoch [5/10] 用时： 0:04:48\n",
      "平均accuracy: 0.826234076433121 平均loss: 0.5541558861732483\n",
      "第5次train_data和train_label数量: 15000 15000\n",
      "Epoch [6/10] 用时： 0:06:00\n",
      "平均accuracy: 0.8458399681528662 平均loss: 0.5054862117691404\n",
      "第6次train_data和train_label数量: 16000 16000\n",
      "Epoch [7/10] 用时： 0:07:16\n",
      "平均accuracy: 0.8442476114649682 平均loss: 0.5299968332242054\n",
      "第7次train_data和train_label数量: 17000 17000\n",
      "Epoch [8/10] 用时： 0:08:28\n",
      "平均accuracy: 0.8595740445859873 平均loss: 0.4721982267442023\n",
      "第8次train_data和train_label数量: 18000 18000\n",
      "Epoch [9/10] 用时： 0:09:38\n",
      "平均accuracy: 0.861265923566879 平均loss: 0.4801494795235859\n",
      "第9次train_data和train_label数量: 19000 19000\n",
      "Epoch [10/10] 用时： 0:10:56\n",
      "平均accuracy: 0.8642515923566879 平均loss: 0.48399723914398507\n"
     ]
    }
   ],
   "source": [
    "run(lstm,train_data,train_label,test_data,test_label,unlabeled_data,unlabeled_label,10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}