zz_prepocessing_details.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "import pickle\n",
    "import gzip\n",
    "import re\n",
    "import subprocess\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "import time "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Parsing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def Inst2ID(instruction):\n",
    "    global VocabDict\n",
    "\n",
    "    className = instruction.split('->')[0].split('$')[0]\n",
    "    if className in VocabDict.index:\n",
    "        return VocabDict[className]\n",
    "    else: \n",
    "        return None\n",
    "\n",
    "def parseMethodCallArguments(instruction : str) -> list:\n",
    "    instruction = instruction.split(': ')[-1]\n",
    "    method = instruction.replace('{', '').replace('}', '').replace('[', '').strip().split(' // ')[0].split(' ')[-1]\n",
    "\n",
    "    method = method.replace(';.', '->')\n",
    "    methodParts = method.split(':')\n",
    "    methodCall = methodParts[0]\n",
    "    if len(methodCall.split('/')[0]) == 2:\n",
    "        methodCall = 'class.local####'\n",
    "        \n",
    "    if len(methodParts) > 1:\n",
    "        methodArgs = methodParts[1].replace('(', '').replace(')', '').split(';')\n",
    "        methodArgs = [item for item in methodArgs if len(item) != 0 and item[0] == 'L']\n",
    "    else:\n",
    "        methodArgs = []\n",
    "        \n",
    "    mtdSplit = methodCall.split('->')\n",
    "    if len(mtdSplit) > 1:\n",
    "        methodCall = mtdSplit[0].split('$')[0] + '->' + mtdSplit[1].split('$')[0]\n",
    "                    \n",
    "    args = [methodCall] + methodArgs\n",
    "    args = [Inst2ID(argo) for argo in args]\n",
    "    argsIDs = np.array([argo for argo in args if argo != None], dtype='uint32')\n",
    "    \n",
    "    return argsIDs\n",
    "\n",
    "def parseDex(FileName : str) -> list:\n",
    "    global outputRoot\n",
    "    global inputRoot\n",
    "    \n",
    "    outputFilePath = outputRoot + FileName\n",
    "    inputFilePath  = inputRoot  + FileName\n",
    "\n",
    "    dex = subprocess.run(['dexdump', '-d', inputFilePath], stdout=subprocess.PIPE).stdout.decode(encoding=\"ISO-8859-1\")\n",
    "\n",
    "    condidatClasses = re.split(\"Class descriptor  : '\", dex)[1:]\n",
    "\n",
    "    collectedMethods = []\n",
    "\n",
    "    for currentClass in condidatClasses:\n",
    "        if currentClass.split('$')[0] not in VocabDict.index:\n",
    "            className = currentClass.split('\\n')[0][1:-2]\n",
    "            currentCondidatMethods = re.split('    #\\d', currentClass)\n",
    "            for method in currentCondidatMethods:\n",
    "                if method != '':\n",
    "                    Instructions = [item for item in method.split('\\n') if '|' in item]\n",
    "                    if len(Instructions) != 0:\n",
    "                        methodMeta = Instructions[0].split()[-1]\n",
    "                        Instructions = Instructions[1:]\n",
    "\n",
    "                        functionMethodCallsArgs    = np.concatenate([parseMethodCallArguments(line) for line in Instructions if ('method@' in line)] + [np.array([], dtype='uint32')])\n",
    "                                                \n",
    "                        collectedMethods.append((className, methodMeta, functionMethodCallsArgs, Instructions))\n",
    "\n",
    "    if len(collectedMethods) > 1:\n",
    "        colNames = ['className', 'methodMeta', 'functionMethodCallsArgs', 'Instructions']\n",
    "        df = pd.DataFrame(collectedMethods, columns=colNames)\n",
    "        #df['className'] = pd.factorize(df.className)[0]\n",
    "        #df['methodMeta'] = pd.factorize(df.methodMeta)[0]\n",
    "\n",
    "        df.to_parquet(outputFilePath)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Drebin Details"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5560\n"
     ]
    }
   ],
   "source": [
    "df = pd.read_msgpack('/ws/mnt/habouch/datasets/android_dataset/meta/drebin_meta.msg')\n",
    "fileNames  = df.sha256.to_list()\n",
    "\n",
    "import glob\n",
    "from multiprocessing import Pool\n",
    "mypool = Pool(processes=40)\n",
    "\n",
    "outputRoot = '/ws/papers/active/petadroid/code/output/drebinDetails/'\n",
    "inputRoot  = '/ws/mnt/habouch/datasets/android_dataset/samples/drebin_dataset/'\n",
    "\n",
    "VocabDict = pd.read_csv('dataset/InstructionDict.csv', index_col=0, names=['ID'])\n",
    "VocabDict = VocabDict.ID\n",
    "\n",
    "doneFileNames = [item.split('/')[-1] for item in glob.glob(outputRoot + '*')]\n",
    "todoFileNames = np.array(list(set(fileNames).difference(set(doneFileNames))))\n",
    "print(len(todoFileNames))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "mypool.map(parseDex, todoFileNames)\n",
    "mypool.close();\n",
    "mypool.join();"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### MiniZoo"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "minizoo_df = pd.read_csv('dataset/minizoo_df.csv')\n",
    "minizoo_df = minizoo_df.sample(5000, random_state=54)\n",
    "fileNames  = minizoo_df.sha256.to_list()\n",
    "\n",
    "import glob\n",
    "from multiprocessing import Pool\n",
    "mypool = Pool(processes=40)\n",
    "\n",
    "outputRoot = '/ws/papers/active/petadroid/code/output/minizoo/'\n",
    "inputRoot  = '/ws/mnt/habouch/datasets/zoo_dataset/'\n",
    "\n",
    "VocabDict = pd.read_csv('dataset/InstructionDict.csv', index_col=0, names=['ID'])\n",
    "VocabDict = VocabDict.ID\n",
    "\n",
    "#fileNames     = [fileName.strip() for fileName in open('dataset/minizoo_df.txt', 'r').readlines()]\n",
    "\n",
    "\n",
    "doneFileNames = [item.split('/')[-1] for item in glob.glob(outputRoot + '*')]\n",
    "todoFileNames = np.array(list(set(fileNames).difference(set(doneFileNames))))\n",
    "print(len(todoFileNames))"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "mypool.map(parseDex, todoFileNames)\n",
    "mypool.close();\n",
    "mypool.join();"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Drebin"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "df = pd.read_msgpack('/ws/mnt/habouch/datasets/android_dataset/meta/drebin_meta.msg')\n",
    "fileNames  = df.sha256.to_list()\n",
    "\n",
    "import glob\n",
    "from multiprocessing import Pool\n",
    "mypool = Pool(processes=40)\n",
    "\n",
    "outputRoot = '/ws/papers/active/petadroid/code/output/drebin/'\n",
    "inputRoot  = '/ws/mnt/habouch/datasets/android_dataset/samples/drebin_dataset/'\n",
    "\n",
    "VocabDict = pd.read_csv('dataset/InstructionDict.csv', index_col=0, names=['ID'])\n",
    "VocabDict = VocabDict.ID\n",
    "\n",
    "doneFileNames = [item.split('/')[-1] for item in glob.glob(outputRoot + '*')]\n",
    "todoFileNames = np.array(list(set(fileNames).difference(set(doneFileNames))))\n",
    "print(len(todoFileNames))"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "mypool.map(parseDex, todoFileNames)\n",
    "mypool.close();\n",
    "mypool.join();"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### MalGenome"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "df = pd.read_msgpack('/ws/mnt/habouch/datasets/android_dataset/meta/genome_meta.msg')\n",
    "fileNames = df.sha1.to_list()\n",
    "\n",
    "import glob\n",
    "from multiprocessing import Pool\n",
    "mypool = Pool(processes=40)\n",
    "\n",
    "outputRoot = '/ws/papers/active/petadroid/code/output/genome/'\n",
    "inputRoot  = '/ws/mnt/habouch/datasets/android_dataset/samples/genome_dataset/'\n",
    "\n",
    "VocabDict = pd.read_csv('dataset/InstructionDict.csv', index_col=0, names=['ID'])\n",
    "VocabDict = VocabDict.ID\n",
    "\n",
    "doneFileNames = [item.split('/')[-1] for item in glob.glob(outputRoot + '*')]\n",
    "todoFileNames = np.array(list(set(fileNames).difference(set(doneFileNames))))\n",
    "print(len(todoFileNames))"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "mypool.map(parseDex, todoFileNames)\n",
    "mypool.close();\n",
    "mypool.join();"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Maldozer"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "df = pd.read_msgpack('/ws/mnt/habouch/datasets/android_dataset/meta/maldozer_meta.msg')\n",
    "fileNames = df.md5.to_list()\n",
    "\n",
    "import glob\n",
    "from multiprocessing import Pool\n",
    "mypool = Pool(processes=40)\n",
    "\n",
    "outputRoot = '/ws/papers/active/petadroid/code/output/maldozer/'\n",
    "inputRoot  = '/ws/mnt/habouch/datasets/android_dataset/samples/maldozer_dataset/'\n",
    "\n",
    "VocabDict = pd.read_csv('dataset/InstructionDict.csv', index_col=0, names=['ID'])\n",
    "VocabDict = VocabDict.ID\n",
    "\n",
    "doneFileNames = [item.split('/')[-1] for item in glob.glob(outputRoot + '*')]\n",
    "todoFileNames = np.array(list(set(fileNames).difference(set(doneFileNames))))\n",
    "print(len(todoFileNames))"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "mypool.map(parseDex, todoFileNames)\n",
    "mypool.close();\n",
    "mypool.join();"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### AMD"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "df = pd.read_msgpack('/ws/mnt/habouch/datasets/android_dataset/meta/amd_meta.msg')\n",
    "fileNames = df.md5.to_list()"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "import glob\n",
    "from multiprocessing import Pool\n",
    "mypool = Pool(processes=40)\n",
    "\n",
    "outputRoot = '/ws/papers/active/petadroid/code/output/amd/'\n",
    "inputRoot  = '/ws/mnt/habouch/datasets/android_dataset/samples/amd_dataset/'\n",
    "\n",
    "VocabDict = pd.read_csv('dataset/InstructionDict.csv', index_col=0, names=['ID'])\n",
    "VocabDict = VocabDict.ID\n",
    "\n",
    "doneFileNames = [item.split('/')[-1] for item in glob.glob(outputRoot + '*')]\n",
    "todoFileNames = np.array(list(set(fileNames).difference(set(doneFileNames))))\n",
    "print(len(todoFileNames))"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "mypool.map(parseDex, todoFileNames)\n",
    "mypool.close();\n",
    "mypool.join();"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Benign"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('/ws/mnt/habouch/datasets/zoows/history/latest.csv')\n",
    "df['sha256'] = df.sha256.str.lower()\n",
    "df = df.loc[df.vt_detection == 0]\n",
    "fileNames = df.sha256.sample(50000, random_state=54).to_list()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1263\n"
     ]
    }
   ],
   "source": [
    "import glob\n",
    "from multiprocessing import Pool\n",
    "mypool = Pool(processes=40)\n",
    "\n",
    "outputRoot = '/ws/papers/active/petadroid/code/output/benign/'\n",
    "inputRoot  = '/ws/mnt/habouch/datasets/zoo_dataset/'\n",
    "\n",
    "VocabDict = pd.read_csv('dataset/InstructionDict.csv', index_col=0, names=['ID'])\n",
    "VocabDict = VocabDict.ID\n",
    "\n",
    "doneFileNames = [item.split('/')[-1] for item in glob.glob(outputRoot + '*')]\n",
    "todoFileNames = np.array(list(set(fileNames).difference(set(doneFileNames))))\n",
    "print(len(todoFileNames))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "mypool.map(parseDex, todoFileNames)\n",
    "mypool.close();\n",
    "mypool.join();"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Obfuscated Malgenome (praguard)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_msgpack('/ws/mnt/habouch/datasets/android_dataset/android_meta/praguard_meta_df.msg')\n",
    "fileNames = df.md5.to_list()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10370\n"
     ]
    }
   ],
   "source": [
    "import glob\n",
    "from multiprocessing import Pool\n",
    "mypool = Pool(processes=40)\n",
    "\n",
    "outputRoot = '/ws/papers/active/petadroid/code/output/praguard/'\n",
    "inputRoot  = '/ws/mnt/habouch/datasets/android_dataset/samples/praguard_dataset/files/'\n",
    "\n",
    "VocabDict = pd.read_csv('dataset/InstructionDict.csv', index_col=0, names=['ID'])\n",
    "VocabDict = VocabDict.ID\n",
    "\n",
    "doneFileNames = [item.split('/')[-1] for item in glob.glob(outputRoot + '*')]\n",
    "todoFileNames = np.array(list(set(fileNames).difference(set(doneFileNames))))\n",
    "print(len(todoFileNames))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "mypool.map(parseDex, todoFileNames)\n",
    "mypool.close();\n",
    "mypool.join();"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Obfuscated Drebin (charmilion)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['000146786111132694d5f0d8f89e17c6',\n",
       " '000379d1a9201758fdae17ccfad374f5',\n",
       " '0006d3fc1fcd3277451fe676fe13b1c9',\n",
       " '0008ac38b8c073e5bab4a7c8b4120fcd',\n",
       " '000909e08eeee76eb4a00a922333da62']"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_msgpack('/ws/mnt/habouch/datasets/android_dataset/meta/drebin_obfus_meta.msg')\n",
    "fileNames = df.md5.to_list()\n",
    "fileNames[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "49625\n"
     ]
    }
   ],
   "source": [
    "import glob\n",
    "from multiprocessing import Pool\n",
    "mypool = Pool(processes=40)\n",
    "\n",
    "outputRoot = '/ws/papers/active/petadroid/code/output/charmi/'\n",
    "inputRoot  = '/ws/mnt/habouch/datasets/android_dataset/samples/drebin_obfus_dataset/files/'\n",
    "\n",
    "VocabDict = pd.read_csv('dataset/InstructionDict.csv', index_col=0, names=['ID'])\n",
    "VocabDict = VocabDict.ID\n",
    "\n",
    "doneFileNames = [item.split('/')[-1] for item in glob.glob(outputRoot + '*')]\n",
    "todoFileNames = np.array(list(set(fileNames).difference(set(doneFileNames))))\n",
    "print(len(todoFileNames))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "mypool.map(parseDex, todoFileNames)\n",
    "mypool.close();\n",
    "mypool.join();"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Obfuscated benign (charmilion)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['0001928ff53c44ca2af68152af336a25',\n",
       " '0002461ffb54834a257c9d4eecd2ebb2',\n",
       " '000315d021e6ba52e1bc1d30ffe592b5',\n",
       " '000497a806c495c4b7b385b155e72ce6',\n",
       " '000520fe345eae9567fdb283161bbe28']"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_msgpack('/ws/mnt/habouch/datasets/android_dataset/meta/benign_obfus_meta_df.msg')\n",
    "fileNames = df.md5.to_list()\n",
    "fileNames[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "41067\n"
     ]
    }
   ],
   "source": [
    "import glob\n",
    "from multiprocessing import Pool\n",
    "mypool = Pool(processes=40)\n",
    "\n",
    "outputRoot = '/ws/papers/active/petadroid/code/output/benign_obfus/'\n",
    "inputRoot  = '/ws/mnt/habouch/datasets/android_dataset/samples/benign_obfus/files/'\n",
    "\n",
    "VocabDict = pd.read_csv('dataset/InstructionDict.csv', index_col=0, names=['ID'])\n",
    "VocabDict = VocabDict.ID\n",
    "\n",
    "doneFileNames = [item.split('/')[-1] for item in glob.glob(outputRoot + '*')]\n",
    "todoFileNames = np.array(list(set(fileNames).difference(set(doneFileNames))))\n",
    "print(len(todoFileNames))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "mypool.map(parseDex, todoFileNames)\n",
    "mypool.close();\n",
    "mypool.join();"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Not Obfuscated benign"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['cf23b28b6e396b70baafa11fb27d3c9792590b6e9dcb0bbf9d0f70b0a2742ec6',\n",
       " '0866ea2f7b8ea45fe9947ab9c44d45b2424cfca0ba2a9f21686f67cc95ff6df8',\n",
       " '8822d55bb6a75c8745d426653ecd8041e4a5e84b062062f53b9ae542bb8afbc7',\n",
       " '273979aadfda6f72ee3c6d02aed768a18787afdc4dd3a77c2df6e473d3909d66',\n",
       " '98a634d1ff7cc79b675f2406a1dfea96a2617d65d8ee4cc418b86368d274cb38']"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_msgpack('/ws/mnt/habouch/datasets/android_dataset/meta/benign_notobfus_meta_df.msg')\n",
    "fileNames = df.sha256.str.lower().to_list()\n",
    "fileNames[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5500\n"
     ]
    }
   ],
   "source": [
    "import glob\n",
    "from multiprocessing import Pool\n",
    "mypool = Pool(processes=40)\n",
    "\n",
    "outputRoot = '/ws/papers/active/petadroid/code/output/benign_notobfus/'\n",
    "inputRoot  = '/ws/mnt/habouch/datasets/zoo_dataset/'\n",
    "\n",
    "VocabDict = pd.read_csv('dataset/InstructionDict.csv', index_col=0, names=['ID'])\n",
    "VocabDict = VocabDict.ID\n",
    "\n",
    "doneFileNames = [item.split('/')[-1] for item in glob.glob(outputRoot + '*')]\n",
    "todoFileNames = np.array(list(set(fileNames).difference(set(doneFileNames))))\n",
    "print(len(todoFileNames))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "mypool.map(parseDex, todoFileNames)\n",
    "mypool.close();\n",
    "mypool.join();"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### VirusShare"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['18750d3a30a52e508aa4a03fdada630e',\n",
       " '1874ed85ba7160d4b8002f682d8b3102',\n",
       " '1873ebb0538fc2656ea67aa93815dba6',\n",
       " '1873ea2a9e071290ff5a2d1b90efbc4e',\n",
       " '187347fd95d5675ac183f86f8409789b']"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_msgpack('/ws/mnt/habouch/datasets/android_dataset/meta/vshare_meta.msg')\n",
    "fileNames = df.md5.to_list()\n",
    "fileNames[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "30340\n"
     ]
    }
   ],
   "source": [
    "import glob\n",
    "from multiprocessing import Pool\n",
    "mypool = Pool(processes=40)\n",
    "\n",
    "outputRoot = '/ws/papers/active/petadroid/code/output/vshare/'\n",
    "inputRoot  = '/ws/mnt/habouch/datasets/android_dataset/samples/vshare_dataset/'\n",
    "\n",
    "VocabDict = pd.read_csv('dataset/InstructionDict.csv', index_col=0, names=['ID'])\n",
    "VocabDict = VocabDict.ID\n",
    "\n",
    "doneFileNames = [item.split('/')[-1] for item in glob.glob(outputRoot + '*')]\n",
    "todoFileNames = np.array(list(set(fileNames).difference(set(doneFileNames))))\n",
    "print(len(todoFileNames))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "mypool.map(parseDex, todoFileNames)\n",
    "mypool.close();\n",
    "mypool.join();"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Zoo Malware"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('/ws/mnt/habouch/datasets/zoows/history/latest.csv')\n",
    "df['sha256'] = df.sha256.str.lower()\n",
    "df = df.loc[df.vt_detection != 0]\n",
    "fileNames = df.sha256.sample(50000, random_state=54).to_list()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "50000\n"
     ]
    }
   ],
   "source": [
    "import glob\n",
    "from multiprocessing import Pool\n",
    "mypool = Pool(processes=40)\n",
    "\n",
    "outputRoot = '/ws/papers/active/petadroid/code/output/zoomalware/'\n",
    "inputRoot  = '/ws/mnt/habouch/datasets/zoo_dataset/'\n",
    "\n",
    "VocabDict = pd.read_csv('dataset/InstructionDict.csv', index_col=0, names=['ID'])\n",
    "VocabDict = VocabDict.ID\n",
    "\n",
    "doneFileNames = [item.split('/')[-1] for item in glob.glob(outputRoot + '*')]\n",
    "todoFileNames = np.array(list(set(fileNames).difference(set(doneFileNames))))\n",
    "print(len(todoFileNames))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "mypool.map(parseDex, todoFileNames)\n",
    "mypool.close();\n",
    "mypool.join();"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "pysciws",
   "language": "python",
   "name": "sciws"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}