Testing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>_id</th>\n",
       "      <th>contributors</th>\n",
       "      <th>coordinates</th>\n",
       "      <th>created_at</th>\n",
       "      <th>entities</th>\n",
       "      <th>extended_entities</th>\n",
       "      <th>favorite_count</th>\n",
       "      <th>favorited</th>\n",
       "      <th>geo</th>\n",
       "      <th>id</th>\n",
       "      <th>...</th>\n",
       "      <th>quoted_status_id_str</th>\n",
       "      <th>retweet_count</th>\n",
       "      <th>retweeted</th>\n",
       "      <th>retweeted_status</th>\n",
       "      <th>source</th>\n",
       "      <th>text</th>\n",
       "      <th>full_text</th>\n",
       "      <th>truncated</th>\n",
       "      <th>user</th>\n",
       "      <th>withheld_in_countries</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ObjectId(59dbede4e6e465a2d67a1062)</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Sun Oct 22 10:34:06 +0000 2017</td>\n",
       "      <td>{\"hashtags\":[{\"text\":\"BanIslam\",\"indices\":[93,...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>9.220484e+17</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4</td>\n",
       "      <td>False</td>\n",
       "      <td>{\"created_at\":\"Fri Oct 20 09:33:45 +0000 2017\"...</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>RT @Private34349909: @AmyMek @Dab7One 1/ @real...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>{\"id\":8.009259051368776e+17,\"id_str\":\"80092590...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ObjectId(59dbede4e6e465a2d67a106c)</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Sun Oct 22 10:13:40 +0000 2017</td>\n",
       "      <td>{\"hashtags\":[{\"text\":\"muslim\",\"indices\":[16,23...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>9.220433e+17</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>{\"created_at\":\"Sat Oct 21 18:37:07 +0000 2017\"...</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/android\" ...</td>\n",
       "      <td>RT @ensine: All #muslim rulers were savages bc...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>{\"id\":375089876,\"id_str\":\"375089876\",\"name\":\"द...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ObjectId(59dbf266e6e465a2d67a4810)</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Sun Oct 22 07:50:47 +0000 2017</td>\n",
       "      <td>{\"hashtags\":[{\"text\":\"Raqqa\",\"indices\":[37,43]...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>9.220073e+17</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>172</td>\n",
       "      <td>False</td>\n",
       "      <td>{\"created_at\":\"Fri Oct 20 23:15:45 +0000 2017\"...</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/android\" ...</td>\n",
       "      <td>RT @SLandinSoCal: Liberated Women of #Raqqa‼️R...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>{\"id\":7.092975641898926e+17,\"id_str\":\"70929756...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ObjectId(59dbf266e6e465a2d67a4828)</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Sun Oct 22 10:38:23 +0000 2017</td>\n",
       "      <td>{\"hashtags\":[],\"symbols\":[],\"user_mentions\":[{...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>9.220495e+17</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>&lt;a href=\"http://twitter.com\" rel=\"nofollow\"&gt;Tw...</td>\n",
       "      <td>@Stormtroepen @cdavandaag @sybrandbuma @gertja...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>True</td>\n",
       "      <td>{\"id\":1340408646,\"id_str\":\"1340408646\",\"name\":...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ObjectId(59e8e630e6e465a2d6238c9c)</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Sun Oct 22 08:59:53 +0000 2017</td>\n",
       "      <td>{\"hashtags\":[{\"text\":\"RT\",\"indices\":[51,54]},{...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>9.220247e+17</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>{\"created_at\":\"Sun Oct 22 01:40:55 +0000 2017\"...</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/android\" ...</td>\n",
       "      <td>RT @PoliticalIslam: Sharia at odds with Articl...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>{\"id\":150595824,\"id_str\":\"150595824\",\"name\":\"🇮...</td>\n",
       "      <td>[\"DE\"]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 33 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  _id  contributors coordinates  \\\n",
       "0  ObjectId(59dbede4e6e465a2d67a1062)           NaN         NaN   \n",
       "1  ObjectId(59dbede4e6e465a2d67a106c)           NaN         NaN   \n",
       "2  ObjectId(59dbf266e6e465a2d67a4810)           NaN         NaN   \n",
       "3  ObjectId(59dbf266e6e465a2d67a4828)           NaN         NaN   \n",
       "4  ObjectId(59e8e630e6e465a2d6238c9c)           NaN         NaN   \n",
       "\n",
       "                       created_at  \\\n",
       "0  Sun Oct 22 10:34:06 +0000 2017   \n",
       "1  Sun Oct 22 10:13:40 +0000 2017   \n",
       "2  Sun Oct 22 07:50:47 +0000 2017   \n",
       "3  Sun Oct 22 10:38:23 +0000 2017   \n",
       "4  Sun Oct 22 08:59:53 +0000 2017   \n",
       "\n",
       "                                            entities extended_entities  \\\n",
       "0  {\"hashtags\":[{\"text\":\"BanIslam\",\"indices\":[93,...               NaN   \n",
       "1  {\"hashtags\":[{\"text\":\"muslim\",\"indices\":[16,23...               NaN   \n",
       "2  {\"hashtags\":[{\"text\":\"Raqqa\",\"indices\":[37,43]...               NaN   \n",
       "3  {\"hashtags\":[],\"symbols\":[],\"user_mentions\":[{...               NaN   \n",
       "4  {\"hashtags\":[{\"text\":\"RT\",\"indices\":[51,54]},{...               NaN   \n",
       "\n",
       "   favorite_count  favorited  geo            id  ...  quoted_status_id_str  \\\n",
       "0               0      False  NaN  9.220484e+17  ...                   NaN   \n",
       "1               0      False  NaN  9.220433e+17  ...                   NaN   \n",
       "2               0      False  NaN  9.220073e+17  ...                   NaN   \n",
       "3               0      False  NaN  9.220495e+17  ...                   NaN   \n",
       "4               0      False  NaN  9.220247e+17  ...                   NaN   \n",
       "\n",
       "  retweet_count  retweeted                                   retweeted_status  \\\n",
       "0             4      False  {\"created_at\":\"Fri Oct 20 09:33:45 +0000 2017\"...   \n",
       "1            10      False  {\"created_at\":\"Sat Oct 21 18:37:07 +0000 2017\"...   \n",
       "2           172      False  {\"created_at\":\"Fri Oct 20 23:15:45 +0000 2017\"...   \n",
       "3             1      False                                                NaN   \n",
       "4            25      False  {\"created_at\":\"Sun Oct 22 01:40:55 +0000 2017\"...   \n",
       "\n",
       "                                              source  \\\n",
       "0  <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "1  <a href=\"http://twitter.com/download/android\" ...   \n",
       "2  <a href=\"http://twitter.com/download/android\" ...   \n",
       "3  <a href=\"http://twitter.com\" rel=\"nofollow\">Tw...   \n",
       "4  <a href=\"http://twitter.com/download/android\" ...   \n",
       "\n",
       "                                                text  full_text truncated  \\\n",
       "0  RT @Private34349909: @AmyMek @Dab7One 1/ @real...        NaN     False   \n",
       "1  RT @ensine: All #muslim rulers were savages bc...        NaN     False   \n",
       "2  RT @SLandinSoCal: Liberated Women of #Raqqa‼️R...        NaN     False   \n",
       "3  @Stormtroepen @cdavandaag @sybrandbuma @gertja...        NaN      True   \n",
       "4  RT @PoliticalIslam: Sharia at odds with Articl...        NaN     False   \n",
       "\n",
       "                                                user withheld_in_countries  \n",
       "0  {\"id\":8.009259051368776e+17,\"id_str\":\"80092590...                   NaN  \n",
       "1  {\"id\":375089876,\"id_str\":\"375089876\",\"name\":\"द...                   NaN  \n",
       "2  {\"id\":7.092975641898926e+17,\"id_str\":\"70929756...                   NaN  \n",
       "3  {\"id\":1340408646,\"id_str\":\"1340408646\",\"name\":...                   NaN  \n",
       "4  {\"id\":150595824,\"id_str\":\"150595824\",\"name\":\"🇮...                [\"DE\"]  \n",
       "\n",
       "[5 rows x 33 columns]"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_1 = pd.read_csv('../noislamophobia-dataset-50k.csv')\n",
    "df_2 = pd.read_csv('../noislamophobia-dataset-75k.csv')\n",
    "df = pd.concat([df_1,df_2])\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['_id', 'contributors', 'coordinates', 'created_at', 'entities',\n",
       "       'extended_entities', 'favorite_count', 'favorited', 'geo', 'id',\n",
       "       'id_str', 'in_reply_to_screen_name', 'in_reply_to_status_id',\n",
       "       'in_reply_to_status_id_str', 'in_reply_to_user_id',\n",
       "       'in_reply_to_user_id_str', 'is_quote_status', 'lang', 'metadata',\n",
       "       'place', 'possibly_sensitive', 'quoted_status', 'quoted_status_id',\n",
       "       'quoted_status_id_str', 'retweet_count', 'retweeted',\n",
       "       'retweeted_status', 'source', 'text', 'full_text', 'truncated', 'user',\n",
       "       'withheld_in_countries'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "df = df.drop_duplicates()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'{\"hashtags\":[{\"text\":\"BanIslam\",\"indices\":[93,102]},{\"text\":\"DeportALLmuslims\",\"indices\":[109,126]}],\"symbols\":[],\"user_mentions\":[{\"screen_name\":\"Private34349909\",\"name\":\"Private User\",\"id\":7.976360710643794e+17,\"id_str\":\"797636071064379392\",\"indices\":[3,19]},{\"screen_name\":\"AmyMek\",\"name\":\"Amy Mek\",\"id\":954124423,\"id_str\":\"954124423\",\"indices\":[21,28]},{\"screen_name\":\"Dab7One\",\"name\":\"Dabney 1\",\"id\":2.850722943e+09,\"id_str\":\"2850722943\",\"indices\":[29,37]},{\"screen_name\":\"realDonaldTrump\",\"name\":\"Donald J. Trump\",\"id\":25073877,\"id_str\":\"25073877\",\"indices\":[41,57]},{\"screen_name\":\"seanhannity\",\"name\":\"Sean Hannity\",\"id\":41634520,\"id_str\":\"41634520\",\"indices\":[58,70]},{\"screen_name\":\"TuckerCarlson\",\"name\":\"Tucker Carlson\",\"id\":22703645,\"id_str\":\"22703645\",\"indices\":[71,85]}],\"urls\":[]}'"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()['entities'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(119019, 33)"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Sun Oct 22 10:34:06 +0000 2017'"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['created_at'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{\"type\":\"Point\",\"coordinates\":[55.1639,-1.58639]}            1\n",
       "{\"type\":\"Point\",\"coordinates\":[26.12046647,-80.1312959]}     1\n",
       "{\"type\":\"Point\",\"coordinates\":[36.99099099,-85.94723833]}    1\n",
       "Name: geo, dtype: int64"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['geo'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'RT @Private34349909: @AmyMek @Dab7One 1/ @realDonaldTrump @seanhannity @TuckerCarlson PLEASE #BanIslam &amp; #DeportALLmuslims They do NOT meet…'"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['text'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(119019, 31)"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_na = df.dropna(axis=1, thresh=.75)\n",
    "df_na.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{\"media\":[{\"id\":9.392413551922299e+17,\"id_str\":\"939241355192229888\",\"indices\":[65,88],\"media_url\":\"http://pbs.twimg.com/media/DQjbCUaW4AArDbD.jpg\",\"media_url_https\":\"https://pbs.twimg.com/media/DQjbCUaW4AArDbD.jpg\",\"url\":\"https://t.co/dEr4552ajK\",\"display_url\":\"pic.twitter.com/dEr4552ajK\",\"expanded_url\":\"https://twitter.com/builditnow/status/939241536960782342/photo/1\",\"type\":\"photo\",\"sizes\":{\"small\":{\"w\":680,\"h\":340,\"resize\":\"fit\"},\"medium\":{\"w\":750,\"h\":375,\"resize\":\"fit\"},\"large\":{\"w\":750,\"h\":375,\"resize\":\"fit\"},\"thumb\":{\"w\":150,\"h\":150,\"resize\":\"crop\"}},\"source_status_id\":9.392415369607823e+17,\"source_status_id_str\":\"939241536960782342\",\"source_user_id\":9.211005303568384e+17,\"source_user_id_str\":\"921100530356838400\"}]}                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        230\n",
       "{\"media\":[{\"id\":9.581689240627978e+17,\"id_str\":\"958168924062797824\",\"indices\":[70,93],\"media_url\":\"http://pbs.twimg.com/media/DUwZj1GU0AAOwDh.jpg\",\"media_url_https\":\"https://pbs.twimg.com/media/DUwZj1GU0AAOwDh.jpg\",\"url\":\"https://t.co/bTWlC7EjUe\",\"display_url\":\"pic.twitter.com/bTWlC7EjUe\",\"expanded_url\":\"https://twitter.com/builditnow/status/958168954274541568/photo/1\",\"type\":\"photo\",\"sizes\":{\"thumb\":{\"w\":150,\"h\":150,\"resize\":\"crop\"},\"medium\":{\"w\":818,\"h\":487,\"resize\":\"fit\"},\"large\":{\"w\":818,\"h\":487,\"resize\":\"fit\"},\"small\":{\"w\":680,\"h\":405,\"resize\":\"fit\"}},\"source_status_id\":9.581689542745416e+17,\"source_status_id_str\":\"958168954274541568\",\"source_user_id\":9.211005303568384e+17,\"source_user_id_str\":\"921100530356838400\"}]}                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        217\n",
       "{\"media\":[{\"id\":9.468503766486221e+17,\"id_str\":\"946850376648622080\",\"indices\":[115,138],\"media_url\":\"http://pbs.twimg.com/media/DSPjZqOWAAAZseo.jpg\",\"media_url_https\":\"https://pbs.twimg.com/media/DSPjZqOWAAAZseo.jpg\",\"url\":\"https://t.co/xXEUtsK2EP\",\"display_url\":\"pic.twitter.com/xXEUtsK2EP\",\"expanded_url\":\"https://twitter.com/builditnow/status/946850407934029824/photo/1\",\"type\":\"photo\",\"sizes\":{\"thumb\":{\"w\":150,\"h\":150,\"resize\":\"crop\"},\"medium\":{\"w\":962,\"h\":947,\"resize\":\"fit\"},\"large\":{\"w\":962,\"h\":947,\"resize\":\"fit\"},\"small\":{\"w\":680,\"h\":669,\"resize\":\"fit\"}},\"source_status_id\":9.468504079340298e+17,\"source_status_id_str\":\"946850407934029824\",\"source_user_id\":9.211005303568384e+17,\"source_user_id_str\":\"921100530356838400\"},{\"id\":9.468503928723087e+17,\"id_str\":\"946850392872308741\",\"indices\":[115,138],\"media_url\":\"http://pbs.twimg.com/media/DSPjamqX0AUT3sX.jpg\",\"media_url_https\":\"https://pbs.twimg.com/media/DSPjamqX0AUT3sX.jpg\",\"url\":\"https://t.co/xXEUtsK2EP\",\"display_url\":\"pic.twitter.com/xXEUtsK2EP\",\"expanded_url\":\"https://twitter.com/builditnow/status/946850407934029824/photo/1\",\"type\":\"photo\",\"sizes\":{\"thumb\":{\"w\":150,\"h\":150,\"resize\":\"crop\"},\"small\":{\"w\":680,\"h\":314,\"resize\":\"fit\"},\"large\":{\"w\":808,\"h\":373,\"resize\":\"fit\"},\"medium\":{\"w\":808,\"h\":373,\"resize\":\"fit\"}},\"source_status_id\":9.468504079340298e+17,\"source_status_id_str\":\"946850407934029824\",\"source_user_id\":9.211005303568384e+17,\"source_user_id_str\":\"921100530356838400\"}]}    116\n",
       "{\"media\":[{\"id\":1.0000618387747512e+18,\"id_str\":\"1000061838774751232\",\"indices\":[99,122],\"media_url\":\"http://pbs.twimg.com/media/DeDu8Z2X0AApQcG.jpg\",\"media_url_https\":\"https://pbs.twimg.com/media/DeDu8Z2X0AApQcG.jpg\",\"url\":\"https://t.co/TzS74ABl5n\",\"display_url\":\"pic.twitter.com/TzS74ABl5n\",\"expanded_url\":\"https://twitter.com/daapdewaarheid/status/1000061851491819520/photo/1\",\"type\":\"photo\",\"sizes\":{\"thumb\":{\"w\":150,\"h\":150,\"resize\":\"crop\"},\"small\":{\"w\":680,\"h\":680,\"resize\":\"fit\"},\"medium\":{\"w\":1000,\"h\":1000,\"resize\":\"fit\"},\"large\":{\"w\":1000,\"h\":1000,\"resize\":\"fit\"}},\"source_status_id\":1.0000618514918195e+18,\"source_status_id_str\":\"1000061851491819520\",\"source_user_id\":9.315941420729262e+17,\"source_user_id_str\":\"931594142072926209\"}]}                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           70\n",
       "{\"media\":[{\"id\":9.290551052567716e+17,\"id_str\":\"929055105256771589\",\"indices\":[93,116],\"media_url\":\"http://pbs.twimg.com/media/DOSqsiWX0AUE9nO.jpg\",\"media_url_https\":\"https://pbs.twimg.com/media/DOSqsiWX0AUE9nO.jpg\",\"url\":\"https://t.co/DXE2v8ZYDd\",\"display_url\":\"pic.twitter.com/DXE2v8ZYDd\",\"expanded_url\":\"https://twitter.com/builditnow/status/929055290359795712/photo/1\",\"type\":\"photo\",\"sizes\":{\"large\":{\"w\":1439,\"h\":856,\"resize\":\"fit\"},\"small\":{\"w\":680,\"h\":405,\"resize\":\"fit\"},\"thumb\":{\"w\":150,\"h\":150,\"resize\":\"crop\"},\"medium\":{\"w\":1200,\"h\":714,\"resize\":\"fit\"}},\"source_status_id\":9.290552903597957e+17,\"source_status_id_str\":\"929055290359795712\",\"source_user_id\":9.211005303568384e+17,\"source_user_id_str\":\"921100530356838400\"}]}                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      68\n",
       "                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                  ... \n",
       "{\"media\":[{\"id\":1.0389802982249882e+18,\"id_str\":\"1038980298224988160\",\"indices\":[92,115],\"media_url\":\"http://pbs.twimg.com/media/DmszE5RUcAAZP6s.jpg\",\"media_url_https\":\"https://pbs.twimg.com/media/DmszE5RUcAAZP6s.jpg\",\"url\":\"https://t.co/9fxotm5S3L\",\"display_url\":\"pic.twitter.com/9fxotm5S3L\",\"expanded_url\":\"https://twitter.com/CatFertitta/status/1038980494225039360/photo/1\",\"type\":\"photo\",\"sizes\":{\"thumb\":{\"w\":150,\"h\":150,\"resize\":\"crop\"},\"medium\":{\"w\":768,\"h\":960,\"resize\":\"fit\"},\"large\":{\"w\":768,\"h\":960,\"resize\":\"fit\"},\"small\":{\"w\":544,\"h\":680,\"resize\":\"fit\"}},\"source_status_id\":1.0389804942250394e+18,\"source_status_id_str\":\"1038980494225039360\",\"source_user_id\":9.834436739742147e+17,\"source_user_id_str\":\"983443673974214656\"}]}                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   1\n",
       "{\"media\":[{\"id\":9.72088645929771e+17,\"id_str\":\"972088645929771010\",\"indices\":[110,133],\"media_url\":\"http://pbs.twimg.com/media/DX2NeR7VoAIPAf4.jpg\",\"media_url_https\":\"https://pbs.twimg.com/media/DX2NeR7VoAIPAf4.jpg\",\"url\":\"https://t.co/LI8ElZdv6x\",\"display_url\":\"pic.twitter.com/LI8ElZdv6x\",\"expanded_url\":\"https://twitter.com/libocrisy/status/972088649192886274/photo/1\",\"type\":\"photo\",\"sizes\":{\"thumb\":{\"w\":150,\"h\":150,\"resize\":\"crop\"},\"large\":{\"w\":722,\"h\":307,\"resize\":\"fit\"},\"medium\":{\"w\":722,\"h\":307,\"resize\":\"fit\"},\"small\":{\"w\":680,\"h\":289,\"resize\":\"fit\"}}}]}                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                1\n",
       "{\"media\":[{\"id\":9.295672096675512e+17,\"id_str\":\"929567209667551232\",\"indices\":[113,136],\"media_url\":\"http://pbs.twimg.com/media/DOZ8c78V4AAmLiW.jpg\",\"media_url_https\":\"https://pbs.twimg.com/media/DOZ8c78V4AAmLiW.jpg\",\"url\":\"https://t.co/X1t4IQNhlj\",\"display_url\":\"pic.twitter.com/X1t4IQNhlj\",\"expanded_url\":\"https://twitter.com/CatholicsTweet/status/929567211714355200/photo/1\",\"type\":\"photo\",\"sizes\":{\"medium\":{\"w\":1149,\"h\":1161,\"resize\":\"fit\"},\"large\":{\"w\":1149,\"h\":1161,\"resize\":\"fit\"},\"small\":{\"w\":673,\"h\":680,\"resize\":\"fit\"},\"thumb\":{\"w\":150,\"h\":150,\"resize\":\"crop\"}}}]}                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      1\n",
       "{\"media\":[{\"id\":1.0196964759352975e+18,\"id_str\":\"1019696475935297536\",\"indices\":[51,74],\"media_url\":\"http://pbs.twimg.com/media/DiawitLXUAAVGdK.jpg\",\"media_url_https\":\"https://pbs.twimg.com/media/DiawitLXUAAVGdK.jpg\",\"url\":\"https://t.co/pUAfcsgyt8\",\"display_url\":\"pic.twitter.com/pUAfcsgyt8\",\"expanded_url\":\"https://twitter.com/Terra_Eri/status/1019696488904036354/photo/1\",\"type\":\"photo\",\"sizes\":{\"thumb\":{\"w\":150,\"h\":150,\"resize\":\"crop\"},\"large\":{\"w\":720,\"h\":960,\"resize\":\"fit\"},\"small\":{\"w\":510,\"h\":680,\"resize\":\"fit\"},\"medium\":{\"w\":720,\"h\":960,\"resize\":\"fit\"}}}]}                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              1\n",
       "{\"media\":[{\"id\":1.257258285918552e+18,\"id_str\":\"1257258285918552066\",\"indices\":[104,127],\"media_url\":\"http://pbs.twimg.com/media/EXKtv_MXgAIE6hI.jpg\",\"media_url_https\":\"https://pbs.twimg.com/media/EXKtv_MXgAIE6hI.jpg\",\"url\":\"https://t.co/FLN5SqzBK5\",\"display_url\":\"pic.twitter.com/FLN5SqzBK5\",\"expanded_url\":\"https://twitter.com/Beerie18/status/1257258288435036161/photo/1\",\"type\":\"photo\",\"sizes\":{\"thumb\":{\"w\":150,\"h\":150,\"resize\":\"crop\"},\"large\":{\"w\":960,\"h\":544,\"resize\":\"fit\"},\"medium\":{\"w\":960,\"h\":544,\"resize\":\"fit\"},\"small\":{\"w\":680,\"h\":385,\"resize\":\"fit\"}},\"source_status_id\":1.2572582884350362e+18,\"source_status_id_str\":\"1257258288435036161\",\"source_user_id\":7.214315371793449e+17,\"source_user_id_str\":\"721431537179344897\"}]}                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      1\n",
       "Name: extended_entities, Length: 3740, dtype: int64"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_na['extended_entities'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>_id</th>\n",
       "      <th>contributors</th>\n",
       "      <th>coordinates</th>\n",
       "      <th>created_at</th>\n",
       "      <th>entities</th>\n",
       "      <th>extended_entities</th>\n",
       "      <th>favorite_count</th>\n",
       "      <th>favorited</th>\n",
       "      <th>geo</th>\n",
       "      <th>id</th>\n",
       "      <th>...</th>\n",
       "      <th>quoted_status_id_str</th>\n",
       "      <th>retweet_count</th>\n",
       "      <th>retweeted</th>\n",
       "      <th>retweeted_status</th>\n",
       "      <th>source</th>\n",
       "      <th>text</th>\n",
       "      <th>full_text</th>\n",
       "      <th>truncated</th>\n",
       "      <th>user</th>\n",
       "      <th>withheld_in_countries</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>31395</th>\n",
       "      <td>ObjectId(5c207934f0a32a0e32d8e004)</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Mon Dec 24 05:58:42 +0000 2018</td>\n",
       "      <td>{\"hashtags\":[{\"text\":\"Islam\",\"indices\":[16,22]...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.077081e+18</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "      <td>{\"created_at\":\"Mon Dec 24 05:13:11 +0000 2018\"...</td>\n",
       "      <td>&lt;a href=\"https://www.botlibre.com\" rel=\"nofoll...</td>\n",
       "      <td>RT @whitey_mac: #Islam #Muslims #MuslimBrother...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>{\"id\":7.319444057168323e+17,\"id_str\":\"73194440...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37774</th>\n",
       "      <td>ObjectId(5b722385a3d81d0e3790d025)</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Tue Aug 14 00:24:15 +0000 2018</td>\n",
       "      <td>{\"hashtags\":[{\"text\":\"Australians\",\"indices\":[...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.029162e+18</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>11</td>\n",
       "      <td>False</td>\n",
       "      <td>{\"created_at\":\"Tue Aug 14 00:18:36 +0000 2018\"...</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/android\" ...</td>\n",
       "      <td>RT @Chris_A10_USA: #Australians told to lock t...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>{\"id\":1973684636,\"id_str\":\"1973684636\",\"name\":...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5154</th>\n",
       "      <td>ObjectId(5a0699e61928776402da83be)</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Sat Nov 11 06:27:04 +0000 2017</td>\n",
       "      <td>{\"hashtags\":[{\"text\":\"GrijzeWolven\",\"indices\":...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>9.292340e+17</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>69</td>\n",
       "      <td>False</td>\n",
       "      <td>{\"created_at\":\"Fri Nov 10 07:46:59 +0000 2017\"...</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/android\" ...</td>\n",
       "      <td>RT @FDW_VB: De #GrijzeWolven stoottroepen van ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>{\"id\":105920014,\"id_str\":\"105920014\",\"name\":\"J...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42340</th>\n",
       "      <td>ObjectId(5ba3e82523dae20e0f9bb12f)</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Thu Sep 20 18:24:18 +0000 2018</td>\n",
       "      <td>{\"hashtags\":[{\"text\":\"Banislam\",\"indices\":[34,...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.042842e+18</td>\n",
       "      <td>...</td>\n",
       "      <td>1.042841e+18</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "      <td>{\"created_at\":\"Thu Sep 20 18:22:09 +0000 2018\"...</td>\n",
       "      <td>&lt;a href=\"http://twitter.com\" rel=\"nofollow\"&gt;Tw...</td>\n",
       "      <td>RT @spacema09104056: Walgelijk....#Banislam ht...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>{\"id\":543579701,\"id_str\":\"543579701\",\"name\":\"P...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39298</th>\n",
       "      <td>ObjectId(5b8101c5a3addb0e0012eade)</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Sat Aug 25 06:54:24 +0000 2018</td>\n",
       "      <td>{\"hashtags\":[{\"text\":\"Bansharia\",\"indices\":[17...</td>\n",
       "      <td>{\"media\":[{\"id\":1.0332244072909292e+18,\"id_str...</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.033246e+18</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>9</td>\n",
       "      <td>False</td>\n",
       "      <td>{\"created_at\":\"Sat Aug 25 05:27:50 +0000 2018\"...</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/android\" ...</td>\n",
       "      <td>RT @JoanaTyler2: #Bansharia https://t.co/172WF...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>{\"id\":146432344,\"id_str\":\"146432344\",\"name\":\"s...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48980</th>\n",
       "      <td>ObjectId(5f04b6ea856a76136b85d799)</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Tue Jul 07 17:51:41 +0000 2020</td>\n",
       "      <td>{\"hashtags\":[{\"text\":\"WinstonChurchill\",\"indic...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.280560e+18</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2</td>\n",
       "      <td>False</td>\n",
       "      <td>{\"created_at\":\"Tue Jul 07 17:50:00 +0000 2020\"...</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/android\" ...</td>\n",
       "      <td>RT @based_belgium: #WinstonChurchill already k...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>{\"id\":8.836478223646351e+17,\"id_str\":\"88364782...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32159</th>\n",
       "      <td>ObjectId(5c2b5995adaaeb0e221c7b8c)</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Tue Jan 01 12:10:23 +0000 2019</td>\n",
       "      <td>{\"hashtags\":[],\"symbols\":[],\"user_mentions\":[{...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.080074e+18</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>69</td>\n",
       "      <td>False</td>\n",
       "      <td>{\"created_at\":\"Tue Jan 01 11:44:55 +0000 2019\"...</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>RT @lilbulli1031: I don’t care where you live,...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>{\"id\":8.16810658381656e+17,\"id_str\":\"816810658...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19184</th>\n",
       "      <td>ObjectId(5b1340e522f69f0e2aa113ee)</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Sun Jun 03 00:56:02 +0000 2018</td>\n",
       "      <td>{\"hashtags\":[{\"text\":\"BREAKING\",\"indices\":[21,...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.003078e+18</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>936</td>\n",
       "      <td>False</td>\n",
       "      <td>{\"created_at\":\"Sat Jun 02 18:51:41 +0000 2018\"...</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/#!/download/ipad\" ...</td>\n",
       "      <td>RT @SwensonBrigette: #BREAKING\\nLinda Sarsour ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>{\"id\":7.156203715930931e+17,\"id_str\":\"71562037...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42630</th>\n",
       "      <td>ObjectId(5d89f5d555989c0e9bd860b1)</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Tue Sep 24 10:43:12 +0000 2019</td>\n",
       "      <td>{\"hashtags\":[{\"text\":\"IStandWithBorisJohnson\",...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.176447e+18</td>\n",
       "      <td>...</td>\n",
       "      <td>1.027096e+18</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "      <td>{\"created_at\":\"Wed Aug 08 13:27:06 +0000 2018\"...</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/#!/download/ipad\" ...</td>\n",
       "      <td>RT @lionsmds: #IStandWithBorisJohnson #BanNiqa...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>{\"id\":3.843640245e+09,\"id_str\":\"3843640245\",\"n...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47351</th>\n",
       "      <td>ObjectId(5e61707021f5fc15a467bbfe)</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Thu Mar 05 21:29:56 +0000 2020</td>\n",
       "      <td>{\"hashtags\":[{\"text\":\"Tlaib\",\"indices\":[93,99]...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.235679e+18</td>\n",
       "      <td>...</td>\n",
       "      <td>1.235361e+18</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "      <td>{\"created_at\":\"Thu Mar 05 15:55:34 +0000 2020\"...</td>\n",
       "      <td>&lt;a href=\"http://twitter.com/download/iphone\" r...</td>\n",
       "      <td>RT @Supreme_Bashar: Bernie Gets AAPAC endorsem...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>{\"id\":1.186034508593877e+18,\"id_str\":\"11860345...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10 rows × 33 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      _id  contributors coordinates  \\\n",
       "31395  ObjectId(5c207934f0a32a0e32d8e004)           NaN         NaN   \n",
       "37774  ObjectId(5b722385a3d81d0e3790d025)           NaN         NaN   \n",
       "5154   ObjectId(5a0699e61928776402da83be)           NaN         NaN   \n",
       "42340  ObjectId(5ba3e82523dae20e0f9bb12f)           NaN         NaN   \n",
       "39298  ObjectId(5b8101c5a3addb0e0012eade)           NaN         NaN   \n",
       "48980  ObjectId(5f04b6ea856a76136b85d799)           NaN         NaN   \n",
       "32159  ObjectId(5c2b5995adaaeb0e221c7b8c)           NaN         NaN   \n",
       "19184  ObjectId(5b1340e522f69f0e2aa113ee)           NaN         NaN   \n",
       "42630  ObjectId(5d89f5d555989c0e9bd860b1)           NaN         NaN   \n",
       "47351  ObjectId(5e61707021f5fc15a467bbfe)           NaN         NaN   \n",
       "\n",
       "                           created_at  \\\n",
       "31395  Mon Dec 24 05:58:42 +0000 2018   \n",
       "37774  Tue Aug 14 00:24:15 +0000 2018   \n",
       "5154   Sat Nov 11 06:27:04 +0000 2017   \n",
       "42340  Thu Sep 20 18:24:18 +0000 2018   \n",
       "39298  Sat Aug 25 06:54:24 +0000 2018   \n",
       "48980  Tue Jul 07 17:51:41 +0000 2020   \n",
       "32159  Tue Jan 01 12:10:23 +0000 2019   \n",
       "19184  Sun Jun 03 00:56:02 +0000 2018   \n",
       "42630  Tue Sep 24 10:43:12 +0000 2019   \n",
       "47351  Thu Mar 05 21:29:56 +0000 2020   \n",
       "\n",
       "                                                entities  \\\n",
       "31395  {\"hashtags\":[{\"text\":\"Islam\",\"indices\":[16,22]...   \n",
       "37774  {\"hashtags\":[{\"text\":\"Australians\",\"indices\":[...   \n",
       "5154   {\"hashtags\":[{\"text\":\"GrijzeWolven\",\"indices\":...   \n",
       "42340  {\"hashtags\":[{\"text\":\"Banislam\",\"indices\":[34,...   \n",
       "39298  {\"hashtags\":[{\"text\":\"Bansharia\",\"indices\":[17...   \n",
       "48980  {\"hashtags\":[{\"text\":\"WinstonChurchill\",\"indic...   \n",
       "32159  {\"hashtags\":[],\"symbols\":[],\"user_mentions\":[{...   \n",
       "19184  {\"hashtags\":[{\"text\":\"BREAKING\",\"indices\":[21,...   \n",
       "42630  {\"hashtags\":[{\"text\":\"IStandWithBorisJohnson\",...   \n",
       "47351  {\"hashtags\":[{\"text\":\"Tlaib\",\"indices\":[93,99]...   \n",
       "\n",
       "                                       extended_entities  favorite_count  \\\n",
       "31395                                                NaN               0   \n",
       "37774                                                NaN               0   \n",
       "5154                                                 NaN               0   \n",
       "42340                                                NaN               0   \n",
       "39298  {\"media\":[{\"id\":1.0332244072909292e+18,\"id_str...               0   \n",
       "48980                                                NaN               0   \n",
       "32159                                                NaN               0   \n",
       "19184                                                NaN               0   \n",
       "42630                                                NaN               0   \n",
       "47351                                                NaN               0   \n",
       "\n",
       "       favorited  geo            id  ...  quoted_status_id_str retweet_count  \\\n",
       "31395      False  NaN  1.077081e+18  ...                   NaN             1   \n",
       "37774      False  NaN  1.029162e+18  ...                   NaN            11   \n",
       "5154       False  NaN  9.292340e+17  ...                   NaN            69   \n",
       "42340      False  NaN  1.042842e+18  ...          1.042841e+18             1   \n",
       "39298      False  NaN  1.033246e+18  ...                   NaN             9   \n",
       "48980      False  NaN  1.280560e+18  ...                   NaN             2   \n",
       "32159      False  NaN  1.080074e+18  ...                   NaN            69   \n",
       "19184      False  NaN  1.003078e+18  ...                   NaN           936   \n",
       "42630      False  NaN  1.176447e+18  ...          1.027096e+18             1   \n",
       "47351      False  NaN  1.235679e+18  ...          1.235361e+18             1   \n",
       "\n",
       "       retweeted                                   retweeted_status  \\\n",
       "31395      False  {\"created_at\":\"Mon Dec 24 05:13:11 +0000 2018\"...   \n",
       "37774      False  {\"created_at\":\"Tue Aug 14 00:18:36 +0000 2018\"...   \n",
       "5154       False  {\"created_at\":\"Fri Nov 10 07:46:59 +0000 2017\"...   \n",
       "42340      False  {\"created_at\":\"Thu Sep 20 18:22:09 +0000 2018\"...   \n",
       "39298      False  {\"created_at\":\"Sat Aug 25 05:27:50 +0000 2018\"...   \n",
       "48980      False  {\"created_at\":\"Tue Jul 07 17:50:00 +0000 2020\"...   \n",
       "32159      False  {\"created_at\":\"Tue Jan 01 11:44:55 +0000 2019\"...   \n",
       "19184      False  {\"created_at\":\"Sat Jun 02 18:51:41 +0000 2018\"...   \n",
       "42630      False  {\"created_at\":\"Wed Aug 08 13:27:06 +0000 2018\"...   \n",
       "47351      False  {\"created_at\":\"Thu Mar 05 15:55:34 +0000 2020\"...   \n",
       "\n",
       "                                                  source  \\\n",
       "31395  <a href=\"https://www.botlibre.com\" rel=\"nofoll...   \n",
       "37774  <a href=\"http://twitter.com/download/android\" ...   \n",
       "5154   <a href=\"http://twitter.com/download/android\" ...   \n",
       "42340  <a href=\"http://twitter.com\" rel=\"nofollow\">Tw...   \n",
       "39298  <a href=\"http://twitter.com/download/android\" ...   \n",
       "48980  <a href=\"http://twitter.com/download/android\" ...   \n",
       "32159  <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "19184  <a href=\"http://twitter.com/#!/download/ipad\" ...   \n",
       "42630  <a href=\"http://twitter.com/#!/download/ipad\" ...   \n",
       "47351  <a href=\"http://twitter.com/download/iphone\" r...   \n",
       "\n",
       "                                                    text  full_text truncated  \\\n",
       "31395  RT @whitey_mac: #Islam #Muslims #MuslimBrother...        NaN     False   \n",
       "37774  RT @Chris_A10_USA: #Australians told to lock t...        NaN     False   \n",
       "5154   RT @FDW_VB: De #GrijzeWolven stoottroepen van ...        NaN     False   \n",
       "42340  RT @spacema09104056: Walgelijk....#Banislam ht...        NaN     False   \n",
       "39298  RT @JoanaTyler2: #Bansharia https://t.co/172WF...        NaN     False   \n",
       "48980  RT @based_belgium: #WinstonChurchill already k...        NaN     False   \n",
       "32159  RT @lilbulli1031: I don’t care where you live,...        NaN     False   \n",
       "19184  RT @SwensonBrigette: #BREAKING\\nLinda Sarsour ...        NaN     False   \n",
       "42630  RT @lionsmds: #IStandWithBorisJohnson #BanNiqa...        NaN     False   \n",
       "47351  RT @Supreme_Bashar: Bernie Gets AAPAC endorsem...        NaN     False   \n",
       "\n",
       "                                                    user withheld_in_countries  \n",
       "31395  {\"id\":7.319444057168323e+17,\"id_str\":\"73194440...                   NaN  \n",
       "37774  {\"id\":1973684636,\"id_str\":\"1973684636\",\"name\":...                   NaN  \n",
       "5154   {\"id\":105920014,\"id_str\":\"105920014\",\"name\":\"J...                   NaN  \n",
       "42340  {\"id\":543579701,\"id_str\":\"543579701\",\"name\":\"P...                   NaN  \n",
       "39298  {\"id\":146432344,\"id_str\":\"146432344\",\"name\":\"s...                   NaN  \n",
       "48980  {\"id\":8.836478223646351e+17,\"id_str\":\"88364782...                   NaN  \n",
       "32159  {\"id\":8.16810658381656e+17,\"id_str\":\"816810658...                   NaN  \n",
       "19184  {\"id\":7.156203715930931e+17,\"id_str\":\"71562037...                   NaN  \n",
       "42630  {\"id\":3.843640245e+09,\"id_str\":\"3843640245\",\"n...                   NaN  \n",
       "47351  {\"id\":1.186034508593877e+18,\"id_str\":\"11860345...                   NaN  \n",
       "\n",
       "[10 rows x 33 columns]"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# can we break down the tweets into their sub-categories? \n",
    "# heatmap for days tweeted and word cloud (3)\n",
    "# most popular hashtags overtime, see how things change over time (top 10 line graph over time)\n",
    "# bar chart race (1)\n",
    "# world map for tweets over time (heat map) (2)\n",
    "df.sample(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False    34397\n",
       "True      6625\n",
       "Name: possibly_sensitive, dtype: int64"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['possibly_sensitive'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "sensitive_df = df[df['possibly_sensitive'] == True]\n",
    "n_sensitive_df = df[df['possibly_sensitive'] == False]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'en'"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sensitive_df['lang'].values[-25]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'RT @ensine: All #muslim rulers were savages bcoz #islamIsTheProblem, even profeet himself \\n#TyrantTipu \\nhttps://t.co/RqR4778Alr'"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "n_sensitive_df['text'].values[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "en     94348\n",
       "und    13477\n",
       "nl      5605\n",
       "de      1256\n",
       "hi      1096\n",
       "sv       971\n",
       "fr       534\n",
       "it       423\n",
       "el       255\n",
       "in       199\n",
       "es       186\n",
       "fa       104\n",
       "sl        78\n",
       "et        77\n",
       "da        67\n",
       "no        43\n",
       "ta        41\n",
       "tl        41\n",
       "ca        34\n",
       "pt        29\n",
       "ro        28\n",
       "ja        20\n",
       "tr        20\n",
       "cy        12\n",
       "pl        10\n",
       "cs         9\n",
       "ur         9\n",
       "lt         8\n",
       "ht         7\n",
       "zh         7\n",
       "hu         6\n",
       "fi         4\n",
       "ar         3\n",
       "is         2\n",
       "vi         2\n",
       "lv         2\n",
       "eu         2\n",
       "bn         1\n",
       "si         1\n",
       "ko         1\n",
       "ru         1\n",
       "Name: lang, dtype: int64"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['lang'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "iso_639_choices = [('ab', 'Abkhaz'),\n",
    "('aa', 'Afar'),\n",
    "('af', 'Afrikaans'),\n",
    "('ak', 'Akan'),\n",
    "('sq', 'Albanian'),\n",
    "('am', 'Amharic'),\n",
    "('ar', 'Arabic'),\n",
    "('an', 'Aragonese'),\n",
    "('hy', 'Armenian'),\n",
    "('as', 'Assamese'),\n",
    "('av', 'Avaric'),\n",
    "('ae', 'Avestan'),\n",
    "('ay', 'Aymara'),\n",
    "('az', 'Azerbaijani'),\n",
    "('bm', 'Bambara'),\n",
    "('ba', 'Bashkir'),\n",
    "('eu', 'Basque'),\n",
    "('be', 'Belarusian'),\n",
    "('bn', 'Bengali'),\n",
    "('bh', 'Bihari'),\n",
    "('bi', 'Bislama'),\n",
    "('bs', 'Bosnian'),\n",
    "('br', 'Breton'),\n",
    "('bg', 'Bulgarian'),\n",
    "('my', 'Burmese'),\n",
    "('ca', 'Catalan; Valencian'),\n",
    "('ch', 'Chamorro'),\n",
    "('ce', 'Chechen'),\n",
    "('ny', 'Chichewa; Chewa; Nyanja'),\n",
    "('zh', 'Chinese'),\n",
    "('cv', 'Chuvash'),\n",
    "('kw', 'Cornish'),\n",
    "('co', 'Corsican'),\n",
    "('cr', 'Cree'),\n",
    "('hr', 'Croatian'),\n",
    "('cs', 'Czech'),\n",
    "('da', 'Danish'),\n",
    "('dv', 'Divehi; Maldivian;'),\n",
    "('nl', 'Dutch'),\n",
    "('dz', 'Dzongkha'),\n",
    "('en', 'English'),\n",
    "('eo', 'Esperanto'),\n",
    "('et', 'Estonian'),\n",
    "('ee', 'Ewe'),\n",
    "('fo', 'Faroese'),\n",
    "('fj', 'Fijian'),\n",
    "('fi', 'Finnish'),\n",
    "('fr', 'French'),\n",
    "('ff', 'Fula'),\n",
    "('gl', 'Galician'),\n",
    "('ka', 'Georgian'),\n",
    "('de', 'German'),\n",
    "('el', 'Greek, Modern'),\n",
    "('gn', 'Guaraní'),\n",
    "('gu', 'Gujarati'),\n",
    "('ht', 'Haitian'),\n",
    "('ha', 'Hausa'),\n",
    "('he', 'Hebrew (modern)'),\n",
    "('hz', 'Herero'),\n",
    "('hi', 'Hindi'),\n",
    "('ho', 'Hiri Motu'),\n",
    "('hu', 'Hungarian'),\n",
    "('ia', 'Interlingua'),\n",
    "('id', 'Indonesian'),\n",
    "('ie', 'Interlingue'),\n",
    "('ga', 'Irish'),\n",
    "('ig', 'Igbo'),\n",
    "('ik', 'Inupiaq'),\n",
    "('io', 'Ido'),\n",
    "('is', 'Icelandic'),\n",
    "('it', 'Italian'),\n",
    "('iu', 'Inuktitut'),\n",
    "('ja', 'Japanese'),\n",
    "('jv', 'Javanese'),\n",
    "('kl', 'Kalaallisut'),\n",
    "('kn', 'Kannada'),\n",
    "('kr', 'Kanuri'),\n",
    "('ks', 'Kashmiri'),\n",
    "('kk', 'Kazakh'),\n",
    "('km', 'Khmer'),\n",
    "('ki', 'Kikuyu, Gikuyu'),\n",
    "('rw', 'Kinyarwanda'),\n",
    "('ky', 'Kirghiz, Kyrgyz'),\n",
    "('kv', 'Komi'),\n",
    "('kg', 'Kongo'),\n",
    "('ko', 'Korean'),\n",
    "('ku', 'Kurdish'),\n",
    "('kj', 'Kwanyama, Kuanyama'),\n",
    "('la', 'Latin'),\n",
    "('lb', 'Luxembourgish'),\n",
    "('lg', 'Luganda'),\n",
    "('li', 'Limburgish'),\n",
    "('ln', 'Lingala'),\n",
    "('lo', 'Lao'),\n",
    "('lt', 'Lithuanian'),\n",
    "('lu', 'Luba-Katanga'),\n",
    "('lv', 'Latvian'),\n",
    "('gv', 'Manx'),\n",
    "('mk', 'Macedonian'),\n",
    "('mg', 'Malagasy'),\n",
    "('ms', 'Malay'),\n",
    "('ml', 'Malayalam'),\n",
    "('mt', 'Maltese'),\n",
    "('mi', 'Māori'),\n",
    "('mr', 'Marathi (Marāṭhī)'),\n",
    "('mh', 'Marshallese'),\n",
    "('mn', 'Mongolian'),\n",
    "('na', 'Nauru'),\n",
    "('nv', 'Navajo, Navaho'),\n",
    "('nb', 'Norwegian Bokmål'),\n",
    "('nd', 'North Ndebele'),\n",
    "('ne', 'Nepali'),\n",
    "('ng', 'Ndonga'),\n",
    "('nn', 'Norwegian Nynorsk'),\n",
    "('no', 'Norwegian'),\n",
    "('ii', 'Nuosu'),\n",
    "('nr', 'South Ndebele'),\n",
    "('oc', 'Occitan'),\n",
    "('oj', 'Ojibwe, Ojibwa'),\n",
    "('cu', 'Old Church Slavonic'),\n",
    "('om', 'Oromo'),\n",
    "('or', 'Oriya'),\n",
    "('os', 'Ossetian, Ossetic'),\n",
    "('pa', 'Panjabi, Punjabi'),\n",
    "('pi', 'Pāli'),\n",
    "('fa', 'Persian'),\n",
    "('pl', 'Polish'),\n",
    "('ps', 'Pashto, Pushto'),\n",
    "('pt', 'Portuguese'),\n",
    "('qu', 'Quechua'),\n",
    "('rm', 'Romansh'),\n",
    "('rn', 'Kirundi'),\n",
    "('ro', 'Romanian, Moldavan'),\n",
    "('ru', 'Russian'),\n",
    "('sa', 'Sanskrit (Saṁskṛta)'),\n",
    "('sc', 'Sardinian'),\n",
    "('sd', 'Sindhi'),\n",
    "('se', 'Northern Sami'),\n",
    "('sm', 'Samoan'),\n",
    "('sg', 'Sango'),\n",
    "('sr', 'Serbian'),\n",
    "('gd', 'Scottish Gaelic'),\n",
    "('sn', 'Shona'),\n",
    "('si', 'Sinhala, Sinhalese'),\n",
    "('sk', 'Slovak'),\n",
    "('sl', 'Slovene'),\n",
    "('so', 'Somali'),\n",
    "('st', 'Southern Sotho'),\n",
    "('es', 'Spanish; Castilian'),\n",
    "('su', 'Sundanese'),\n",
    "('sw', 'Swahili'),\n",
    "('ss', 'Swati'),\n",
    "('sv', 'Swedish'),\n",
    "('ta', 'Tamil'),\n",
    "('te', 'Telugu'),\n",
    "('tg', 'Tajik'),\n",
    "('th', 'Thai'),\n",
    "('ti', 'Tigrinya'),\n",
    "('bo', 'Tibetan'),\n",
    "('tk', 'Turkmen'),\n",
    "('tl', 'Tagalog'),\n",
    "('tn', 'Tswana'),\n",
    "('to', 'Tonga'),\n",
    "('tr', 'Turkish'),\n",
    "('ts', 'Tsonga'),\n",
    "('tt', 'Tatar'),\n",
    "('tw', 'Twi'),\n",
    "('ty', 'Tahitian'),\n",
    "('ug', 'Uighur, Uyghur'),\n",
    "('uk', 'Ukrainian'),\n",
    "('ur', 'Urdu'),\n",
    "('uz', 'Uzbek'),\n",
    "('ve', 'Venda'),\n",
    "('vi', 'Vietnamese'),\n",
    "('vo', 'Volapük'),\n",
    "('wa', 'Walloon'),\n",
    "('cy', 'Welsh'),\n",
    "('wo', 'Wolof'),\n",
    "('fy', 'Western Frisian'),\n",
    "('xh', 'Xhosa'),\n",
    "('yi', 'Yiddish'),\n",
    "('yo', 'Yoruba'),\n",
    "('za', 'Zhuang, Chuang'),\n",
    "('zu', 'Zulu'),]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pycountry\n",
    "import numpy as np\n",
    "\n",
    "countries = np.unique(df['lang'].values)\n",
    "codes = {}\n",
    "\n",
    "for c in countries:\n",
    "    try:\n",
    "        codes[c] = dict(iso_639_choices)[c]\n",
    "    except:\n",
    "        codes[c] = 'None'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'ar': 'Arabic',\n",
       " 'bn': 'Bengali',\n",
       " 'ca': 'Catalan; Valencian',\n",
       " 'cs': 'Czech',\n",
       " 'cy': 'Welsh',\n",
       " 'da': 'Danish',\n",
       " 'de': 'German',\n",
       " 'el': 'Greek, Modern',\n",
       " 'en': 'English',\n",
       " 'es': 'Spanish; Castilian',\n",
       " 'et': 'Estonian',\n",
       " 'eu': 'Basque',\n",
       " 'fa': 'Persian',\n",
       " 'fi': 'Finnish',\n",
       " 'fr': 'French',\n",
       " 'hi': 'Hindi',\n",
       " 'ht': 'Haitian',\n",
       " 'hu': 'Hungarian',\n",
       " 'in': 'None',\n",
       " 'is': 'Icelandic',\n",
       " 'it': 'Italian',\n",
       " 'ja': 'Japanese',\n",
       " 'ko': 'Korean',\n",
       " 'lt': 'Lithuanian',\n",
       " 'lv': 'Latvian',\n",
       " 'nl': 'Dutch',\n",
       " 'no': 'Norwegian',\n",
       " 'pl': 'Polish',\n",
       " 'pt': 'Portuguese',\n",
       " 'ro': 'Romanian, Moldavan',\n",
       " 'ru': 'Russian',\n",
       " 'si': 'Sinhala, Sinhalese',\n",
       " 'sl': 'Slovene',\n",
       " 'sv': 'Swedish',\n",
       " 'ta': 'Tamil',\n",
       " 'tl': 'Tagalog',\n",
       " 'tr': 'Turkish',\n",
       " 'und': 'None',\n",
       " 'ur': 'Urdu',\n",
       " 'vi': 'Vietnamese',\n",
       " 'zh': 'Chinese'}"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "codes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'{\"hashtags\":[{\"text\":\"BanIslam\",\"indices\":[93,102]},{\"text\":\"DeportALLmuslims\",\"indices\":[109,126]}],\"symbols\":[],\"user_mentions\":[{\"screen_name\":\"Private34349909\",\"name\":\"Private User\",\"id\":7.976360710643794e+17,\"id_str\":\"797636071064379392\",\"indices\":[3,19]},{\"screen_name\":\"AmyMek\",\"name\":\"Amy Mek\",\"id\":954124423,\"id_str\":\"954124423\",\"indices\":[21,28]},{\"screen_name\":\"Dab7One\",\"name\":\"Dabney 1\",\"id\":2.850722943e+09,\"id_str\":\"2850722943\",\"indices\":[29,37]},{\"screen_name\":\"realDonaldTrump\",\"name\":\"Donald J. Trump\",\"id\":25073877,\"id_str\":\"25073877\",\"indices\":[41,57]},{\"screen_name\":\"seanhannity\",\"name\":\"Sean Hannity\",\"id\":41634520,\"id_str\":\"41634520\",\"indices\":[58,70]},{\"screen_name\":\"TuckerCarlson\",\"name\":\"Tucker Carlson\",\"id\":22703645,\"id_str\":\"22703645\",\"indices\":[71,85]}],\"urls\":[]}'"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['entities'][0]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}