Tweet Preprocessing .ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Import libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from pandas import DataFrame\n",
    "import nltk,emoji,re,string"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Read data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>comment_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>Explanation\\r\\nWhy the edits made under my use...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>D'aww! He matches this background colour I'm s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>Hey man, I'm really not trying to edit war. It...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>\"\\r\\nMore\\r\\nI can't make any real suggestions...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>You, sir, are my hero. Any chance you remember...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0                                       comment_text\n",
       "0           0  Explanation\\r\\nWhy the edits made under my use...\n",
       "1           1  D'aww! He matches this background colour I'm s...\n",
       "2           2  Hey man, I'm really not trying to edit war. It...\n",
       "3           3  \"\\r\\nMore\\r\\nI can't make any real suggestions...\n",
       "4           4  You, sir, are my hero. Any chance you remember..."
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data=pd.read_csv(\"tweet_train.csv\",nrows=10).dropna()\n",
    "data.drop(['toxic'],axis=1,inplace=True)\n",
    "\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preprocessing code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def url_remove(text):\n",
    "    t=re.sub(r'https://\\S+','',text)\n",
    "    return t\n",
    "def punc_remove(text):\n",
    "    t=re.sub(r'[^\\w\\s]',' ',text)\n",
    "    return t\n",
    "def emoji_remove(text):\n",
    "    return emoji.get_emoji_regexp().sub(\"\",text).strip()\n",
    "def hashtag_remove(text):\n",
    "    reg=\"#(\\w+)\"\n",
    "    return re.sub(reg,\" \",text)\n",
    "\n",
    "cleaned=[]\n",
    "def clean_text(text):\n",
    "    for i in text:\n",
    "        cleaned.append(emoji_remove(punc_remove(hashtag_remove(url_remove(re.sub(\"[\\n\\r\\t\\xa0]\",\" \",i).strip())))))\n",
    "clean_text(data['comment_text'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>comment_text</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>Explanation\\r\\nWhy the edits made under my use...</td>\n",
       "      <td>Explanation  Why the edits made under my usern...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>D'aww! He matches this background colour I'm s...</td>\n",
       "      <td>D aww  He matches this background colour I m s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>Hey man, I'm really not trying to edit war. It...</td>\n",
       "      <td>Hey man  I m really not trying to edit war  It...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>\"\\r\\nMore\\r\\nI can't make any real suggestions...</td>\n",
       "      <td>More  I can t make any real suggestions on imp...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>You, sir, are my hero. Any chance you remember...</td>\n",
       "      <td>You  sir  are my hero  Any chance you remember...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0                                       comment_text  \\\n",
       "0           0  Explanation\\r\\nWhy the edits made under my use...   \n",
       "1           1  D'aww! He matches this background colour I'm s...   \n",
       "2           2  Hey man, I'm really not trying to edit war. It...   \n",
       "3           3  \"\\r\\nMore\\r\\nI can't make any real suggestions...   \n",
       "4           4  You, sir, are my hero. Any chance you remember...   \n",
       "\n",
       "                                                text  \n",
       "0  Explanation  Why the edits made under my usern...  \n",
       "1  D aww  He matches this background colour I m s...  \n",
       "2  Hey man  I m really not trying to edit war  It...  \n",
       "3  More  I can t make any real suggestions on imp...  \n",
       "4  You  sir  are my hero  Any chance you remember...  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['text']=cleaned\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Write output to excel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "tt={\"Text\":data['comment_text'],\n",
    "   \"Cleaned_Text\":data['text']}\n",
    "\n",
    "df=pd.DataFrame(tt)\n",
    "df.to_excel(\"op.xlsx\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}