preprocessing_better_view.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "3298392d",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from dash import html\n",
    "\n",
    "# Read The Ted DataSet\n",
    "df = pd.read_csv(\"ted_main.csv\")\n",
    "\n",
    "# Print The First Row of the Data(df)\n",
    "# print(df.head(1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "6b9f0d3e",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 2550 entries, 0 to 2549\n",
      "Data columns (total 17 columns):\n",
      " #   Column              Non-Null Count  Dtype \n",
      "---  ------              --------------  ----- \n",
      " 0   comments            2550 non-null   int64 \n",
      " 1   description         2550 non-null   object\n",
      " 2   duration            2550 non-null   int64 \n",
      " 3   event               2550 non-null   object\n",
      " 4   film_date           2550 non-null   int64 \n",
      " 5   languages           2550 non-null   int64 \n",
      " 6   main_speaker        2550 non-null   object\n",
      " 7   name                2550 non-null   object\n",
      " 8   num_speaker         2550 non-null   int64 \n",
      " 9   published_date      2550 non-null   int64 \n",
      " 10  ratings             2550 non-null   object\n",
      " 11  related_talks       2550 non-null   object\n",
      " 12  speaker_occupation  2544 non-null   object\n",
      " 13  tags                2550 non-null   object\n",
      " 14  title               2550 non-null   object\n",
      " 15  url                 2550 non-null   object\n",
      " 16  views               2550 non-null   int64 \n",
      "dtypes: int64(7), object(10)\n",
      "memory usage: 338.8+ KB\n",
      "None\n",
      "comments              0\n",
      "description           0\n",
      "duration              0\n",
      "event                 0\n",
      "film_date             0\n",
      "languages             0\n",
      "main_speaker          0\n",
      "name                  0\n",
      "num_speaker           0\n",
      "published_date        0\n",
      "ratings               0\n",
      "related_talks         0\n",
      "speaker_occupation    6\n",
      "tags                  0\n",
      "title                 0\n",
      "url                   0\n",
      "views                 0\n",
      "dtype: int64\n"
     ]
    }
   ],
   "source": [
    "# get Data_set Information (Nulls , Numbers of Records , Data Types)\n",
    "print(df.info())\n",
    "\n",
    "# Count Null Values of each Column\n",
    "print(df.isna().sum())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "140107a4",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['speaker_occupation'].fillna('unknown', inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "520cd8dc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 2550 entries, 0 to 2549\n",
      "Data columns (total 17 columns):\n",
      " #   Column              Non-Null Count  Dtype \n",
      "---  ------              --------------  ----- \n",
      " 0   comments            2550 non-null   int64 \n",
      " 1   description         2550 non-null   object\n",
      " 2   duration            2550 non-null   int64 \n",
      " 3   event               2550 non-null   object\n",
      " 4   film_date           2550 non-null   int64 \n",
      " 5   languages           2550 non-null   int64 \n",
      " 6   main_speaker        2550 non-null   object\n",
      " 7   name                2550 non-null   object\n",
      " 8   num_speaker         2550 non-null   int64 \n",
      " 9   published_date      2550 non-null   int64 \n",
      " 10  ratings             2550 non-null   object\n",
      " 11  related_talks       2550 non-null   object\n",
      " 12  speaker_occupation  2550 non-null   object\n",
      " 13  tags                2550 non-null   object\n",
      " 14  title               2550 non-null   object\n",
      " 15  url                 2550 non-null   object\n",
      " 16  views               2550 non-null   int64 \n",
      "dtypes: int64(7), object(10)\n",
      "memory usage: 338.8+ KB\n"
     ]
    }
   ],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "0e198c64",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Preprocessing The Data\n",
    "\n",
    "# Convert The Date Col. from Unix Timestamp to Date Type\n",
    "df['Date'] = pd.to_datetime(df['published_date'], unit='s')\n",
    "\n",
    "# Drop Some Columns.\n",
    "df.drop(columns=['event', 'film_date', 'languages', 'name', 'num_speaker',\n",
    "                 'published_date', 'description', 'related_talks'], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "52ac7776",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>comments</th>\n",
       "      <th>duration</th>\n",
       "      <th>main_speaker</th>\n",
       "      <th>ratings</th>\n",
       "      <th>speaker_occupation</th>\n",
       "      <th>tags</th>\n",
       "      <th>title</th>\n",
       "      <th>url</th>\n",
       "      <th>views</th>\n",
       "      <th>Date</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4553</td>\n",
       "      <td>1164</td>\n",
       "      <td>Ken Robinson</td>\n",
       "      <td>[{'id': 7, 'name': 'Funny', 'count': 19645}, {...</td>\n",
       "      <td>Author/educator</td>\n",
       "      <td>['children', 'creativity', 'culture', 'dance',...</td>\n",
       "      <td>Do schools kill creativity?</td>\n",
       "      <td>https://www.ted.com/talks/ken_robinson_says_sc...</td>\n",
       "      <td>47227110</td>\n",
       "      <td>2006-06-27 00:11:00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   comments  duration  main_speaker  \\\n",
       "0      4553      1164  Ken Robinson   \n",
       "\n",
       "                                             ratings speaker_occupation  \\\n",
       "0  [{'id': 7, 'name': 'Funny', 'count': 19645}, {...    Author/educator   \n",
       "\n",
       "                                                tags  \\\n",
       "0  ['children', 'creativity', 'culture', 'dance',...   \n",
       "\n",
       "                         title  \\\n",
       "0  Do schools kill creativity?   \n",
       "\n",
       "                                                 url     views  \\\n",
       "0  https://www.ted.com/talks/ken_robinson_says_sc...  47227110   \n",
       "\n",
       "                 Date  \n",
       "0 2006-06-27 00:11:00  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "aa9ffe94",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 2550 entries, 0 to 2549\n",
      "Data columns (total 10 columns):\n",
      " #   Column              Non-Null Count  Dtype         \n",
      "---  ------              --------------  -----         \n",
      " 0   comments            2550 non-null   int64         \n",
      " 1   duration            2550 non-null   int64         \n",
      " 2   main_speaker        2550 non-null   object        \n",
      " 3   ratings             2550 non-null   object        \n",
      " 4   speaker_occupation  2550 non-null   object        \n",
      " 5   tags                2550 non-null   object        \n",
      " 6   title               2550 non-null   object        \n",
      " 7   url                 2550 non-null   object        \n",
      " 8   views               2550 non-null   int64         \n",
      " 9   Date                2550 non-null   datetime64[ns]\n",
      "dtypes: datetime64[ns](1), int64(3), object(6)\n",
      "memory usage: 199.3+ KB\n"
     ]
    }
   ],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "409eb617",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['month'] = df['Date'].dt.month_name()\n",
    "df[\"year\"] = df['Date'].map(lambda x: x.year)\n",
    "\n",
    "df.drop('Date', inplace=True, axis=1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "49b5b1b3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>comments</th>\n",
       "      <th>duration</th>\n",
       "      <th>main_speaker</th>\n",
       "      <th>ratings</th>\n",
       "      <th>speaker_occupation</th>\n",
       "      <th>tags</th>\n",
       "      <th>title</th>\n",
       "      <th>url</th>\n",
       "      <th>views</th>\n",
       "      <th>Date</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4553</td>\n",
       "      <td>1164</td>\n",
       "      <td>Ken Robinson</td>\n",
       "      <td>[{'id': 7, 'name': 'Funny', 'count': 19645}, {...</td>\n",
       "      <td>Author/educator</td>\n",
       "      <td>['children', 'creativity', 'culture', 'dance',...</td>\n",
       "      <td>Do schools kill creativity?</td>\n",
       "      <td>https://www.ted.com/talks/ken_robinson_says_sc...</td>\n",
       "      <td>47227110</td>\n",
       "      <td>2006-06-27 00:11:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>265</td>\n",
       "      <td>977</td>\n",
       "      <td>Al Gore</td>\n",
       "      <td>[{'id': 7, 'name': 'Funny', 'count': 544}, {'i...</td>\n",
       "      <td>Climate advocate</td>\n",
       "      <td>['alternative energy', 'cars', 'climate change...</td>\n",
       "      <td>Averting the climate crisis</td>\n",
       "      <td>https://www.ted.com/talks/al_gore_on_averting_...</td>\n",
       "      <td>3200520</td>\n",
       "      <td>2006-06-27 00:11:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>124</td>\n",
       "      <td>1286</td>\n",
       "      <td>David Pogue</td>\n",
       "      <td>[{'id': 7, 'name': 'Funny', 'count': 964}, {'i...</td>\n",
       "      <td>Technology columnist</td>\n",
       "      <td>['computers', 'entertainment', 'interface desi...</td>\n",
       "      <td>Simplicity sells</td>\n",
       "      <td>https://www.ted.com/talks/david_pogue_says_sim...</td>\n",
       "      <td>1636292</td>\n",
       "      <td>2006-06-27 00:11:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>200</td>\n",
       "      <td>1116</td>\n",
       "      <td>Majora Carter</td>\n",
       "      <td>[{'id': 3, 'name': 'Courageous', 'count': 760}...</td>\n",
       "      <td>Activist for environmental justice</td>\n",
       "      <td>['MacArthur grant', 'activism', 'business', 'c...</td>\n",
       "      <td>Greening the ghetto</td>\n",
       "      <td>https://www.ted.com/talks/majora_carter_s_tale...</td>\n",
       "      <td>1697550</td>\n",
       "      <td>2006-06-27 00:11:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>593</td>\n",
       "      <td>1190</td>\n",
       "      <td>Hans Rosling</td>\n",
       "      <td>[{'id': 9, 'name': 'Ingenious', 'count': 3202}...</td>\n",
       "      <td>Global health expert; data visionary</td>\n",
       "      <td>['Africa', 'Asia', 'Google', 'demo', 'economic...</td>\n",
       "      <td>The best stats you've ever seen</td>\n",
       "      <td>https://www.ted.com/talks/hans_rosling_shows_t...</td>\n",
       "      <td>12005869</td>\n",
       "      <td>2006-06-27 20:38:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>672</td>\n",
       "      <td>1305</td>\n",
       "      <td>Tony Robbins</td>\n",
       "      <td>[{'id': 7, 'name': 'Funny', 'count': 1102}, {'...</td>\n",
       "      <td>Life coach; expert in leadership psychology</td>\n",
       "      <td>['business', 'culture', 'entertainment', 'goal...</td>\n",
       "      <td>Why we do what we do</td>\n",
       "      <td>https://www.ted.com/talks/tony_robbins_asks_wh...</td>\n",
       "      <td>20685401</td>\n",
       "      <td>2006-06-27 20:38:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>919</td>\n",
       "      <td>992</td>\n",
       "      <td>Julia Sweeney</td>\n",
       "      <td>[{'id': 3, 'name': 'Courageous', 'count': 477}...</td>\n",
       "      <td>Actor, comedian, playwright</td>\n",
       "      <td>['Christianity', 'God', 'atheism', 'comedy', '...</td>\n",
       "      <td>Letting go of God</td>\n",
       "      <td>https://www.ted.com/talks/julia_sweeney_on_let...</td>\n",
       "      <td>3769987</td>\n",
       "      <td>2006-07-10 00:11:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>46</td>\n",
       "      <td>1198</td>\n",
       "      <td>Joshua Prince-Ramus</td>\n",
       "      <td>[{'id': 9, 'name': 'Ingenious', 'count': 195},...</td>\n",
       "      <td>Architect</td>\n",
       "      <td>['architecture', 'collaboration', 'culture', '...</td>\n",
       "      <td>Behind the design of Seattle's library</td>\n",
       "      <td>https://www.ted.com/talks/joshua_prince_ramus_...</td>\n",
       "      <td>967741</td>\n",
       "      <td>2006-07-10 00:11:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>852</td>\n",
       "      <td>1485</td>\n",
       "      <td>Dan Dennett</td>\n",
       "      <td>[{'id': 3, 'name': 'Courageous', 'count': 562}...</td>\n",
       "      <td>Philosopher, cognitive scientist</td>\n",
       "      <td>['God', 'TED Brain Trust', 'atheism', 'brain',...</td>\n",
       "      <td>Let's teach religion -- all religion -- in sch...</td>\n",
       "      <td>https://www.ted.com/talks/dan_dennett_s_respon...</td>\n",
       "      <td>2567958</td>\n",
       "      <td>2006-07-18 00:11:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>900</td>\n",
       "      <td>1262</td>\n",
       "      <td>Rick Warren</td>\n",
       "      <td>[{'id': 21, 'name': 'Unconvincing', 'count': 8...</td>\n",
       "      <td>Pastor, author</td>\n",
       "      <td>['Christianity', 'God', 'culture', 'happiness'...</td>\n",
       "      <td>A life of purpose</td>\n",
       "      <td>https://www.ted.com/talks/rick_warren_on_a_lif...</td>\n",
       "      <td>3095993</td>\n",
       "      <td>2006-07-18 00:11:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>79</td>\n",
       "      <td>1414</td>\n",
       "      <td>Cameron Sinclair</td>\n",
       "      <td>[{'id': 3, 'name': 'Courageous', 'count': 169}...</td>\n",
       "      <td>Co-founder, Architecture for Humanity</td>\n",
       "      <td>['activism', 'architecture', 'collaboration', ...</td>\n",
       "      <td>My wish: A call for open-source architecture</td>\n",
       "      <td>https://www.ted.com/talks/cameron_sinclair_on_...</td>\n",
       "      <td>1211416</td>\n",
       "      <td>2006-07-25 00:11:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>55</td>\n",
       "      <td>1538</td>\n",
       "      <td>Jehane Noujaim</td>\n",
       "      <td>[{'id': 1, 'name': 'Beautiful', 'count': 70}, ...</td>\n",
       "      <td>Filmmaker</td>\n",
       "      <td>['TED Prize', 'art', 'culture', 'entertainment...</td>\n",
       "      <td>My wish: A global day of film</td>\n",
       "      <td>https://www.ted.com/talks/jehane_noujaim_inspi...</td>\n",
       "      <td>387877</td>\n",
       "      <td>2006-07-25 00:11:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>71</td>\n",
       "      <td>1550</td>\n",
       "      <td>Larry Brilliant</td>\n",
       "      <td>[{'id': 8, 'name': 'Informative', 'count': 188...</td>\n",
       "      <td>Epidemiologist, philanthropist</td>\n",
       "      <td>['TED Prize', 'collaboration', 'disease', 'ebo...</td>\n",
       "      <td>My wish: Help me stop pandemics</td>\n",
       "      <td>https://www.ted.com/talks/larry_brilliant_want...</td>\n",
       "      <td>693341</td>\n",
       "      <td>2006-07-25 00:11:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>242</td>\n",
       "      <td>527</td>\n",
       "      <td>Jeff Han</td>\n",
       "      <td>[{'id': 9, 'name': 'Ingenious', 'count': 1014}...</td>\n",
       "      <td>Human-computer interface designer</td>\n",
       "      <td>['demo', 'design', 'interface design', 'techno...</td>\n",
       "      <td>The radical promise of the multi-touch interface</td>\n",
       "      <td>https://www.ted.com/talks/jeff_han_demos_his_b...</td>\n",
       "      <td>4531020</td>\n",
       "      <td>2006-08-01 00:11:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>99</td>\n",
       "      <td>1057</td>\n",
       "      <td>Nicholas Negroponte</td>\n",
       "      <td>[{'id': 3, 'name': 'Courageous', 'count': 83},...</td>\n",
       "      <td>Tech visionary</td>\n",
       "      <td>['children', 'design', 'education', 'entrepren...</td>\n",
       "      <td>One Laptop per Child</td>\n",
       "      <td>https://www.ted.com/talks/nicholas_negroponte_...</td>\n",
       "      <td>358304</td>\n",
       "      <td>2006-08-01 00:11:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>325</td>\n",
       "      <td>1481</td>\n",
       "      <td>Sirena Huang</td>\n",
       "      <td>[{'id': 1, 'name': 'Beautiful', 'count': 1671}...</td>\n",
       "      <td>Violinist</td>\n",
       "      <td>['entertainment', 'music', 'performance', 'vio...</td>\n",
       "      <td>An 11-year-old's magical violin</td>\n",
       "      <td>https://www.ted.com/talks/sirena_huang_dazzles...</td>\n",
       "      <td>2702470</td>\n",
       "      <td>2006-08-08 00:11:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>305</td>\n",
       "      <td>1445</td>\n",
       "      <td>Jennifer Lin</td>\n",
       "      <td>[{'id': 1, 'name': 'Beautiful', 'count': 1107}...</td>\n",
       "      <td>Pianist, composer</td>\n",
       "      <td>['creativity', 'entertainment', 'music', 'perf...</td>\n",
       "      <td>Improvising on piano, aged 14</td>\n",
       "      <td>https://www.ted.com/talks/jennifer_lin_improvs...</td>\n",
       "      <td>1628912</td>\n",
       "      <td>2006-08-08 00:11:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>88</td>\n",
       "      <td>906</td>\n",
       "      <td>Amy Smith</td>\n",
       "      <td>[{'id': 9, 'name': 'Ingenious', 'count': 321},...</td>\n",
       "      <td>inventor, engineer</td>\n",
       "      <td>['MacArthur grant', 'alternative energy', 'des...</td>\n",
       "      <td>Simple designs to save a life</td>\n",
       "      <td>https://www.ted.com/talks/amy_smith_shares_sim...</td>\n",
       "      <td>1415724</td>\n",
       "      <td>2006-08-15 00:11:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>163</td>\n",
       "      <td>1170</td>\n",
       "      <td>Ross Lovegrove</td>\n",
       "      <td>[{'id': 1, 'name': 'Beautiful', 'count': 259},...</td>\n",
       "      <td>Industrial designer</td>\n",
       "      <td>['DNA', 'biology', 'creativity', 'design', 'in...</td>\n",
       "      <td>Organic design, inspired by nature</td>\n",
       "      <td>https://www.ted.com/talks/ross_lovegrove_share...</td>\n",
       "      <td>1074081</td>\n",
       "      <td>2006-08-15 00:11:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>84</td>\n",
       "      <td>1201</td>\n",
       "      <td>Jimmy Wales</td>\n",
       "      <td>[{'id': 1, 'name': 'Beautiful', 'count': 104},...</td>\n",
       "      <td>Founder of Wikipedia</td>\n",
       "      <td>['business', 'collaboration', 'culture', 'inve...</td>\n",
       "      <td>The birth of Wikipedia</td>\n",
       "      <td>https://www.ted.com/talks/jimmy_wales_on_the_b...</td>\n",
       "      <td>1106561</td>\n",
       "      <td>2006-08-21 00:11:00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    comments  duration         main_speaker  \\\n",
       "0       4553      1164         Ken Robinson   \n",
       "1        265       977              Al Gore   \n",
       "2        124      1286          David Pogue   \n",
       "3        200      1116        Majora Carter   \n",
       "4        593      1190         Hans Rosling   \n",
       "5        672      1305         Tony Robbins   \n",
       "6        919       992        Julia Sweeney   \n",
       "7         46      1198  Joshua Prince-Ramus   \n",
       "8        852      1485          Dan Dennett   \n",
       "9        900      1262          Rick Warren   \n",
       "10        79      1414     Cameron Sinclair   \n",
       "11        55      1538       Jehane Noujaim   \n",
       "12        71      1550      Larry Brilliant   \n",
       "13       242       527             Jeff Han   \n",
       "14        99      1057  Nicholas Negroponte   \n",
       "15       325      1481         Sirena Huang   \n",
       "16       305      1445         Jennifer Lin   \n",
       "17        88       906            Amy Smith   \n",
       "18       163      1170       Ross Lovegrove   \n",
       "19        84      1201          Jimmy Wales   \n",
       "\n",
       "                                              ratings  \\\n",
       "0   [{'id': 7, 'name': 'Funny', 'count': 19645}, {...   \n",
       "1   [{'id': 7, 'name': 'Funny', 'count': 544}, {'i...   \n",
       "2   [{'id': 7, 'name': 'Funny', 'count': 964}, {'i...   \n",
       "3   [{'id': 3, 'name': 'Courageous', 'count': 760}...   \n",
       "4   [{'id': 9, 'name': 'Ingenious', 'count': 3202}...   \n",
       "5   [{'id': 7, 'name': 'Funny', 'count': 1102}, {'...   \n",
       "6   [{'id': 3, 'name': 'Courageous', 'count': 477}...   \n",
       "7   [{'id': 9, 'name': 'Ingenious', 'count': 195},...   \n",
       "8   [{'id': 3, 'name': 'Courageous', 'count': 562}...   \n",
       "9   [{'id': 21, 'name': 'Unconvincing', 'count': 8...   \n",
       "10  [{'id': 3, 'name': 'Courageous', 'count': 169}...   \n",
       "11  [{'id': 1, 'name': 'Beautiful', 'count': 70}, ...   \n",
       "12  [{'id': 8, 'name': 'Informative', 'count': 188...   \n",
       "13  [{'id': 9, 'name': 'Ingenious', 'count': 1014}...   \n",
       "14  [{'id': 3, 'name': 'Courageous', 'count': 83},...   \n",
       "15  [{'id': 1, 'name': 'Beautiful', 'count': 1671}...   \n",
       "16  [{'id': 1, 'name': 'Beautiful', 'count': 1107}...   \n",
       "17  [{'id': 9, 'name': 'Ingenious', 'count': 321},...   \n",
       "18  [{'id': 1, 'name': 'Beautiful', 'count': 259},...   \n",
       "19  [{'id': 1, 'name': 'Beautiful', 'count': 104},...   \n",
       "\n",
       "                             speaker_occupation  \\\n",
       "0                               Author/educator   \n",
       "1                              Climate advocate   \n",
       "2                          Technology columnist   \n",
       "3            Activist for environmental justice   \n",
       "4          Global health expert; data visionary   \n",
       "5   Life coach; expert in leadership psychology   \n",
       "6                   Actor, comedian, playwright   \n",
       "7                                     Architect   \n",
       "8              Philosopher, cognitive scientist   \n",
       "9                                Pastor, author   \n",
       "10        Co-founder, Architecture for Humanity   \n",
       "11                                    Filmmaker   \n",
       "12               Epidemiologist, philanthropist   \n",
       "13            Human-computer interface designer   \n",
       "14                               Tech visionary   \n",
       "15                                    Violinist   \n",
       "16                            Pianist, composer   \n",
       "17                           inventor, engineer   \n",
       "18                          Industrial designer   \n",
       "19                         Founder of Wikipedia   \n",
       "\n",
       "                                                 tags  \\\n",
       "0   ['children', 'creativity', 'culture', 'dance',...   \n",
       "1   ['alternative energy', 'cars', 'climate change...   \n",
       "2   ['computers', 'entertainment', 'interface desi...   \n",
       "3   ['MacArthur grant', 'activism', 'business', 'c...   \n",
       "4   ['Africa', 'Asia', 'Google', 'demo', 'economic...   \n",
       "5   ['business', 'culture', 'entertainment', 'goal...   \n",
       "6   ['Christianity', 'God', 'atheism', 'comedy', '...   \n",
       "7   ['architecture', 'collaboration', 'culture', '...   \n",
       "8   ['God', 'TED Brain Trust', 'atheism', 'brain',...   \n",
       "9   ['Christianity', 'God', 'culture', 'happiness'...   \n",
       "10  ['activism', 'architecture', 'collaboration', ...   \n",
       "11  ['TED Prize', 'art', 'culture', 'entertainment...   \n",
       "12  ['TED Prize', 'collaboration', 'disease', 'ebo...   \n",
       "13  ['demo', 'design', 'interface design', 'techno...   \n",
       "14  ['children', 'design', 'education', 'entrepren...   \n",
       "15  ['entertainment', 'music', 'performance', 'vio...   \n",
       "16  ['creativity', 'entertainment', 'music', 'perf...   \n",
       "17  ['MacArthur grant', 'alternative energy', 'des...   \n",
       "18  ['DNA', 'biology', 'creativity', 'design', 'in...   \n",
       "19  ['business', 'collaboration', 'culture', 'inve...   \n",
       "\n",
       "                                                title  \\\n",
       "0                         Do schools kill creativity?   \n",
       "1                         Averting the climate crisis   \n",
       "2                                    Simplicity sells   \n",
       "3                                 Greening the ghetto   \n",
       "4                     The best stats you've ever seen   \n",
       "5                                Why we do what we do   \n",
       "6                                   Letting go of God   \n",
       "7              Behind the design of Seattle's library   \n",
       "8   Let's teach religion -- all religion -- in sch...   \n",
       "9                                   A life of purpose   \n",
       "10       My wish: A call for open-source architecture   \n",
       "11                      My wish: A global day of film   \n",
       "12                    My wish: Help me stop pandemics   \n",
       "13   The radical promise of the multi-touch interface   \n",
       "14                               One Laptop per Child   \n",
       "15                    An 11-year-old's magical violin   \n",
       "16                      Improvising on piano, aged 14   \n",
       "17                      Simple designs to save a life   \n",
       "18                 Organic design, inspired by nature   \n",
       "19                             The birth of Wikipedia   \n",
       "\n",
       "                                                  url     views  \\\n",
       "0   https://www.ted.com/talks/ken_robinson_says_sc...  47227110   \n",
       "1   https://www.ted.com/talks/al_gore_on_averting_...   3200520   \n",
       "2   https://www.ted.com/talks/david_pogue_says_sim...   1636292   \n",
       "3   https://www.ted.com/talks/majora_carter_s_tale...   1697550   \n",
       "4   https://www.ted.com/talks/hans_rosling_shows_t...  12005869   \n",
       "5   https://www.ted.com/talks/tony_robbins_asks_wh...  20685401   \n",
       "6   https://www.ted.com/talks/julia_sweeney_on_let...   3769987   \n",
       "7   https://www.ted.com/talks/joshua_prince_ramus_...    967741   \n",
       "8   https://www.ted.com/talks/dan_dennett_s_respon...   2567958   \n",
       "9   https://www.ted.com/talks/rick_warren_on_a_lif...   3095993   \n",
       "10  https://www.ted.com/talks/cameron_sinclair_on_...   1211416   \n",
       "11  https://www.ted.com/talks/jehane_noujaim_inspi...    387877   \n",
       "12  https://www.ted.com/talks/larry_brilliant_want...    693341   \n",
       "13  https://www.ted.com/talks/jeff_han_demos_his_b...   4531020   \n",
       "14  https://www.ted.com/talks/nicholas_negroponte_...    358304   \n",
       "15  https://www.ted.com/talks/sirena_huang_dazzles...   2702470   \n",
       "16  https://www.ted.com/talks/jennifer_lin_improvs...   1628912   \n",
       "17  https://www.ted.com/talks/amy_smith_shares_sim...   1415724   \n",
       "18  https://www.ted.com/talks/ross_lovegrove_share...   1074081   \n",
       "19  https://www.ted.com/talks/jimmy_wales_on_the_b...   1106561   \n",
       "\n",
       "                  Date  \n",
       "0  2006-06-27 00:11:00  \n",
       "1  2006-06-27 00:11:00  \n",
       "2  2006-06-27 00:11:00  \n",
       "3  2006-06-27 00:11:00  \n",
       "4  2006-06-27 20:38:00  \n",
       "5  2006-06-27 20:38:00  \n",
       "6  2006-07-10 00:11:00  \n",
       "7  2006-07-10 00:11:00  \n",
       "8  2006-07-18 00:11:00  \n",
       "9  2006-07-18 00:11:00  \n",
       "10 2006-07-25 00:11:00  \n",
       "11 2006-07-25 00:11:00  \n",
       "12 2006-07-25 00:11:00  \n",
       "13 2006-08-01 00:11:00  \n",
       "14 2006-08-01 00:11:00  \n",
       "15 2006-08-08 00:11:00  \n",
       "16 2006-08-08 00:11:00  \n",
       "17 2006-08-15 00:11:00  \n",
       "18 2006-08-15 00:11:00  \n",
       "19 2006-08-21 00:11:00  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(20)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "4d06fb08",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 2550 entries, 0 to 2549\n",
      "Data columns (total 10 columns):\n",
      " #   Column              Non-Null Count  Dtype         \n",
      "---  ------              --------------  -----         \n",
      " 0   comments            2550 non-null   int64         \n",
      " 1   duration            2550 non-null   int64         \n",
      " 2   main_speaker        2550 non-null   object        \n",
      " 3   ratings             2550 non-null   object        \n",
      " 4   speaker_occupation  2550 non-null   object        \n",
      " 5   tags                2550 non-null   object        \n",
      " 6   title               2550 non-null   object        \n",
      " 7   url                 2550 non-null   object        \n",
      " 8   views               2550 non-null   int64         \n",
      " 9   Date                2550 non-null   datetime64[ns]\n",
      "dtypes: datetime64[ns](1), int64(3), object(6)\n",
      "memory usage: 199.3+ KB\n"
     ]
    }
   ],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "c1db8226",
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"tags\"] = df[\"tags\"].apply(eval)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "35eb433c",
   "metadata": {},
   "outputs": [],
   "source": [
    "link = []\n",
    "for i in df['url']:\n",
    "    link.append(html.A(html.P('Talk url'), href=i, target='_blank'))\n",
    "df['url'] = link\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "98ab0c3b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# preprocessing column ratings\n",
    "df1 = pd.concat([pd.DataFrame(eval(x)) for x in df['ratings']], keys=df.index).reset_index(level=1, drop=True)\n",
    "df = df.join(df1).reset_index(drop=True)\n",
    "df = df.loc[df.groupby(['title'])['count'].idxmax()]\n",
    "new_data = df.drop(['id', 'ratings'], axis=1).reset_index(drop=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "5204a15e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2550\n"
     ]
    }
   ],
   "source": [
    "print(len(new_data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "162177d4",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_dataFrame():\n",
    "    return new_data\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}