02_01_Tokenization.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "5c158be7",
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.tokenize import word_tokenize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "8b555650",
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "7cd63e6a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Do', \"n't\", 'be', 'fooled', 'by', 'the', 'dark', 'sounding', 'name', ',', 'Mr.', 'Jone', \"'s\", 'Orphanage', 'is', 'as', 'cheery', 'as', 'cheery', 'goes', 'for', 'a', 'pastry', 'shop', '.']\n"
     ]
    }
   ],
   "source": [
    "from nltk.tokenize import word_tokenize  \n",
    "print(word_tokenize(\"Don't be fooled by the dark sounding name, Mr. Jone's Orphanage is as cheery as cheery goes for a pastry shop.\")) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "a0196f02",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Don', \"'\", 't', 'be', 'fooled', 'by', 'the', 'dark', 'sounding', 'name', ',', 'Mr', '.', 'Jone', \"'\", 's', 'Orphanage', 'is', 'as', 'cheery', 'as', 'cheery', 'goes', 'for', 'a', 'pastry', 'shop', '.']\n"
     ]
    }
   ],
   "source": [
    "from nltk.tokenize import WordPunctTokenizer  \n",
    "print(WordPunctTokenizer().tokenize(\"Don't be fooled by the dark sounding name, Mr. Jone's Orphanage is as cheery as cheery goes for a pastry shop.\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "da3caaf6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[\"don't\", 'be', 'fooled', 'by', 'the', 'dark', 'sounding', 'name', 'mr', \"jone's\", 'orphanage', 'is', 'as', 'cheery', 'as', 'cheery', 'goes', 'for', 'a', 'pastry', 'shop']\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.keras.preprocessing.text import text_to_word_sequence\n",
    "print(text_to_word_sequence(\"Don't be fooled by the dark sounding name, Mr. Jone's Orphanage is as cheery as cheery goes for a pastry shop.\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "b1790ee6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['안녕하세요.', '저는 전영준입니다.']\n"
     ]
    }
   ],
   "source": [
    "import kss\n",
    "text = \"안녕하세요. 저는 전영준입니다.\"\n",
    "print(kss.split_sentences(text))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "98eee47f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['열심히', '코딩', '하', 'ㄴ', '당신', ',', '연휴', '에', '는', '여행', '을', '가보', '아요']\n",
      "[('열심히', 'MAG'), ('코딩', 'NNG'), ('하', 'XSV'), ('ㄴ', 'ETD'), ('당신', 'NP'), (',', 'SP'), ('연휴', 'NNG'), ('에', 'JKM'), ('는', 'JX'), ('여행', 'NNG'), ('을', 'JKO'), ('가보', 'VV'), ('아요', 'EFN')]\n",
      "['코딩', '당신', '연휴', '여행']\n"
     ]
    }
   ],
   "source": [
    "import konlpy\n",
    "from konlpy.tag import Kkma\n",
    "kkma=Kkma()\n",
    "print(kkma.morphs(\"열심히 코딩한 당신, 연휴에는 여행을 가봐요\"))\n",
    "print(kkma.pos(\"열심히 코딩한 당신, 연휴에는 여행을 가봐요\"))  \n",
    "print(kkma.nouns(\"열심히 코딩한 당신, 연휴에는 여행을 가봐요\"))  "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}