Lesson5.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 并发编程"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "### 创建线程"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import threading\n",
    "import time\n",
    " \n",
    "def counter(n):  \n",
    "    cnt = 0;  \n",
    "    for i in xrange(n):  \n",
    "        cnt += 1\n",
    "        time.sleep(0.1)\n",
    "        print cnt             \n",
    "               \n",
    "th = threading.Thread(target=counter, args=(10,));   \n",
    "th.start();  \n",
    "th.join(); \n",
    "print 'main thread task done'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 创建线程-继承类"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import threading, time, random  \n",
    "  \n",
    "def counter():  \n",
    "    cnt = 0;  \n",
    "    for i in xrange(10000):  \n",
    "        for j in xrange(i):  \n",
    "            cnt += j;  \n",
    "  \n",
    "class SubThread(threading.Thread):  \n",
    "    def __init__(self, name):  \n",
    "        threading.Thread.__init__(self, name=name);  \n",
    "        pass\n",
    "  \n",
    "    def run(self):  \n",
    "        i = 0;  \n",
    "        while i < 3:  \n",
    "            print self.name,'counting...\\n';  \n",
    "            counter();  \n",
    "            print self.name,'finish\\n';  \n",
    "            i += 1;  \n",
    "\n",
    "th = SubThread('thread-1');  \n",
    "th.start();  \n",
    "th.join();  \n",
    "print 'all done'; "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import threading, time\n",
    "  \n",
    "class SubThread(threading.Thread):  \n",
    "    def __init__(self, name):  \n",
    "        threading.Thread.__init__(self, name=name);  \n",
    "  \n",
    "    def run(self):  \n",
    "        i = 0;  \n",
    "        while i < 3:  \n",
    "            print self.name,'counting...\\n';  \n",
    "            time.sleep(1) \n",
    "            print self.name,'finish\\n';  \n",
    "            i += 1;  \n",
    "\n",
    "th = SubThread('thread-1');\n",
    "print 'main start'\n",
    "th.setDaemon(False)\n",
    "th.start()  \n",
    "th.join() \n",
    "print 'main end'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Daemon守护线程\n",
    "### 外部运行"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 线程同步"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from threading import Thread\n",
    "some_var = 0\n",
    "class IncrementThread(Thread):\n",
    "    def run(self):\n",
    "        global some_var\n",
    "        read_value = some_var\n",
    "        print \"some_var in %s is %d\" % (self.name, read_value)\n",
    "        some_var = read_value + 1\n",
    "        #print \"some_var in %s after increment is %d\" % (self.name, some_var)\n",
    "def use_increment_thread():\n",
    "    threads = []\n",
    "    for i in range(50):\n",
    "        t = IncrementThread()\n",
    "        threads.append(t)\n",
    "        t.start()\n",
    "    for t in threads:\n",
    "        t.join()\n",
    "    print \"After 50 modifications, some_var should have become 50\"\n",
    "    print \"After 50 modifications, some_var is %d\" % (some_var,)\n",
    "use_increment_thread()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# lock\n",
    "# 可以通过下面两种方式创建一个Lock对象，新创建的 Lock 对象处于未上锁的状态：\n",
    "import threading\n",
    "l = threading.Lock()\n",
    "l"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from threading import Lock, Thread\n",
    "lock = Lock()\n",
    "some_var = 0\n",
    "class IncrementThread(Thread):\n",
    "    def run(self):\n",
    "        #we want to read a global variable\n",
    "        #and then increment it\n",
    "        global some_var\n",
    "        lock.acquire()\n",
    "        read_value = some_var\n",
    "        print \"some_var in %s is %d\" % (self.name, read_value)\n",
    "        some_var = read_value + 1\n",
    "        print \"some_var in %s after increment is %d\" % (self.name, some_var)\n",
    "        lock.release()\n",
    "def use_increment_thread():\n",
    "    threads = []\n",
    "    for i in range(50):\n",
    "        t = IncrementThread()\n",
    "        threads.append(t)\n",
    "        t.start()\n",
    "    for t in threads:\n",
    "        t.join()\n",
    "    print \"After 50 modifications, some_var should have become 50\"\n",
    "    print \"After 50 modifications, some_var is %d\" % (some_var,)\n",
    "use_increment_thread()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 不加锁计数器\n",
    "import time\n",
    "from threading import Thread\n",
    "value = 0\n",
    "def getlock():\n",
    "    global value\n",
    "    new = value + 1\n",
    "    time.sleep(0.001)  # 使用sleep让线程有机会切换\n",
    "    value = new\n",
    "threads = []\n",
    "for i in range(100):\n",
    "    t = Thread(target=getlock)\n",
    "    t.start()\n",
    "    threads.append(t)\n",
    "for t in threads:\n",
    "    t.join()\n",
    "print value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 加锁保证结果\n",
    "import time\n",
    "from threading import Thread, Lock\n",
    "value = 0\n",
    "lock = Lock()\n",
    "def getlock():\n",
    "    global value\n",
    "    with lock:\n",
    "        new = value + 1\n",
    "        time.sleep(0.001)\n",
    "        value = new\n",
    "threads = []\n",
    "for i in range(100):\n",
    "    t = Thread(target=getlock)\n",
    "    t.start()\n",
    "    threads.append(t)\n",
    "for t in threads:\n",
    "    t.join()\n",
    "print value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import threading  \n",
    "import time  \n",
    "  \n",
    "def test_xc(): \n",
    "    mutex.acquire()#取得锁  \n",
    "    f = open(\"test.txt\",\"a\")  \n",
    "    f.write(\"test_dxc\"+'\\n')  \n",
    "    f.close()  \n",
    "    mutex.release()#释放锁  \n",
    " \n",
    "mutex = threading.Lock()#创建锁 \n",
    "threads = []\n",
    "for i in xrange(5):  \n",
    "    t = threading.Thread(target=test_xc)  \n",
    "    t.start()  \n",
    "    threads.append(t)\n",
    "for t in threads:\n",
    "    t.join()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 可重入锁"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import threading\n",
    "\n",
    "print 'lock acquire'\n",
    "lock = threading.Lock()\n",
    "lock.acquire()\n",
    "lock.acquire()\n",
    "lock.release()\n",
    "lock.release()\n",
    "print 'done'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import threading\n",
    "\n",
    "print 'lock acquire'\n",
    "lock = threading.RLock()\n",
    "lock.acquire()\n",
    "lock.acquire()\n",
    "lock.release()\n",
    "lock.release()\n",
    "print 'done'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Condition"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import threading, time\n",
    "class Hider(threading.Thread):\n",
    "    def __init__(self, cond, name):\n",
    "        super(Hider, self).__init__()\n",
    "        self.cond = cond\n",
    "        self.name = name\n",
    "    def run(self):\n",
    "        time.sleep(1) #确保先运行Seeker中的方法\n",
    "        self.cond.acquire() #b\n",
    "        print self.name + ': 我已经把眼睛蒙上了'\n",
    "        self.cond.notify()\n",
    "        self.cond.wait() #c\n",
    "                         #f\n",
    "        print self.name + ': 我找到你了 ~_~'\n",
    "        self.cond.notify()\n",
    "        self.cond.release()\n",
    "                            #g\n",
    "        print self.name + ': 我赢了'   #h\n",
    "class Seeker(threading.Thread):\n",
    "    def __init__(self, cond, name):\n",
    "        super(Seeker, self).__init__()\n",
    "        self.cond = cond\n",
    "        self.name = name\n",
    "    def run(self):\n",
    "        self.cond.acquire()\n",
    "        self.cond.wait()    #a    #释放对琐的占用，同时线程挂起在这里，直到被notify并重新占有琐。\n",
    "                            #d\n",
    "        print self.name + ': 我已经藏好了，你快来找我吧'\n",
    "        self.cond.notify()\n",
    "        self.cond.wait()    #e\n",
    "                            #h\n",
    "        self.cond.release()\n",
    "        print self.name + ': 被你找到了，哎~~~'\n",
    "cond = threading.Condition()\n",
    "seeker = Seeker(cond, 'seeker')\n",
    "hider = Hider(cond, 'hider')\n",
    "seeker.start()\n",
    "hider.start()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Event 交通灯"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import threading\n",
    "import random\n",
    "import time\n",
    "\n",
    "\n",
    "class VehicleThread(threading.Thread):\n",
    "    \"\"\"Class representing a motor vehicle at an intersection\"\"\"\n",
    "\n",
    "    def __init__(self, threadName, event):\n",
    "        \"\"\"Initializes thread\"\"\"\n",
    "\n",
    "        threading.Thread.__init__(self, name=threadName)\n",
    "\n",
    "        # ensures that each vehicle waits for a green light\n",
    "        self.threadEvent = event\n",
    "\n",
    "    def run(self):\n",
    "        \"\"\"Vehicle waits unless/until light is green\"\"\"\n",
    "\n",
    "        # stagger arrival times\n",
    "        time.sleep(random.randrange(1, 10))\n",
    "\n",
    "        # prints arrival time of car at intersection\n",
    "        print \"%s arrived at %s\\n\" % \\\n",
    "              (self.getName(), time.ctime(time.time()))\n",
    "\n",
    "        # wait for green light\n",
    "        self.threadEvent.wait()\n",
    "\n",
    "        # displays time that car departs intersection\n",
    "        print \"%s passes through intersection at %s\\n\" % \\\n",
    "              (self.getName(), time.ctime(time.time()))\n",
    "\n",
    "\n",
    "greenLight = threading.Event()\n",
    "vehicleThreads = []\n",
    "\n",
    "# creates and starts five Vehicle threads\n",
    "for i in range(1, 5):\n",
    "    vehicleThreads.append(VehicleThread(\"Vehicle\" + str(i),\n",
    "                                        greenLight))\n",
    "\n",
    "for vehicle in vehicleThreads:\n",
    "    vehicle.start()\n",
    "\n",
    "while threading.activeCount() > 1:\n",
    "    # sets the Event's flag to false -- block all incoming vehicles\n",
    "    greenLight.clear()\n",
    "    print \"RED LIGHT! at\", time.ctime(time.time())\n",
    "    time.sleep(3)\n",
    "\n",
    "    # sets the Event's flag to true -- awaken all waiting vehicles\n",
    "    print \"GREEN LIGHT! at\", time.ctime(time.time())\n",
    "    greenLight.set()\n",
    "    time.sleep(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## 信号量"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 acquire sema1 acquire sema 2 acquire sema\n",
      "\n",
      "\n",
      "2 release sema3 acquire sema\n",
      "\n",
      "4 acquire sema\n",
      "0 release sema\n",
      "3 release sema\n",
      "4 release sema\n",
      "1 release sema\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "from random import random\n",
    "from threading import Thread, Semaphore\n",
    "sema = Semaphore(3)\n",
    "def foo(tid):\n",
    "    with sema:\n",
    "        print '{} acquire sema'.format(tid)\n",
    "        wt = random() * 2\n",
    "        time.sleep(wt)\n",
    "    print '{} release sema'.format(tid)\n",
    "threads = []\n",
    "for i in range(5):\n",
    "    t = Thread(target=foo, args=(i,))\n",
    "    threads.append(t)\n",
    "    t.start()\n",
    "for t in threads:\n",
    "    t.join()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 进程"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "called function in process: 0\n",
      "called function in process: 1\n",
      "called function in process: 2\n",
      "called function in process: 3\n",
      "called function in process: 4\n"
     ]
    }
   ],
   "source": [
    "import multiprocessing\n",
    "\n",
    "def foo(i):\n",
    "   print 'called function in process: %s' % i\n",
    "   return\n",
    "    \n",
    "Process_jobs = []\n",
    "for i in range(5):\n",
    "   p = multiprocessing.Process(target=foo, args=(i,))\n",
    "   Process_jobs.append(p)\n",
    "   p.start()\n",
    "   p.join()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 后台进程"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting background_process \n",
      "\n",
      "Starting NO_background_process \n",
      "\n",
      "Exiting background_process \n",
      "\n",
      "Exiting NO_background_process \n",
      "\n"
     ]
    }
   ],
   "source": [
    "import multiprocessing\n",
    "import time\n",
    "\n",
    "def foo():\n",
    "   name = multiprocessing.current_process().name\n",
    "   print (\"Starting %s \\n\" %name)\n",
    "   time.sleep(3)\n",
    "   print (\"Exiting %s \\n\" %name)\n",
    "\n",
    "background_process = multiprocessing.Process\\\n",
    "                    (name='background_process',\\\n",
    "                     target=foo)\n",
    "background_process.daemon = True\n",
    "NO_background_process = multiprocessing.Process\\\n",
    "                          (name='NO_background_process',\\\n",
    "                           target=foo)\n",
    "NO_background_process.daemon = False\n",
    "background_process.start()\n",
    "NO_background_process.start()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 杀死进程"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('Process before execution:', <Process(Process-#Test#, initial)>, False)\n",
      "('Process running:', <Process(Process-#Test#, started)>, True)\n",
      "('Process terminated:', <Process(Process-#Test#, started)>, True)\n",
      "('Process joined:', <Process(Process-#Test#, stopped[SIGTERM])>, False)\n",
      "('Process exit code:', -15)\n"
     ]
    }
   ],
   "source": [
    "import multiprocessing\n",
    "import time\n",
    "def foo():\n",
    "    print ('Starting function')\n",
    "    time.sleep(0.1)\n",
    "    print ('Finished function')\n",
    "\n",
    "p = multiprocessing.Process(target=foo, name='Process-#Test#')\n",
    "print ('Process before execution:', p, p.is_alive())\n",
    "p.start()\n",
    "print ('Process running:', p, p.is_alive())\n",
    "p.terminate()\n",
    "print ('Process terminated:', p, p.is_alive())\n",
    "p.join()\n",
    "print ('Process joined:', p, p.is_alive())\n",
    "print ('Process exit code:', p.exitcode)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## 继承创建进程"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "called run method in process: MyProcess-9\n",
      "called run method in process: MyProcess-10\n",
      "called run method in process: MyProcess-11\n",
      "called run method in process: MyProcess-12\n",
      "called run method in process: MyProcess-13\n"
     ]
    }
   ],
   "source": [
    "import multiprocessing\n",
    "class MyProcess(multiprocessing.Process):\n",
    "  def run(self):\n",
    "     print ('called run method in process: %s' %self.name)\n",
    "     return\n",
    "\n",
    "\n",
    "jobs = []\n",
    "for i in range(5):\n",
    "    p = MyProcess ()\n",
    "    jobs.append(p)\n",
    "    p.start()\n",
    "    p.join()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 进程间通信"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 进程共享内存"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3.1415927\n",
      "[0, -1, -2, -3, -4, -5, -6, -7, -8, -9]\n"
     ]
    }
   ],
   "source": [
    "from multiprocessing import Process, Value, Array\n",
    "\n",
    "def f(n, a):\n",
    "    n.value = 3.1415927\n",
    "    for i in range(len(a)):\n",
    "        a[i] = -a[i]\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    num = Value('d', 0.0)\n",
    "    arr = Array('i', range(10))\n",
    "\n",
    "    p = Process(target=f, args=(num, arr))\n",
    "    p.start()\n",
    "    p.join()\n",
    "\n",
    "    print num.value\n",
    "    print arr[:]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 进程共享-队列"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import multiprocessing\n",
    "import random\n",
    "import time\n",
    "class producer(multiprocessing.Process):\n",
    "    def __init__(self, queue):\n",
    "        multiprocessing.Process.__init__(self)\n",
    "        self.queue = queue\n",
    "    def run(self) :\n",
    "        for i in range(10):\n",
    "            item = random.randint(0, 256)\n",
    "            self.queue.put(item)\n",
    "            print (\"Process Producer : item %d appended to queue %s\" % (item,self.name))\n",
    "            time.sleep(1)\n",
    "            print (\"The size of queue is %s\" % self.queue.qsize())\n",
    "class consumer(multiprocessing.Process):\n",
    "    def __init__(self, queue):\n",
    "        multiprocessing.Process.__init__(self)\n",
    "        self.queue = queue\n",
    "    def run(self):\n",
    "        while True:\n",
    "            if (self.queue.empty()):\n",
    "                print(\"the queue is empty\")\n",
    "                break\n",
    "            else :\n",
    "                time.sleep(2)\n",
    "                item = self.queue.get()\n",
    "                print ('Process Consumer : item %d popped from by %s \\n' % (item, self.name))\n",
    "        time.sleep(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Process Producer : item 15 appended to queue producer-15\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Process producer-15:\n",
      "Traceback (most recent call last):\n",
      "  File \"/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/multiprocessing/process.py\", line 258, in _bootstrap\n",
      "    self.run()\n",
      "  File \"<ipython-input-7-ec6a1a3b1a52>\", line 14, in run\n",
      "    print (\"The size of queue is %s\" % self.queue.qsize())\n",
      "  File \"/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/multiprocessing/queues.py\", line 143, in qsize\n",
      "    return self._maxsize - self._sem._semlock._get_value()\n",
      "NotImplementedError\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Process Consumer : item 15 popped from by consumer-16 \n",
      "\n",
      "the queue is empty\n"
     ]
    }
   ],
   "source": [
    "queue = multiprocessing.Queue()\n",
    "process_producer = producer(queue)\n",
    "process_consumer = consumer(queue)\n",
    "process_producer.start()\n",
    "process_consumer.start()\n",
    "process_producer.join()\n",
    "process_consumer.join()\n",
    "\n",
    "# 如果多个consumer，进程同步"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 管道"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Producer Received: Consumer Words\n",
      "Consumer Received: Producer Words\n",
      "Ended!\n"
     ]
    }
   ],
   "source": [
    "from multiprocessing import Process, Pipe\n",
    " \n",
    " \n",
    "class Consumer(Process):\n",
    "    def __init__(self, pipe):\n",
    "        Process.__init__(self)\n",
    "        self.pipe = pipe\n",
    " \n",
    "    def run(self):\n",
    "        self.pipe.send(\"Consumer Words\")\n",
    "        print \"Consumer Received:\", self.pipe.recv()\n",
    " \n",
    " \n",
    "class Producer(Process):\n",
    "    def __init__(self, pipe):\n",
    "        Process.__init__(self)\n",
    "        self.pipe = pipe\n",
    " \n",
    "    def run(self):\n",
    "        print \"Producer Received:\", self.pipe.recv()\n",
    "        self.pipe.send(\"Producer Words\")\n",
    " \n",
    " \n",
    "pipe = Pipe()\n",
    "p = Producer(pipe[0])\n",
    "c = Consumer(pipe[1])\n",
    "p.daemon = c.daemon = True\n",
    "p.start()\n",
    "c.start()\n",
    "p.join()\n",
    "c.join()\n",
    "print \"Ended!\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 进程池"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start process:  0\n",
      "End process 0\n",
      "Start process:  1\n",
      "End process 1\n",
      "Start process:  2\n",
      "End process 2\n",
      "Start process:  3\n",
      "End process 3\n",
      "Started processes\n",
      "Subprocess done.\n"
     ]
    }
   ],
   "source": [
    "#阻塞方式\n",
    "from multiprocessing import Lock, Pool\n",
    "import time\n",
    " \n",
    " \n",
    "def function(index):\n",
    "    print \"Start process: \", index\n",
    "    time.sleep(3)\n",
    "    print \"End process\", index\n",
    " \n",
    " \n",
    "pool = Pool(processes=3)\n",
    "for i in xrange(4):\n",
    "    pool.apply(function, (i,))\n",
    "\n",
    "print \"Started processes\"\n",
    "pool.close()\n",
    "pool.join()\n",
    "print \"Subprocess done.\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start process:  1\n",
      "Start process:  0\n",
      "Start process:  2\n",
      "Started processes\n",
      "End process 0\n",
      "End process 2\n",
      "End process 1\n",
      "Start process:  3\n",
      "End process 3\n",
      "Subprocess done.\n"
     ]
    }
   ],
   "source": [
    "# 非阻塞方式\n",
    "from multiprocessing import Lock, Pool\n",
    "import time\n",
    " \n",
    " \n",
    "def function(index):\n",
    "    print \"Start process: \", index\n",
    "    time.sleep(3)\n",
    "    print \"End process\", index\n",
    " \n",
    " \n",
    "pool = Pool(processes=3)\n",
    "for i in xrange(4):\n",
    "    pool.apply_async(function, (i,))\n",
    "\n",
    "print \"Started processes\"\n",
    "pool.close()\n",
    "pool.join()\n",
    "print \"Subprocess done.\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 正则表达式"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "s = r\"<html><body><h1>hello world</h1></body></html>\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hello world\n"
     ]
    }
   ],
   "source": [
    "start = s.find(\"<h1>\")\n",
    "end = s.find(\"</h1>\")\n",
    "print s[start+4:end]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Help on module re:\n",
      "\n",
      "NAME\n",
      "    re - Support for regular expressions (RE).\n",
      "\n",
      "FILE\n",
      "    /Users/heming03/python-env/lib/python2.7/re.py\n",
      "\n",
      "DESCRIPTION\n",
      "    This module provides regular expression matching operations similar to\n",
      "    those found in Perl.  It supports both 8-bit and Unicode strings; both\n",
      "    the pattern and the strings being processed can contain null bytes and\n",
      "    characters outside the US ASCII range.\n",
      "    \n",
      "    Regular expressions can contain both special and ordinary characters.\n",
      "    Most ordinary characters, like \"A\", \"a\", or \"0\", are the simplest\n",
      "    regular expressions; they simply match themselves.  You can\n",
      "    concatenate ordinary characters, so last matches the string 'last'.\n",
      "    \n",
      "    The special characters are:\n",
      "        \".\"      Matches any character except a newline.\n",
      "        \"^\"      Matches the start of the string.\n",
      "        \"$\"      Matches the end of the string or just before the newline at\n",
      "                 the end of the string.\n",
      "        \"*\"      Matches 0 or more (greedy) repetitions of the preceding RE.\n",
      "                 Greedy means that it will match as many repetitions as possible.\n",
      "        \"+\"      Matches 1 or more (greedy) repetitions of the preceding RE.\n",
      "        \"?\"      Matches 0 or 1 (greedy) of the preceding RE.\n",
      "        *?,+?,?? Non-greedy versions of the previous three special characters.\n",
      "        {m,n}    Matches from m to n repetitions of the preceding RE.\n",
      "        {m,n}?   Non-greedy version of the above.\n",
      "        \"\\\\\"     Either escapes special characters or signals a special sequence.\n",
      "        []       Indicates a set of characters.\n",
      "                 A \"^\" as the first character indicates a complementing set.\n",
      "        \"|\"      A|B, creates an RE that will match either A or B.\n",
      "        (...)    Matches the RE inside the parentheses.\n",
      "                 The contents can be retrieved or matched later in the string.\n",
      "        (?iLmsux) Set the I, L, M, S, U, or X flag for the RE (see below).\n",
      "        (?:...)  Non-grouping version of regular parentheses.\n",
      "        (?P<name>...) The substring matched by the group is accessible by name.\n",
      "        (?P=name)     Matches the text matched earlier by the group named name.\n",
      "        (?#...)  A comment; ignored.\n",
      "        (?=...)  Matches if ... matches next, but doesn't consume the string.\n",
      "        (?!...)  Matches if ... doesn't match next.\n",
      "        (?<=...) Matches if preceded by ... (must be fixed length).\n",
      "        (?<!...) Matches if not preceded by ... (must be fixed length).\n",
      "        (?(id/name)yes|no) Matches yes pattern if the group with id/name matched,\n",
      "                           the (optional) no pattern otherwise.\n",
      "    \n",
      "    The special sequences consist of \"\\\\\" and a character from the list\n",
      "    below.  If the ordinary character is not on the list, then the\n",
      "    resulting RE will match the second character.\n",
      "        \\number  Matches the contents of the group of the same number.\n",
      "        \\A       Matches only at the start of the string.\n",
      "        \\Z       Matches only at the end of the string.\n",
      "        \\b       Matches the empty string, but only at the start or end of a word.\n",
      "        \\B       Matches the empty string, but not at the start or end of a word.\n",
      "        \\d       Matches any decimal digit; equivalent to the set [0-9].\n",
      "        \\D       Matches any non-digit character; equivalent to the set [^0-9].\n",
      "        \\s       Matches any whitespace character; equivalent to [ \\t\\n\\r\\f\\v].\n",
      "        \\S       Matches any non-whitespace character; equiv. to [^ \\t\\n\\r\\f\\v].\n",
      "        \\w       Matches any alphanumeric character; equivalent to [a-zA-Z0-9_].\n",
      "                 With LOCALE, it will match the set [0-9_] plus characters defined\n",
      "                 as letters for the current locale.\n",
      "        \\W       Matches the complement of \\w.\n",
      "        \\\\       Matches a literal backslash.\n",
      "    \n",
      "    This module exports the following functions:\n",
      "        match    Match a regular expression pattern to the beginning of a string.\n",
      "        search   Search a string for the presence of a pattern.\n",
      "        sub      Substitute occurrences of a pattern found in a string.\n",
      "        subn     Same as sub, but also return the number of substitutions made.\n",
      "        split    Split a string by the occurrences of a pattern.\n",
      "        findall  Find all occurrences of a pattern in a string.\n",
      "        finditer Return an iterator yielding a match object for each match.\n",
      "        compile  Compile a pattern into a RegexObject.\n",
      "        purge    Clear the regular expression cache.\n",
      "        escape   Backslash all non-alphanumerics in a string.\n",
      "    \n",
      "    Some of the functions in this module takes flags as optional parameters:\n",
      "        I  IGNORECASE  Perform case-insensitive matching.\n",
      "        L  LOCALE      Make \\w, \\W, \\b, \\B, dependent on the current locale.\n",
      "        M  MULTILINE   \"^\" matches the beginning of lines (after a newline)\n",
      "                       as well as the string.\n",
      "                       \"$\" matches the end of lines (before a newline) as well\n",
      "                       as the end of the string.\n",
      "        S  DOTALL      \".\" matches any character at all, including the newline.\n",
      "        X  VERBOSE     Ignore whitespace and comments for nicer looking RE's.\n",
      "        U  UNICODE     Make \\w, \\W, \\b, \\B, dependent on the Unicode locale.\n",
      "    \n",
      "    This module also defines an exception 'error'.\n",
      "\n",
      "CLASSES\n",
      "    exceptions.Exception(exceptions.BaseException)\n",
      "        sre_constants.error\n",
      "    \n",
      "    class error(exceptions.Exception)\n",
      "     |  Method resolution order:\n",
      "     |      error\n",
      "     |      exceptions.Exception\n",
      "     |      exceptions.BaseException\n",
      "     |      __builtin__.object\n",
      "     |  \n",
      "     |  Data descriptors defined here:\n",
      "     |  \n",
      "     |  __weakref__\n",
      "     |      list of weak references to the object (if defined)\n",
      "     |  \n",
      "     |  ----------------------------------------------------------------------\n",
      "     |  Methods inherited from exceptions.Exception:\n",
      "     |  \n",
      "     |  __init__(...)\n",
      "     |      x.__init__(...) initializes x; see help(type(x)) for signature\n",
      "     |  \n",
      "     |  ----------------------------------------------------------------------\n",
      "     |  Data and other attributes inherited from exceptions.Exception:\n",
      "     |  \n",
      "     |  __new__ = <built-in method __new__ of type object>\n",
      "     |      T.__new__(S, ...) -> a new object with type S, a subtype of T\n",
      "     |  \n",
      "     |  ----------------------------------------------------------------------\n",
      "     |  Methods inherited from exceptions.BaseException:\n",
      "     |  \n",
      "     |  __delattr__(...)\n",
      "     |      x.__delattr__('name') <==> del x.name\n",
      "     |  \n",
      "     |  __getattribute__(...)\n",
      "     |      x.__getattribute__('name') <==> x.name\n",
      "     |  \n",
      "     |  __getitem__(...)\n",
      "     |      x.__getitem__(y) <==> x[y]\n",
      "     |  \n",
      "     |  __getslice__(...)\n",
      "     |      x.__getslice__(i, j) <==> x[i:j]\n",
      "     |      \n",
      "     |      Use of negative indices is not supported.\n",
      "     |  \n",
      "     |  __reduce__(...)\n",
      "     |  \n",
      "     |  __repr__(...)\n",
      "     |      x.__repr__() <==> repr(x)\n",
      "     |  \n",
      "     |  __setattr__(...)\n",
      "     |      x.__setattr__('name', value) <==> x.name = value\n",
      "     |  \n",
      "     |  __setstate__(...)\n",
      "     |  \n",
      "     |  __str__(...)\n",
      "     |      x.__str__() <==> str(x)\n",
      "     |  \n",
      "     |  __unicode__(...)\n",
      "     |  \n",
      "     |  ----------------------------------------------------------------------\n",
      "     |  Data descriptors inherited from exceptions.BaseException:\n",
      "     |  \n",
      "     |  __dict__\n",
      "     |  \n",
      "     |  args\n",
      "     |  \n",
      "     |  message\n",
      "\n",
      "FUNCTIONS\n",
      "    compile(pattern, flags=0)\n",
      "        Compile a regular expression pattern, returning a pattern object.\n",
      "    \n",
      "    escape(pattern)\n",
      "        Escape all non-alphanumeric characters in pattern.\n",
      "    \n",
      "    findall(pattern, string, flags=0)\n",
      "        Return a list of all non-overlapping matches in the string.\n",
      "        \n",
      "        If one or more groups are present in the pattern, return a\n",
      "        list of groups; this will be a list of tuples if the pattern\n",
      "        has more than one group.\n",
      "        \n",
      "        Empty matches are included in the result.\n",
      "    \n",
      "    finditer(pattern, string, flags=0)\n",
      "        Return an iterator over all non-overlapping matches in the\n",
      "        string.  For each match, the iterator returns a match object.\n",
      "        \n",
      "        Empty matches are included in the result.\n",
      "    \n",
      "    match(pattern, string, flags=0)\n",
      "        Try to apply the pattern at the start of the string, returning\n",
      "        a match object, or None if no match was found.\n",
      "    \n",
      "    purge()\n",
      "        Clear the regular expression cache\n",
      "    \n",
      "    search(pattern, string, flags=0)\n",
      "        Scan through string looking for a match to the pattern, returning\n",
      "        a match object, or None if no match was found.\n",
      "    \n",
      "    split(pattern, string, maxsplit=0, flags=0)\n",
      "        Split the source string by the occurrences of the pattern,\n",
      "        returning a list containing the resulting substrings.\n",
      "    \n",
      "    sub(pattern, repl, string, count=0, flags=0)\n",
      "        Return the string obtained by replacing the leftmost\n",
      "        non-overlapping occurrences of the pattern in string by the\n",
      "        replacement repl.  repl can be either a string or a callable;\n",
      "        if a string, backslash escapes in it are processed.  If it is\n",
      "        a callable, it's passed the match object and must return\n",
      "        a replacement string to be used.\n",
      "    \n",
      "    subn(pattern, repl, string, count=0, flags=0)\n",
      "        Return a 2-tuple containing (new_string, number).\n",
      "        new_string is the string obtained by replacing the leftmost\n",
      "        non-overlapping occurrences of the pattern in the source\n",
      "        string by the replacement repl.  number is the number of\n",
      "        substitutions that were made. repl can be either a string or a\n",
      "        callable; if a string, backslash escapes in it are processed.\n",
      "        If it is a callable, it's passed the match object and must\n",
      "        return a replacement string to be used.\n",
      "    \n",
      "    template(pattern, flags=0)\n",
      "        Compile a template pattern, returning a pattern object\n",
      "\n",
      "DATA\n",
      "    DOTALL = 16\n",
      "    I = 2\n",
      "    IGNORECASE = 2\n",
      "    L = 4\n",
      "    LOCALE = 4\n",
      "    M = 8\n",
      "    MULTILINE = 8\n",
      "    S = 16\n",
      "    U = 32\n",
      "    UNICODE = 32\n",
      "    VERBOSE = 64\n",
      "    X = 64\n",
      "    __all__ = ['match', 'search', 'sub', 'subn', 'split', 'findall', 'comp...\n",
      "    __version__ = '2.2.1'\n",
      "\n",
      "VERSION\n",
      "    2.2.1\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import re\n",
    "help(re)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 导入库\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hello world\n"
     ]
    }
   ],
   "source": [
    "p1 = r\".*<h1>(.*?)</h1>.*\"\n",
    "pattern = re.compile(p1)\n",
    "groups = re.match(pattern, s)\n",
    "print groups.group(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 查找字符串 Match & Search\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tiger ige\n"
     ]
    }
   ],
   "source": [
    "name=\"Hello,My name is tiger,nice to meet you...\"\n",
    "k=re.search(r't(ige)r',name)\n",
    "if k:\n",
    "    print k.group(0),k.group(1)\n",
    "else:\n",
    "    print \"not search!\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Hello \n",
      "ello\n"
     ]
    }
   ],
   "source": [
    "name=\"Hello,My name is tiger,nice to meet you...\"\n",
    "k=re.match(r\"H(....)\", name)\n",
    "if k:\n",
    "    print k.group(0),'\\n',k.group(1)\n",
    "else:\n",
    "    print \"not match!\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 查找所有 FindAll & FindIter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['user01@mail.com', 'user02@mail.com', 'user04@mail.com']"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mail='<user01@mail.com> <user02@mail.com> user04@mail.com'\n",
    "re.findall(r'(\\w+@m....[a-z]{3})', mail)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "mail_list_iter = re.finditer(r'(\\w+@m....[a-z]{3})', mail)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<type '_sre.SRE_Match'>\n",
      "user01@mail.com\n",
      "<type '_sre.SRE_Match'>\n",
      "user02@mail.com\n",
      "<type '_sre.SRE_Match'>\n",
      "user04@mail.com\n"
     ]
    }
   ],
   "source": [
    "for i in mail_list_iter:\n",
    "    print type(i)\n",
    "    print i.group()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 替换"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Help on function sub in module re:\n",
      "\n",
      "sub(pattern, repl, string, count=0, flags=0)\n",
      "    Return the string obtained by replacing the leftmost\n",
      "    non-overlapping occurrences of the pattern in string by the\n",
      "    replacement repl.  repl can be either a string or a callable;\n",
      "    if a string, backslash escapes in it are processed.  If it is\n",
      "    a callable, it's passed the match object and must return\n",
      "    a replacement string to be used.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "help(re.sub)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Hi,-nice-to-meet-you-where-are-you-from?'"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test=\"Hi, nice to meet you where are you from?\"\n",
    "re.sub(r'\\s','-',test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Hi,-nice-to-meet you where are you from?'"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "re.sub(r'\\s','-',test, 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Help on function subn in module re:\n",
      "\n",
      "subn(pattern, repl, string, count=0, flags=0)\n",
      "    Return a 2-tuple containing (new_string, number).\n",
      "    new_string is the string obtained by replacing the leftmost\n",
      "    non-overlapping occurrences of the pattern in the source\n",
      "    string by the replacement repl.  number is the number of\n",
      "    substitutions that were made. repl can be either a string or a\n",
      "    callable; if a string, backslash escapes in it are processed.\n",
      "    If it is a callable, it's passed the match object and must\n",
      "    return a replacement string to be used.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "help(re.subn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('Hi,-nice-to-meet you where are you from?', 3)"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "re.subn(r'\\s','-',test, 3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 分隔字符串"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Hi,', 'nice', 'to', 'meet', 'you', 'where', 'are', 'you', 'from?']"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test=\"Hi, nice to meet you where are you from?\"\n",
    "re.split(r\"\\s+\",test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Hi,', 'nice', 'to', 'meet you where are you from?']"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "re.split(r\"\\s+\",test,3)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}