exploration/clusting.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics.pairwise import pairwise_distances\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from matplotlib import pyplot as plt\n",
    "%matplotlib inline\n",
    "from model.xgboost.xgboost_data_util import generate_training_set\n",
    "from model.seq2seq.seq2seq_data_util import get_training_statistics, generate_dev_set\n",
    "\n",
    "from utils.kmedoids import cluster as kmedoids_cluster\n",
    "\n",
    "from utils.information import bj_station_list, bj_X_aq_list, bj_y_aq_list\n",
    "from utils.information import ld_station_list, ld_X_aq_list, ld_y_aq_list, ld_X_meo_list\n",
    "bj_X_meo_list = [\"temperature\",\"pressure\",\"humidity\",\"direction\",\"speed\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def plot_feature(y, index, norm=True):\n",
    "    feature_of_stations = []\n",
    "    for i in range(index, len(output_features),3):\n",
    "        feature_name = output_features[i]\n",
    "        f = y[i]\n",
    "        if norm :\n",
    "            f = f * statistics.loc['std'][feature_name] + statistics.loc['mean'][feature_name]\n",
    "        feature_of_stations.append(f)\n",
    "    plt.scatter(range(len(feature_of_stations)), feature_of_stations)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def cluster(y_all, index):\n",
    "    feature_dis_list = []\n",
    "    for k in range(y_all.shape[0]):\n",
    "        y = y_all[k,:]\n",
    "        index = 0\n",
    "        feature = y[index::3]\n",
    "        feature_dis = np.zeros((len(feature), len(feature)))\n",
    "        for i in range(len(feature)):\n",
    "            for j in range(len(feature)):\n",
    "                feature_dis[i, j] = np.abs(feature[i] - feature[j])\n",
    "        feature_dis_list.append(feature_dis)\n",
    "\n",
    "    feature_dis_mean = np.zeros(feature_dis_list[0].shape)\n",
    "    for feature_dis in feature_dis_list:\n",
    "        feature_dis_mean += feature_dis\n",
    "    feature_dis_mean = feature_dis_mean / len(feature_dis_list)\n",
    "    return feature_dis_mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "city = \"bj\"\n",
    "\n",
    "station_list = bj_station_list\n",
    "X_aq_list = bj_X_aq_list\n",
    "y_aq_list = bj_y_aq_list\n",
    "X_meo_list = bj_X_meo_list\n",
    "\n",
    "output_features = []\n",
    "for station in station_list : \n",
    "    for aq_feature in y_aq_list :\n",
    "        output_features.append(station + \"_\" + aq_feature)\n",
    "output_features.sort()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "ename": "FileNotFoundError",
     "evalue": "File b'preprocessed_data/before_split/bj_aq_describe.csv' does not exist",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-5-ce0089e506e5>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mstatistics\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_training_statistics\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcity\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/notebooks/KDD_Cup_2018/model/model_data_util.py\u001b[0m in \u001b[0;36mget_training_statistics\u001b[0;34m(city)\u001b[0m\n\u001b[1;32m     53\u001b[0m     \u001b[0mGet\u001b[0m \u001b[0mstatics\u001b[0m \u001b[0mvalues\u001b[0m \u001b[0mof\u001b[0m \u001b[0maq\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mmeo\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     54\u001b[0m     '''\n\u001b[0;32m---> 55\u001b[0;31m     \u001b[0maq_describe\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"preprocessed_data/before_split/%s_aq_describe.csv\"\u001b[0m \u001b[0;34m%\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcity\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     56\u001b[0m     \u001b[0maq_describe\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_index\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Unnamed: 0\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minplace\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     57\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.5/dist-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36mparser_f\u001b[0;34m(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, squeeze, prefix, mangle_dupe_cols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, dayfirst, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, escapechar, comment, encoding, dialect, tupleize_cols, error_bad_lines, warn_bad_lines, skipfooter, skip_footer, doublequote, delim_whitespace, as_recarray, compact_ints, use_unsigned, low_memory, buffer_lines, memory_map, float_precision)\u001b[0m\n\u001b[1;32m    653\u001b[0m                     skip_blank_lines=skip_blank_lines)\n\u001b[1;32m    654\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 655\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0m_read\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    656\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    657\u001b[0m     \u001b[0mparser_f\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__name__\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.5/dist-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m_read\u001b[0;34m(filepath_or_buffer, kwds)\u001b[0m\n\u001b[1;32m    403\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    404\u001b[0m     \u001b[0;31m# Create the parser.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 405\u001b[0;31m     \u001b[0mparser\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mTextFileReader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    406\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    407\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mchunksize\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0miterator\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.5/dist-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, f, engine, **kwds)\u001b[0m\n\u001b[1;32m    762\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'has_index_names'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'has_index_names'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    763\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 764\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_make_engine\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mengine\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    765\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    766\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.5/dist-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m_make_engine\u001b[0;34m(self, engine)\u001b[0m\n\u001b[1;32m    983\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_make_engine\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mengine\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'c'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    984\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mengine\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'c'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 985\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mCParserWrapper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    986\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    987\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mengine\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'python'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.5/dist-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, src, **kwds)\u001b[0m\n\u001b[1;32m   1603\u001b[0m         \u001b[0mkwds\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'allow_leading_cols'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mindex_col\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1604\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1605\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_reader\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mparsers\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTextReader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msrc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1606\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1607\u001b[0m         \u001b[0;31m# XXX\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader.__cinit__ (pandas/_libs/parsers.c:4209)\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader._setup_parser_source (pandas/_libs/parsers.c:8873)\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;31mFileNotFoundError\u001b[0m: File b'preprocessed_data/before_split/bj_aq_describe.csv' does not exist"
     ]
    }
   ],
   "source": [
    "statistics = get_training_statistics(city)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "stations = output_features[0::3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pair = {}\n",
    "for i in range(len(stations)):\n",
    "    pair[i] = stations[i]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0: 'aotizhongxin_aq_O3',\n",
       " 1: 'badaling_aq_O3',\n",
       " 2: 'beibuxinqu_aq_O3',\n",
       " 3: 'daxing_aq_O3',\n",
       " 4: 'dingling_aq_O3',\n",
       " 5: 'donggaocun_aq_O3',\n",
       " 6: 'dongsi_aq_O3',\n",
       " 7: 'dongsihuan_aq_O3',\n",
       " 8: 'fangshan_aq_O3',\n",
       " 9: 'fengtaihuayuan_aq_O3',\n",
       " 10: 'guanyuan_aq_O3',\n",
       " 11: 'gucheng_aq_O3',\n",
       " 12: 'huairou_aq_O3',\n",
       " 13: 'liulihe_aq_O3',\n",
       " 14: 'mentougou_aq_O3',\n",
       " 15: 'miyun_aq_O3',\n",
       " 16: 'miyunshuiku_aq_O3',\n",
       " 17: 'nansanhuan_aq_O3',\n",
       " 18: 'nongzhanguan_aq_O3',\n",
       " 19: 'pingchang_aq_O3',\n",
       " 20: 'pinggu_aq_O3',\n",
       " 21: 'qianmen_aq_O3',\n",
       " 22: 'shunyi_aq_O3',\n",
       " 23: 'tiantan_aq_O3',\n",
       " 24: 'tongzhou_aq_O3',\n",
       " 25: 'wanliu_aq_O3',\n",
       " 26: 'wanshouxigong_aq_O3',\n",
       " 27: 'xizhimenbei_aq_O3',\n",
       " 28: 'yanqin_aq_O3',\n",
       " 29: 'yizhuang_aq_O3',\n",
       " 30: 'yongdingmennei_aq_O3',\n",
       " 31: 'yongledian_aq_O3',\n",
       " 32: 'yufa_aq_O3',\n",
       " 33: 'yungang_aq_O3',\n",
       " 34: 'zhiwuyuan_aq_O3'}"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pair"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "根据均值进行的聚类"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train_mean, y_train_mean = generate_training_set(city=\"bj\", \n",
    "                          station_list=station_list, \n",
    "                          X_aq_list=X_aq_list, \n",
    "                          y_aq_list=y_aq_list, \n",
    "                          X_meo_list=X_meo_list, \n",
    "                          use_day=True, \n",
    "                          pre_days=5, \n",
    "                          gap=1,\n",
    "                          use_day_model=True,\n",
    "                          generate_mean=True,\n",
    "                          generate_range=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_all = y_train_mean.reshape(-1, 105)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "O3_dis = cluster(y_all, 0)\n",
    "PM10_dis = cluster(y_all, 1)\n",
    "PM25_dis = cluster(y_all, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "all_stations, all_target_stations = [],[]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "stations, target_stations = kmedoids_cluster(O3_dis, k=4)\n",
    "all_stations.append(stations)\n",
    "all_target_stations.append(target_stations)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "stations, target_stations = kmedoids_cluster(PM10_dis, k=4)\n",
    "all_stations.append(stations)\n",
    "all_target_stations.append(target_stations)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "stations, target_stations = kmedoids_cluster(PM25_dis, k=4)\n",
    "all_stations.append(stations)\n",
    "all_target_stations.append(target_stations)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "对range进行聚类"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train_range, y_train_range = generate_training_set(city=\"bj\", \n",
    "                                                      station_list=station_list, \n",
    "                                                      X_aq_list=X_aq_list, \n",
    "                                                      y_aq_list=y_aq_list, \n",
    "                                                      X_meo_list=X_meo_list, \n",
    "                                                      use_day=True, \n",
    "                                                      pre_days=5, \n",
    "                                                      gap=1,\n",
    "                                                      use_day_model=True,\n",
    "                                                      generate_mean=False,\n",
    "                                                      generate_range=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_all = y_train_range.reshape(-1, 105)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "O3_dis = cluster(y_all, 0)\n",
    "PM10_dis = cluster(y_all, 1)\n",
    "PM25_dis = cluster(y_all, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "stations, target_stations = kmedoids_cluster(O3_dis, k=4)\n",
    "all_stations.append(stations)\n",
    "all_target_stations.append(target_stations)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "stations, target_stations = kmedoids_cluster(PM10_dis, k=4)\n",
    "all_stations.append(stations)\n",
    "all_target_stations.append(target_stations)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "stations, target_stations = kmedoids_cluster(PM25_dis, k=4)\n",
    "all_stations.append(stations)\n",
    "all_target_stations.append(target_stations)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "代表性站点\n",
    "26,6 'wanshouxigong_aq_O3'\n",
    "12,6 'huairou_aq_O3'\n",
    "7,4  'dongsihuan_aq_O3'\n",
    "22,2 'shunyi_aq_O3'\n",
    "18,1 'nongzhanguan_aq_O3'\n",
    "30,1 'yongdingmennei_aq_O3'\n",
    "3,1  'daxing_aq_O3'\n",
    "17,1 'nansanhuan_aq_O3'\n",
    "6,1  'dongsi_aq_O3'\n",
    "24,1 'tongzhou_aq_O3'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[array([26, 26, 26,  8, 12, 20, 26, 26,  8, 26, 26, 26, 12,  8, 12, 12, 20,\n",
       "        26, 26, 26, 20, 26, 12, 26, 26, 26, 26, 26, 12,  8, 26, 26, 26,  8,\n",
       "         8]),\n",
       " array([26, 26, 26, 26, 12, 12, 26,  7, 24, 26, 26, 26, 12, 26, 12, 12, 12,\n",
       "         7, 24, 26, 12, 26, 24, 26, 24, 26, 26, 26, 12, 24,  7, 24, 26, 12,\n",
       "        24]),\n",
       " array([26,  6, 26,  6, 12, 12,  6,  7,  6, 26, 26, 26, 12, 26, 12, 12, 12,\n",
       "         7,  6, 26, 12, 26, 12, 26,  6, 26, 26, 26, 12,  6,  7,  6, 26, 12,\n",
       "         6]),\n",
       " array([ 0,  1,  0, 21,  1, 21, 21, 21, 11, 21, 21, 11, 21, 11, 11, 21, 21,\n",
       "        21, 21,  0, 21, 21,  0, 21, 21, 21, 21, 21,  1, 21, 21, 11, 21, 11,\n",
       "        11]),\n",
       " array([21, 25, 25, 21, 25, 17, 21, 21,  8, 21, 25, 25, 25,  8, 25, 25, 21,\n",
       "        17, 21, 25, 21, 21, 25, 21, 21, 25, 21, 21, 25, 21, 17,  8,  8, 25,\n",
       "        25]),\n",
       " array([21, 19, 21, 21, 19, 16, 21, 21, 21, 21, 21, 21, 16, 21, 21, 21, 16,\n",
       "        17, 21, 19, 21, 21, 21, 21, 21, 21, 21, 21, 19, 21, 17, 21, 21, 21,\n",
       "        21])]"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_stations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[array([ 8, 20, 12, 26]),\n",
       " array([24, 12,  7, 26]),\n",
       " array([26, 12,  7,  6]),\n",
       " array([21,  1,  0, 11]),\n",
       " array([21,  8, 25, 17]),\n",
       " array([16, 19, 21, 17])]"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_target_stations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from collections import Counter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dic = {}\n",
    "for i in range(len(all_stations[0])):\n",
    "    target_station = []\n",
    "    for station in all_stations:\n",
    "        target_station.append(station[i])\n",
    "    c = Counter(target_station)\n",
    "    dic[i] = c"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0: Counter({0: 1, 21: 2, 26: 3}),\n",
       " 1: Counter({1: 1, 6: 1, 19: 1, 25: 1, 26: 2}),\n",
       " 2: Counter({0: 1, 21: 1, 25: 1, 26: 3}),\n",
       " 3: Counter({6: 1, 8: 1, 21: 3, 26: 1}),\n",
       " 4: Counter({1: 1, 12: 3, 19: 1, 25: 1}),\n",
       " 5: Counter({12: 2, 16: 1, 17: 1, 20: 1, 21: 1}),\n",
       " 6: Counter({6: 1, 21: 3, 26: 2}),\n",
       " 7: Counter({7: 2, 21: 3, 26: 1}),\n",
       " 8: Counter({6: 1, 8: 2, 11: 1, 21: 1, 24: 1}),\n",
       " 9: Counter({21: 3, 26: 3}),\n",
       " 10: Counter({21: 2, 25: 1, 26: 3}),\n",
       " 11: Counter({11: 1, 21: 1, 25: 1, 26: 3}),\n",
       " 12: Counter({12: 3, 16: 1, 21: 1, 25: 1}),\n",
       " 13: Counter({8: 2, 11: 1, 21: 1, 26: 2}),\n",
       " 14: Counter({11: 1, 12: 3, 21: 1, 25: 1}),\n",
       " 15: Counter({12: 3, 21: 2, 25: 1}),\n",
       " 16: Counter({12: 2, 16: 1, 20: 1, 21: 2}),\n",
       " 17: Counter({7: 2, 17: 2, 21: 1, 26: 1}),\n",
       " 18: Counter({6: 1, 21: 3, 24: 1, 26: 1}),\n",
       " 19: Counter({0: 1, 19: 1, 25: 1, 26: 3}),\n",
       " 20: Counter({12: 2, 20: 1, 21: 3}),\n",
       " 21: Counter({21: 3, 26: 3}),\n",
       " 22: Counter({0: 1, 12: 2, 21: 1, 24: 1, 25: 1}),\n",
       " 23: Counter({21: 3, 26: 3}),\n",
       " 24: Counter({6: 1, 21: 3, 24: 1, 26: 1}),\n",
       " 25: Counter({21: 2, 25: 1, 26: 3}),\n",
       " 26: Counter({21: 3, 26: 3}),\n",
       " 27: Counter({21: 3, 26: 3}),\n",
       " 28: Counter({1: 1, 12: 3, 19: 1, 25: 1}),\n",
       " 29: Counter({6: 1, 8: 1, 21: 3, 24: 1}),\n",
       " 30: Counter({7: 2, 17: 2, 21: 1, 26: 1}),\n",
       " 31: Counter({6: 1, 8: 1, 11: 1, 21: 1, 24: 1, 26: 1}),\n",
       " 32: Counter({8: 1, 21: 2, 26: 3}),\n",
       " 33: Counter({8: 1, 11: 1, 12: 2, 21: 1, 25: 1}),\n",
       " 34: Counter({6: 1, 8: 1, 11: 1, 21: 1, 24: 1, 25: 1})}"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0: 'aotizhongxin_aq_O3',\n",
       " 1: 'badaling_aq_O3',\n",
       " 2: 'beibuxinqu_aq_O3',\n",
       " 3: 'daxing_aq_O3',\n",
       " 4: 'dingling_aq_O3',\n",
       " 5: 'donggaocun_aq_O3',\n",
       " 6: 'dongsi_aq_O3',\n",
       " 7: 'dongsihuan_aq_O3',\n",
       " 8: 'fangshan_aq_O3',\n",
       " 9: 'fengtaihuayuan_aq_O3',\n",
       " 10: 'guanyuan_aq_O3',\n",
       " 11: 'gucheng_aq_O3',\n",
       " 12: 'huairou_aq_O3',\n",
       " 13: 'liulihe_aq_O3',\n",
       " 14: 'mentougou_aq_O3',\n",
       " 15: 'miyun_aq_O3',\n",
       " 16: 'miyunshuiku_aq_O3',\n",
       " 17: 'nansanhuan_aq_O3',\n",
       " 18: 'nongzhanguan_aq_O3',\n",
       " 19: 'pingchang_aq_O3',\n",
       " 20: 'pinggu_aq_O3',\n",
       " 21: 'qianmen_aq_O3',\n",
       " 22: 'shunyi_aq_O3',\n",
       " 23: 'tiantan_aq_O3',\n",
       " 24: 'tongzhou_aq_O3',\n",
       " 25: 'wanliu_aq_O3',\n",
       " 26: 'wanshouxigong_aq_O3',\n",
       " 27: 'xizhimenbei_aq_O3',\n",
       " 28: 'yanqin_aq_O3',\n",
       " 29: 'yizhuang_aq_O3',\n",
       " 30: 'yongdingmennei_aq_O3',\n",
       " 31: 'yongledian_aq_O3',\n",
       " 32: 'yufa_aq_O3',\n",
       " 33: 'yungang_aq_O3',\n",
       " 34: 'zhiwuyuan_aq_O3'}"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pair"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>stationName</th>\n",
       "      <th>longitude</th>\n",
       "      <th>latitude</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>dongsi_aq</td>\n",
       "      <td>116.417</td>\n",
       "      <td>39.929</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>tiantan_aq</td>\n",
       "      <td>116.407</td>\n",
       "      <td>39.886</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>guanyuan_aq</td>\n",
       "      <td>116.339</td>\n",
       "      <td>39.929</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>wanshouxigong_aq</td>\n",
       "      <td>116.352</td>\n",
       "      <td>39.878</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>aotizhongxin_aq</td>\n",
       "      <td>116.397</td>\n",
       "      <td>39.982</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        stationName  longitude  latitude\n",
       "0         dongsi_aq    116.417    39.929\n",
       "1        tiantan_aq    116.407    39.886\n",
       "2       guanyuan_aq    116.339    39.929\n",
       "3  wanshouxigong_aq    116.352    39.878\n",
       "4   aotizhongxin_aq    116.397    39.982"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 站点的具体位置\n",
    "station_locations = pd.read_excel(\"./data/Beijing/location/Beijing_AirQuality_Stations_locations.xlsx\")\n",
    "station_locations.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "让所有特征的 mean 和 range 一起投个票"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "聚类的结果表明，城区和交通有相当一致的特征（range and mean）,而郊区和对照点的特征并不集中．"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}