Air Quality Prediction with Weather.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Attaching package: ‘lubridate’\n",
      "\n",
      "The following object is masked from ‘package:base’:\n",
      "\n",
      "    date\n",
      "\n",
      "randomForest 4.6-14\n",
      "Type rfNews() to see new features/changes/bug fixes.\n",
      "Loading required package: xts\n",
      "Loading required package: zoo\n",
      "\n",
      "Attaching package: ‘zoo’\n",
      "\n",
      "The following object is masked from ‘package:imputeTS’:\n",
      "\n",
      "    na.locf\n",
      "\n",
      "The following objects are masked from ‘package:base’:\n",
      "\n",
      "    as.Date, as.Date.numeric\n",
      "\n",
      "Loading required package: forecast\n"
     ]
    }
   ],
   "source": [
    "rm(list = ls())\n",
    "\n",
    "library(imputeTS)\n",
    "library(lubridate)\n",
    "library(tictoc)\n",
    "library(randomForest)\n",
    "require(xts)\n",
    "require(forecast)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "###################################\n",
    "#\n",
    "# Set Flags and Properties here\n",
    "#\n",
    "###################################\n",
    "\n",
    "# To turn on validation\n",
    "isValidation <- 0\n",
    "\n",
    "# History folder name containing historical datasets\n",
    "loadHistoricalFolderName <- \"historicalWithWeather\"\n",
    "\n",
    "# Output Folder name for writing updated historical data with latest data\n",
    "updateHistorical <- 1\n",
    "writeHistoricalFolderName <- \"historicalWithWeather\"\n",
    "\n",
    "# Flag to turn RF training ON/Off\n",
    "trainRF <- 0\n",
    "ntree <- 100\n",
    "mtry <- 4\n",
    "\n",
    "# Flag to turn SVR training ON/Off\n",
    "trainSVR <- 1\n",
    "\n",
    "# Flag to predict RF or SVR.  \"RF\" - RF;  \"SVR\" - SVR\n",
    "toPredict <- \"SVR\"\n",
    "###################################"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting Total time...\n"
     ]
    }
   ],
   "source": [
    "## Using custom callbacks in tic/toc\n",
    "my.msg.tic <- function(tic, msg)\n",
    "{\n",
    "   if (is.null(msg) || is.na(msg) || length(msg) == 0)\n",
    "   {\n",
    "      outmsg <- paste(round(toc - tic, 3), \" seconds elapsed\", sep=\"\")\n",
    "   }\n",
    "   else\n",
    "   {\n",
    "      outmsg <- paste(\"Starting \", msg, \"...\", sep=\"\")\n",
    "   }\n",
    "}\n",
    "\n",
    "my.msg.toc <- function(tic, toc, msg, info)\n",
    "{\n",
    "   if (is.null(msg) || is.na(msg) || length(msg) == 0)\n",
    "   {\n",
    "      outmsg <- paste(round(toc - tic, 3), \" seconds elapsed\", sep=\"\")\n",
    "   }\n",
    "   else\n",
    "   {\n",
    "      outmsg <- paste(info, \": \", msg, \": \",\n",
    "                   round(toc - tic, 3), \" seconds elapsed\", sep=\"\")\n",
    "   }\n",
    "}\n",
    "tic.clearlog()\n",
    "tic(\"Total time\", quiet = FALSE, func.tic = my.msg.tic)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting Load latest data...\n",
      "[1] \"number of latest aq rows of bj: 1575\"\n",
      "[1] \"number of latest aq rows of ld: 1330\"\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<ol class=list-inline>\n",
       "\t<li>'dongsi_aq'</li>\n",
       "\t<li>'tiantan_aq'</li>\n",
       "\t<li>'guanyuan_aq'</li>\n",
       "\t<li>'wanshouxigong_aq'</li>\n",
       "\t<li>'aotizhongxin_aq'</li>\n",
       "\t<li>'nongzhanguan_aq'</li>\n",
       "\t<li>'wanliu_aq'</li>\n",
       "\t<li>'beibuxinqu_aq'</li>\n",
       "\t<li>'zhiwuyuan_aq'</li>\n",
       "\t<li>'fengtaihuayuan_aq'</li>\n",
       "\t<li>'yungang_aq'</li>\n",
       "\t<li>'gucheng_aq'</li>\n",
       "\t<li>'fangshan_aq'</li>\n",
       "\t<li>'daxing_aq'</li>\n",
       "\t<li>'yizhuang_aq'</li>\n",
       "\t<li>'tongzhou_aq'</li>\n",
       "\t<li>'shunyi_aq'</li>\n",
       "\t<li>'pingchang_aq'</li>\n",
       "\t<li>'mentougou_aq'</li>\n",
       "\t<li>'pinggu_aq'</li>\n",
       "\t<li>'huairou_aq'</li>\n",
       "\t<li>'miyun_aq'</li>\n",
       "\t<li>'yanqin_aq'</li>\n",
       "\t<li>'dingling_aq'</li>\n",
       "\t<li>'badaling_aq'</li>\n",
       "\t<li>'miyunshuiku_aq'</li>\n",
       "\t<li>'donggaocun_aq'</li>\n",
       "\t<li>'yongledian_aq'</li>\n",
       "\t<li>'yufa_aq'</li>\n",
       "\t<li>'liulihe_aq'</li>\n",
       "\t<li>'qianmen_aq'</li>\n",
       "\t<li>'yongdingmennei_aq'</li>\n",
       "\t<li>'xizhimenbei_aq'</li>\n",
       "\t<li>'nansanhuan_aq'</li>\n",
       "\t<li>'dongsihuan_aq'</li>\n",
       "</ol>\n"
      ],
      "text/latex": [
       "\\begin{enumerate*}\n",
       "\\item 'dongsi\\_aq'\n",
       "\\item 'tiantan\\_aq'\n",
       "\\item 'guanyuan\\_aq'\n",
       "\\item 'wanshouxigong\\_aq'\n",
       "\\item 'aotizhongxin\\_aq'\n",
       "\\item 'nongzhanguan\\_aq'\n",
       "\\item 'wanliu\\_aq'\n",
       "\\item 'beibuxinqu\\_aq'\n",
       "\\item 'zhiwuyuan\\_aq'\n",
       "\\item 'fengtaihuayuan\\_aq'\n",
       "\\item 'yungang\\_aq'\n",
       "\\item 'gucheng\\_aq'\n",
       "\\item 'fangshan\\_aq'\n",
       "\\item 'daxing\\_aq'\n",
       "\\item 'yizhuang\\_aq'\n",
       "\\item 'tongzhou\\_aq'\n",
       "\\item 'shunyi\\_aq'\n",
       "\\item 'pingchang\\_aq'\n",
       "\\item 'mentougou\\_aq'\n",
       "\\item 'pinggu\\_aq'\n",
       "\\item 'huairou\\_aq'\n",
       "\\item 'miyun\\_aq'\n",
       "\\item 'yanqin\\_aq'\n",
       "\\item 'dingling\\_aq'\n",
       "\\item 'badaling\\_aq'\n",
       "\\item 'miyunshuiku\\_aq'\n",
       "\\item 'donggaocun\\_aq'\n",
       "\\item 'yongledian\\_aq'\n",
       "\\item 'yufa\\_aq'\n",
       "\\item 'liulihe\\_aq'\n",
       "\\item 'qianmen\\_aq'\n",
       "\\item 'yongdingmennei\\_aq'\n",
       "\\item 'xizhimenbei\\_aq'\n",
       "\\item 'nansanhuan\\_aq'\n",
       "\\item 'dongsihuan\\_aq'\n",
       "\\end{enumerate*}\n"
      ],
      "text/markdown": [
       "1. 'dongsi_aq'\n",
       "2. 'tiantan_aq'\n",
       "3. 'guanyuan_aq'\n",
       "4. 'wanshouxigong_aq'\n",
       "5. 'aotizhongxin_aq'\n",
       "6. 'nongzhanguan_aq'\n",
       "7. 'wanliu_aq'\n",
       "8. 'beibuxinqu_aq'\n",
       "9. 'zhiwuyuan_aq'\n",
       "10. 'fengtaihuayuan_aq'\n",
       "11. 'yungang_aq'\n",
       "12. 'gucheng_aq'\n",
       "13. 'fangshan_aq'\n",
       "14. 'daxing_aq'\n",
       "15. 'yizhuang_aq'\n",
       "16. 'tongzhou_aq'\n",
       "17. 'shunyi_aq'\n",
       "18. 'pingchang_aq'\n",
       "19. 'mentougou_aq'\n",
       "20. 'pinggu_aq'\n",
       "21. 'huairou_aq'\n",
       "22. 'miyun_aq'\n",
       "23. 'yanqin_aq'\n",
       "24. 'dingling_aq'\n",
       "25. 'badaling_aq'\n",
       "26. 'miyunshuiku_aq'\n",
       "27. 'donggaocun_aq'\n",
       "28. 'yongledian_aq'\n",
       "29. 'yufa_aq'\n",
       "30. 'liulihe_aq'\n",
       "31. 'qianmen_aq'\n",
       "32. 'yongdingmennei_aq'\n",
       "33. 'xizhimenbei_aq'\n",
       "34. 'nansanhuan_aq'\n",
       "35. 'dongsihuan_aq'\n",
       "\n",
       "\n"
      ],
      "text/plain": [
       " [1] \"dongsi_aq\"         \"tiantan_aq\"        \"guanyuan_aq\"      \n",
       " [4] \"wanshouxigong_aq\"  \"aotizhongxin_aq\"   \"nongzhanguan_aq\"  \n",
       " [7] \"wanliu_aq\"         \"beibuxinqu_aq\"     \"zhiwuyuan_aq\"     \n",
       "[10] \"fengtaihuayuan_aq\" \"yungang_aq\"        \"gucheng_aq\"       \n",
       "[13] \"fangshan_aq\"       \"daxing_aq\"         \"yizhuang_aq\"      \n",
       "[16] \"tongzhou_aq\"       \"shunyi_aq\"         \"pingchang_aq\"     \n",
       "[19] \"mentougou_aq\"      \"pinggu_aq\"         \"huairou_aq\"       \n",
       "[22] \"miyun_aq\"          \"yanqin_aq\"         \"dingling_aq\"      \n",
       "[25] \"badaling_aq\"       \"miyunshuiku_aq\"    \"donggaocun_aq\"    \n",
       "[28] \"yongledian_aq\"     \"yufa_aq\"           \"liulihe_aq\"       \n",
       "[31] \"qianmen_aq\"        \"yongdingmennei_aq\" \"xizhimenbei_aq\"   \n",
       "[34] \"nansanhuan_aq\"     \"dongsihuan_aq\"    "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "45"
      ],
      "text/latex": [
       "45"
      ],
      "text/markdown": [
       "45"
      ],
      "text/plain": [
       "[1] 45"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO: Load latest data: 0.131 seconds elapsed\n"
     ]
    }
   ],
   "source": [
    "tic(\"Load latest data\", quiet = FALSE, func.tic = my.msg.tic)\n",
    "# Load the data from the csv files\n",
    "read.data = function(file = ''){\n",
    "  ## Read the csv file\n",
    "  air.quality <- read.csv(file, header = TRUE, \n",
    "                      stringsAsFactors = FALSE)\n",
    "}\n",
    "\n",
    "bj.aq.latest = read.data('bj_airquality_latest.csv')\n",
    "ld.aq.latest = read.data('ld_airquality_latest.csv')\n",
    "# Create a list of the beijing station ID names\n",
    "bj_station_names_vec<-head(bj.aq.latest,35)$station_id\n",
    "# Manually input the station names that we need for London\n",
    "ld_station_names_vec <- c('CD1','BL0','GR4','MY7','HV1','GN3','GR9','LW2','GN0','KF1','CD9','ST5','TH4')\n",
    "\n",
    "correct_col_names <- c('id', 'stationId', 'utc_time', 'PM2.5', 'PM10', 'NO2','CO', 'O3', 'SO2')\n",
    "\n",
    "#rename bj and ld to latest column names\n",
    "names(bj.aq.latest) <- correct_col_names\n",
    "names(ld.aq.latest) <- correct_col_names\n",
    "\n",
    "# Drop the id column of the latest bj aq file\n",
    "bj.aq.latest <- bj.aq.latest[ , !(names(bj.aq.latest) %in% c(\"id\"))]\n",
    "# Drop the id column of the latest ld aq file\n",
    "ld.aq.latest <- ld.aq.latest[, !(names(ld.aq.latest) %in% c(\"id\", \"CO\", \"O3\", \"SO2\"))]\n",
    "\n",
    "print(paste(\"number of latest aq rows of bj:\",nrow(bj.aq.latest)))\n",
    "print(paste(\"number of latest aq rows of ld:\",nrow(ld.aq.latest)))\n",
    "\n",
    "bj_station_names_vec\n",
    "nrow(bj.aq.latest)/35\n",
    "\n",
    "toc(log = TRUE, quiet = FALSE, func.toc = my.msg.toc, info = \"INFO\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting Load Historical data...\n",
      "INFO: Load Historical data: 1.03 seconds elapsed\n"
     ]
    }
   ],
   "source": [
    "tic(\"Load Historical data\", quiet = FALSE, func.tic = my.msg.tic)\n",
    "bj.fullStationData <- list()\n",
    "ld.fullStationData <- list()\n",
    "for (s in 1:35) {\n",
    "    bj.fullStationData[[s]] <- read.data(paste(loadHistoricalFolderName,\"/\",bj_station_names_vec[s],\"_historical.csv\",sep=\"\"))\n",
    "}\n",
    "\n",
    "for (s in 1:13) {\n",
    "    ld.fullStationData[[s]] <- read.data(paste(loadHistoricalFolderName,\"/\",ld_station_names_vec[s],\"_historical.csv\",sep=\"\"))\n",
    "}\n",
    "toc(log = TRUE, quiet = FALSE, func.toc = my.msg.toc, info = \"INFO\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[1] \"2018-05-29 01:00:00 UTC\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "[1] \"2018-05-31 22:00:00 UTC\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "1575"
      ],
      "text/latex": [
       "1575"
      ],
      "text/markdown": [
       "1575"
      ],
      "text/plain": [
       "[1] 1575"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "[1] \"2018-05-29 UTC\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "[1] \"2018-05-31 21:00:00 UTC\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "1328"
      ],
      "text/latex": [
       "1328"
      ],
      "text/markdown": [
       "1328"
      ],
      "text/plain": [
       "[1] 1328"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Convert the bj date time from string to POSIX time in UTC in order to add missing rows\n",
    "bj.aq.latest$utc_time <- as.POSIXct(bj.aq.latest$utc_time, format=\"%Y-%m-%d %H:%M:%S\", tz=\"UTC\")\n",
    "# get starting and ending time\n",
    "bj.startingTime <- head(bj.aq.latest$utc_time,1)\n",
    "bj.startingTime\n",
    "bj.endingTime <- tail(bj.aq.latest$utc_time,1)\n",
    "bj.endingTime\n",
    "\n",
    "#remove possible duplicate rows\n",
    "bj.aq.latest <- unique(bj.aq.latest)\n",
    "nrow(bj.aq.latest)\n",
    "\n",
    "# Repeat for London\n",
    "ld.aq.latest$utc_time <- as.POSIXct(ld.aq.latest$utc_time, format=\"%Y-%m-%d %H:%M:%S\", tz=\"UTC\")\n",
    "# get starting and ending time\n",
    "ld.startingTime <- head(ld.aq.latest$utc_time,1)\n",
    "ld.startingTime\n",
    "ld.endingTime <- tail(ld.aq.latest$utc_time,1)\n",
    "ld.endingTime\n",
    "\n",
    "#remove duplicate rows\n",
    "ld.aq.latest <- unique(ld.aq.latest)\n",
    "nrow(ld.aq.latest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "70"
      ],
      "text/latex": [
       "70"
      ],
      "text/markdown": [
       "70"
      ],
      "text/plain": [
       "[1] 70"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "70"
      ],
      "text/latex": [
       "70"
      ],
      "text/markdown": [
       "70"
      ],
      "text/plain": [
       "[1] 70"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Find the difference between the latest bj first and last record\n",
    "bj.diff <- as.integer(bj.endingTime - bj.startingTime + 1)  * 24 - hour(bj.startingTime) - (23 - hour(bj.endingTime))\n",
    "bj.diff\n",
    "\n",
    "# Repeat for London\n",
    "ld.diff <- as.integer(ld.endingTime - ld.startingTime + 1)  * 24 - hour(ld.startingTime) - (23 - hour(ld.endingTime))\n",
    "ld.diff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "35"
      ],
      "text/latex": [
       "35"
      ],
      "text/markdown": [
       "35"
      ],
      "text/plain": [
       "[1] 35"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "13"
      ],
      "text/latex": [
       "13"
      ],
      "text/markdown": [
       "13"
      ],
      "text/plain": [
       "[1] 13"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Break the latest data into a list of dataframes for each station and sort by time\n",
    "bj.stationData<-list()\n",
    "for (i in 1:35) {\n",
    "    bj.stationData[[i]]<-bj.aq.latest[bj.aq.latest$stationId==bj_station_names_vec[i],]\n",
    "    bj.stationData[[i]] <- bj.stationData[[i]][order(bj.stationData[[i]]$utc_time),]\n",
    "}\n",
    "\n",
    "ld.stationData<-list()\n",
    "for (i in 1:13) {\n",
    "    ld.stationData[[i]]<-ld.aq.latest[ld.aq.latest$stationId==ld_station_names_vec[i],]\n",
    "    ld.stationData[[i]] <- ld.stationData[[i]][order(ld.stationData[[i]]$utc_time),]\n",
    "}\n",
    "\n",
    "length(bj.stationData)\n",
    "length(ld.stationData)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a function that adds missing rows if time isn't continuous\n",
    "addMissingRows <- function(stationData, timeDiff, colNames, startingTime) {\n",
    "    \n",
    "fullStationData <- list()\n",
    "for (s in 1:length(stationData)) {\n",
    "    #first create a dataframe with 'diff' number of rows\n",
    "    fullStationData[[s]] <- as.data.frame(matrix(NA, nrow = timeDiff, ncol = length(colNames)))\n",
    "    names(fullStationData[[s]]) <- colNames\n",
    "    \n",
    "    len_station_data <- nrow(stationData[[s]])\n",
    "    new_index <- 1  #current index at fullStationData\n",
    "    old_index <- 1  #current index at stationData\n",
    "    fill_time <- startingTime\n",
    "    while (new_index <= timeDiff && old_index <= len_station_data) {\n",
    "       if (fill_time == stationData[[s]]$utc_time[old_index]) {\n",
    "           # if same time, copy row from stationData to fullStationData\n",
    "           fullStationData[[s]][new_index,] <- stationData[[s]][old_index,]\n",
    "           # and increment both index\n",
    "           old_index <- old_index + 1   \n",
    "           \n",
    "       } else {\n",
    "           # if not the same time, then only increment new_index, and set stationId and utc_time\n",
    "           fullStationData[[s]]$stationId[new_index] <- stationData[[s]]$stationId[1]\n",
    "           fullStationData[[s]]$utc_time[new_index] <- fill_time\n",
    "       }\n",
    "        \n",
    "        new_index <- new_index + 1\n",
    "        fill_time <- fill_time + 3600 # increment time by 1 hour  \n",
    "    }\n",
    "    \n",
    "    # fill fullStationData with the rest if there's any\n",
    "    while (new_index <= timeDiff) {\n",
    "        fullStationData[[s]]$stationId[new_index] <- stationData[[s]]$stationId[1]\n",
    "        fullStationData[[s]]$utc_time[new_index] <- fill_time\n",
    "        new_index <- new_index + 1\n",
    "        fill_time <- fill_time + 3600\n",
    "    }   \n",
    "}\n",
    "    \n",
    "return(fullStationData)\n",
    "\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting Fill missing rows for latest current data...\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "70"
      ],
      "text/latex": [
       "70"
      ],
      "text/markdown": [
       "70"
      ],
      "text/plain": [
       "[1] 70"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "70"
      ],
      "text/latex": [
       "70"
      ],
      "text/markdown": [
       "70"
      ],
      "text/plain": [
       "[1] 70"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO: Fill missing rows for latest current data: 0.928 seconds elapsed\n"
     ]
    }
   ],
   "source": [
    "tic(\"Fill missing rows for latest current data\", quiet = FALSE, func.tic = my.msg.tic)\n",
    "# fill missing rows for latest beijing and london aq data\n",
    "bj.aq.latest.filled <- addMissingRows(bj.stationData, bj.diff, names(bj.aq.latest), bj.startingTime)\n",
    "ld.aq.latest.filled <- addMissingRows(ld.stationData, ld.diff, names(ld.aq.latest), ld.startingTime)\n",
    "\n",
    "# Verify that the number of rows is equal to the time diff now\n",
    "nrow(bj.aq.latest.filled[[1]])\n",
    "nrow(ld.aq.latest.filled[[1]])\n",
    "\n",
    "rm(bj.aq.latest)\n",
    "rm(ld.aq.latest)\n",
    "rm(bj.stationData)\n",
    "rm(ld.stationData)\n",
    "toc(log = TRUE, quiet = FALSE, func.toc = my.msg.toc, info = \"INFO\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting Fill in missing data with Kalman...\n",
      "INFO: Fill in missing data with Kalman: 0.445 seconds elapsed\n"
     ]
    }
   ],
   "source": [
    "#fill in empty/missing data for each pollutant for each station using Kalman filters\n",
    "tic(\"Fill in missing data with Kalman\", quiet = FALSE, func.tic = my.msg.tic)\n",
    "colNames <- c('PM2.5', 'PM10','O3')\n",
    "\n",
    "for (s in 1:35) {\n",
    "    for (col in colNames) {\n",
    "        #sometimes a station may not have any data at all, skip it\n",
    "        if (length(which(is.na(bj.aq.latest.filled[[s]][col]))) > nrow(bj.aq.latest.filled[[s]]) - 3) {\n",
    "            bj.aq.latest.filled[[s]][col]<-lapply(bj.aq.latest.filled[[s]][col],function(x) x=0)\n",
    "           next \n",
    "        }\n",
    "        bj.aq.latest.filled[[s]][col] <- na.kalman(bj.aq.latest.filled[[s]][col])\n",
    "    }\n",
    "    bj.aq.latest.filled[[s]]$utc_time <- as.POSIXct(bj.aq.latest.filled[[s]]$utc_time, origin=\"1970-01-01\", tz=\"UTC\")\n",
    "}\n",
    "\n",
    "#London\n",
    "colNames <- c('PM2.5', 'PM10')\n",
    "\n",
    "for (s in 1:13) {\n",
    "    for (col in colNames) {\n",
    "        #sometimes a station may not have any data at all, skip it\n",
    "        if (length(which(is.na(ld.aq.latest.filled[[s]][col]))) > nrow(ld.aq.latest.filled[[s]]) - 3) {\n",
    "            ld.aq.latest.filled[[s]][col]<-lapply(ld.aq.latest.filled[[s]][col],function(x) x=0)\n",
    "           next \n",
    "        }\n",
    "        ld.aq.latest.filled[[s]][col] <- na.kalman(ld.aq.latest.filled[[s]][col])\n",
    "    }\n",
    "    ld.aq.latest.filled[[s]]$utc_time <- as.POSIXct(ld.aq.latest.filled[[s]]$utc_time, origin=\"1970-01-01\", tz=\"UTC\")\n",
    "}\n",
    "toc(log = TRUE, quiet = FALSE, func.toc = my.msg.toc, info = \"INFO\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Add time features to latest data\n",
    "# Function to find the season of the year given the date\n",
    "find_season <- function(date) {\n",
    "    spring <- paste(year(date),\"-03-20\", sep=\"\")\n",
    "    spring <- as.POSIXct(spring, origin=\"1970-01-01\", tz=\"UTC\")\n",
    "    summer <- paste(year(date),\"-06-21\", sep=\"\")\n",
    "    summer <- as.POSIXct(summer, origin=\"1970-01-01\", tz=\"UTC\")\n",
    "    fall <- paste(year(date),\"-09-22\", sep=\"\")\n",
    "    fall <- as.POSIXct(fall, origin=\"1970-01-01\", tz=\"UTC\")\n",
    "    winter <- paste(year(date),\"-12-21\", sep=\"\")\n",
    "    winter <- as.POSIXct(winter, origin=\"1970-01-01\", tz=\"UTC\")\n",
    "    if (date >= spring && date < summer )\n",
    "        return(1) # Spring\n",
    "    else if (date >= summer && date < fall)\n",
    "        return(2) # Summer\n",
    "    else if (date >= fall && date < winter)\n",
    "        return(3) # Fall\n",
    "    else\n",
    "        return(4) # Winter    \n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Functions to find last week's values for beijing and london\n",
    "# from historical\n",
    "bj_find_last_week_val <- function(station_index, datetime, pollutant) {\n",
    "    lastweektime = datetime - 604800\n",
    "    historicalLast10Day <- tail(bj.fullStationData[[station_index]],240)\n",
    "    lastVal <- historicalLast10Day[[pollutant]][historicalLast10Day$utc_time==lastweektime][1]\n",
    "    if (is.na(lastVal)) {\n",
    "        lastVal <- bj.aq.latest.filled[[station_index]][[pollutant]][bj.aq.latest.filled[[station_index]]$utc_time==lastweektime][1]\n",
    "    }\n",
    "    return (ifelse(is.na(lastVal),0,lastVal))\n",
    "}\n",
    "\n",
    "ld_find_last_week_val <- function(station_index, datetime, pollutant) {\n",
    "    lastweektime = datetime - 604800\n",
    "    historicalLast10Day <- tail(ld.fullStationData[[station_index]],240)\n",
    "    lastVal <- historicalLast10Day[[pollutant]][historicalLast10Day$utc_time==lastweektime][1]\n",
    "    if (is.na(lastVal)) {\n",
    "        lastVal <- ld.aq.latest.filled[[station_index]][[pollutant]][ld.aq.latest.filled[[station_index]]$utc_time==lastweektime][1]\n",
    "    }\n",
    "    return (ifelse(is.na(lastVal),0,lastVal))\n",
    "}\n",
    "\n",
    "# Functions to find two days ago's values for beijing and london\n",
    "bj_find_two_day_val <- function(station_index, datetime, pollutant, aq_latest) {\n",
    "    twodaytime = datetime - 172800\n",
    "    historicalLast5Day <- tail(bj.fullStationData[[station_index]],120)\n",
    "    lastVal <- historicalLast5Day[[pollutant]][historicalLast5Day$utc_time==twodaytime][1]\n",
    "    if (is.na(lastVal)) {\n",
    "        lastVal <- bj.aq.latest.filled[[station_index]][[pollutant]][bj.aq.latest.filled[[station_index]]$utc_time==twodaytime][1]\n",
    "    }\n",
    "    return (ifelse(is.na(lastVal),0,lastVal))\n",
    "}\n",
    "\n",
    "ld_find_two_day_val <- function(station_index, datetime, pollutant) {\n",
    "    twodaytime = datetime - 172800\n",
    "    historicalLast5Day <- tail(ld.fullStationData[[station_index]],120)\n",
    "    lastVal <- historicalLast5Day[[pollutant]][historicalLast5Day$utc_time==twodaytime][1]\n",
    "    if (is.na(lastVal)) {\n",
    "        lastVal <- ld.aq.latest.filled[[station_index]][[pollutant]][ld.aq.latest.filled[[station_index]]$utc_time==twodaytime][1]\n",
    "    }\n",
    "    return (ifelse(is.na(lastVal),0,lastVal))\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting Add time features to latest Beijing data...\n",
      "INFO: Add time features to latest Beijing data: 81.758 seconds elapsed\n"
     ]
    }
   ],
   "source": [
    "#add time features to latest beijing data\n",
    "# We need the time only once, for all bj stations\n",
    "tic(\"Add time features to latest Beijing data\", quiet = FALSE, func.tic = my.msg.tic)\n",
    "totalRows <- nrow(bj.aq.latest.filled[[1]])\n",
    "bj.timeVector <- as.data.frame(matrix(nrow = totalRows, ncol = 5)) \n",
    "names(bj.timeVector) <- c(\"hour\", \"month\", \"weekday\", \"year\", \"season\")\n",
    "for (r in 1:totalRows) {\n",
    "    t <- bj.aq.latest.filled[[1]]$utc_time[r]\n",
    "    bj.timeVector$hour[r] <- hour(t)\n",
    "    bj.timeVector$month[r] <- month(t)\n",
    "    bj.timeVector$weekday[r] <- wday(t)\n",
    "    bj.timeVector$year[r] <- year(t)\n",
    "    bj.timeVector$season[r] <- find_season(t)\n",
    "}\n",
    "\n",
    "\n",
    "for (s in 1:35) {\n",
    "    for (r in 1:nrow(bj.aq.latest.filled[[s]])) {\n",
    "        t <- bj.aq.latest.filled[[s]]$utc_time[r]\n",
    "        bj.aq.latest.filled[[s]]$hour[r] <- bj.timeVector$hour[r]\n",
    "        bj.aq.latest.filled[[s]]$month[r] <- bj.timeVector$month[r]\n",
    "        bj.aq.latest.filled[[s]]$weekday[r] <- bj.timeVector$weekday[r]\n",
    "        bj.aq.latest.filled[[s]]$year[r] <- bj.timeVector$year[r]\n",
    "        bj.aq.latest.filled[[s]]$season[r] <- bj.timeVector$season[r]\n",
    "        bj.aq.latest.filled[[s]]$lwPM2.5[r] <- bj_find_last_week_val(s,t,\"PM2.5\")\n",
    "        bj.aq.latest.filled[[s]]$lwPM10[r] <- bj_find_last_week_val(s,t,\"PM10\")\n",
    "        bj.aq.latest.filled[[s]]$lwO3[r] <- bj_find_last_week_val(s,t,\"O3\")\n",
    "        bj.aq.latest.filled[[s]]$tdPM2.5[r] <- bj_find_two_day_val(s,t,\"PM2.5\")\n",
    "        bj.aq.latest.filled[[s]]$tdPM10[r] <- bj_find_two_day_val(s,t,\"PM10\")\n",
    "        bj.aq.latest.filled[[s]]$tdO3[r] <- bj_find_two_day_val(s,t,\"O3\")   \n",
    "    }\n",
    "}\n",
    "rm(bj.timeVector)\n",
    "toc(log = TRUE, quiet = FALSE, func.toc = my.msg.toc, info = \"INFO\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting Add time features to latest London data...\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th></th><th scope=col>used</th><th scope=col>(Mb)</th><th scope=col>gc trigger</th><th scope=col>(Mb)</th><th scope=col>limit (Mb)</th><th scope=col>max used</th><th scope=col>(Mb)</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><th scope=row>Ncells</th><td> 934811</td><td>50.0   </td><td>1869915</td><td>99.9   </td><td>   NA  </td><td>1869915</td><td>99.9   </td></tr>\n",
       "\t<tr><th scope=row>Vcells</th><td>3147942</td><td>24.1   </td><td>8388608</td><td>64.0   </td><td>16384  </td><td>8387797</td><td>64.0   </td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|lllllll}\n",
       "  & used & (Mb) & gc trigger & (Mb) & limit (Mb) & max used & (Mb)\\\\\n",
       "\\hline\n",
       "\tNcells &  934811 & 50.0    & 1869915 & 99.9    &    NA   & 1869915 & 99.9   \\\\\n",
       "\tVcells & 3147942 & 24.1    & 8388608 & 64.0    & 16384   & 8387797 & 64.0   \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/markdown": [
       "\n",
       "| <!--/--> | used | (Mb) | gc trigger | (Mb) | limit (Mb) | max used | (Mb) | \n",
       "|---|---|\n",
       "| Ncells |  934811 | 50.0    | 1869915 | 99.9    |    NA   | 1869915 | 99.9    | \n",
       "| Vcells | 3147942 | 24.1    | 8388608 | 64.0    | 16384   | 8387797 | 64.0    | \n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "       used    (Mb) gc trigger (Mb) limit (Mb) max used (Mb)\n",
       "Ncells  934811 50.0 1869915    99.9    NA      1869915  99.9\n",
       "Vcells 3147942 24.1 8388608    64.0 16384      8387797  64.0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO: Add time features to latest London data: 20.056 seconds elapsed\n"
     ]
    }
   ],
   "source": [
    "#add time features to london data\n",
    "# We need the time only once, for all ld stations\n",
    "tic(\"Add time features to latest London data\", quiet = FALSE, func.tic = my.msg.tic)\n",
    "totalRows <- nrow(ld.aq.latest.filled[[1]])\n",
    "ld.timeVector <- as.data.frame(matrix(nrow = totalRows, ncol = 5)) \n",
    "names(ld.timeVector) <- c(\"hour\", \"month\", \"weekday\", \"year\", \"season\")\n",
    "for (r in 1:totalRows) {\n",
    "    t <- ld.aq.latest.filled[[1]]$utc_time[r]\n",
    "    ld.timeVector$hour[r] <- hour(t)\n",
    "    ld.timeVector$month[r] <- month(t)\n",
    "    ld.timeVector$weekday[r] <- wday(t)\n",
    "    ld.timeVector$year[r] <- year(t)\n",
    "    ld.timeVector$season[r] <- find_season(t)\n",
    "}\n",
    "\n",
    "for (s in 1:13) {\n",
    "    for (r in 1:nrow(ld.aq.latest.filled[[s]])) {\n",
    "        t <- ld.aq.latest.filled[[s]]$utc_time[r]\n",
    "        ld.aq.latest.filled[[s]]$hour[r] <- ld.timeVector$hour[r]\n",
    "        ld.aq.latest.filled[[s]]$month[r] <- ld.timeVector$month[r]\n",
    "        ld.aq.latest.filled[[s]]$weekday[r] <- ld.timeVector$weekday[r]\n",
    "        ld.aq.latest.filled[[s]]$year[r] <- ld.timeVector$year[r]\n",
    "        ld.aq.latest.filled[[s]]$season[r] <- ld.timeVector$season[r]\n",
    "        ld.aq.latest.filled[[s]]$lwPM2.5[r] <- ld_find_last_week_val(s,t,\"PM2.5\")\n",
    "        ld.aq.latest.filled[[s]]$lwPM10[r] <- ld_find_last_week_val(s,t,\"PM10\")\n",
    "        ld.aq.latest.filled[[s]]$tdPM2.5[r] <- ld_find_two_day_val(s,t,\"PM2.5\")\n",
    "        ld.aq.latest.filled[[s]]$tdPM10[r] <- ld_find_two_day_val(s,t,\"PM10\")\n",
    "    }\n",
    "}\n",
    "rm(ld.timeVector)\n",
    "gc()\n",
    "toc(log = TRUE, quiet = FALSE, func.toc = my.msg.toc, info = \"INFO\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Add meo features to latest data\n",
    "# Load file to df\n",
    "bj.meo.latest = read.data('bj_grid_meteorology_latest.csv')\n",
    "bj_aq_grid = read.data('beijing_aq_grid_relation.csv')\n",
    "ld.meo.latest = read.data('ld_grid_meteorology_latest.csv')\n",
    "ld_aq_grid = read.data('london_aq_grid_relation.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Convert time to POSIX\n",
    "bj.meo.latest$time <- as.POSIXct(bj.meo.latest$time, format=\"%Y-%m-%d %H:%M:%S\", tz=\"UTC\")\n",
    "ld.meo.latest$time <- as.POSIXct(ld.meo.latest$time, format=\"%Y-%m-%d %H:%M:%S\", tz=\"UTC\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Functions that get weather features\n",
    "getTemp <- function(meo.latest, gridName, time) {\n",
    "    temp <- meo.latest$temperature[meo.latest$station_id==gridName & meo.latest$time==time][1]\n",
    "    if (is.na(temp)) {\n",
    "        temp <- 20\n",
    "    }\n",
    "    return (temp)\n",
    "}\n",
    "\n",
    "getHumidity <- function(meo.latest, gridName, time) {\n",
    "    humidity <- meo.latest$humidity[meo.latest$station_id==gridName & meo.latest$time==time][1]\n",
    "    if (is.na(humidity)) {\n",
    "        humidity <- 22\n",
    "    }\n",
    "    return (humidity)\n",
    "}\n",
    "\n",
    "getPressure <- function(meo.latest, gridName, time) {\n",
    "    pressure <- meo.latest$pressure[meo.latest$station_id==gridName & meo.latest$time==time][1]\n",
    "    if (is.na(pressure)) {\n",
    "        pressure <- 940\n",
    "    }\n",
    "    return (pressure)\n",
    "}\n",
    "\n",
    "getWindSpeed <- function(meo.latest, gridName, time) {\n",
    "    windSpeed <- meo.latest$wind_speed[meo.latest$station_id==gridName & meo.latest$time==time][1]\n",
    "    if (is.na(windSpeed)) {\n",
    "        windSpeed <- 18\n",
    "    }\n",
    "    return (windSpeed)\n",
    "}\n",
    "\n",
    "getWindDir <- function(meo.latest, gridName, time) {\n",
    "    windDir <- meo.latest$wind_direction[meo.latest$station_id==gridName & meo.latest$time==time][1]\n",
    "    if (is.na(windDir)) {\n",
    "        return (\"NO\")\n",
    "    }\n",
    "    windDir <- as.numeric(windDir)\n",
    "    if (windDir > 337 & windDir<361 | windDir >=0 & windDir <= 22) {\n",
    "        windDir <- \"N\"\n",
    "    } else if (windDir > 22 & windDir <= 67) {\n",
    "        windDir <- \"NE\"\n",
    "    } else if (windDir > 67 & windDir <= 112) {\n",
    "        windDir <- \"E\"\n",
    "    }  else if (windDir > 112 & windDir <= 157) {\n",
    "        windDir <- \"SE\"\n",
    "    }  else if (windDir > 157 & windDir <= 202) {\n",
    "        windDir <- \"S\"\n",
    "    }  else if (windDir > 202 & windDir <= 247) {\n",
    "        windDir <- \"SW\"\n",
    "    }  else if (windDir > 247 & windDir <= 292) {\n",
    "        windDir <- \"W\"\n",
    "    }  else if (windDir > 292 & windDir <= 337) {\n",
    "        windDir <- \"NW\"\n",
    "    }  else {\n",
    "        windDir <- \"NO\"\n",
    "    }  \n",
    "    return (windDir)\n",
    "}\n",
    "\n",
    "getWeatherDescription <- function(description) {\n",
    "    if (grepl(\"11\", tolower(description)) | grepl(\"storm\", tolower(description))) {\n",
    "        return (\"storm\")\n",
    "    }\n",
    "    else if (grepl(\"09\", tolower(description)) | grepl(\"10\", tolower(description)) | grepl(\"rain\", tolower(description))) {\n",
    "        return (\"rainy\")\n",
    "    }\n",
    "    else if (grepl(\"13\", tolower(description)) | grepl(\"snow\", tolower(description))) {\n",
    "        return (\"snowy\")\n",
    "    }\n",
    "    else if (grepl(\"50\", tolower(description)) | grepl(\"fog\", tolower(description)) | grepl(\"haze\", tolower(description))) {\n",
    "        return (\"haze\")\n",
    "    }\n",
    "    else if (grepl(\"01\", tolower(description)) | grepl(\"clear\", tolower(description)) | grepl(\"sun\", tolower(description))) {\n",
    "        return (\"clear\")\n",
    "    }\n",
    "    else if (grepl(\"02\", tolower(description)) | grepl(\"03\", tolower(description)) | \n",
    "             grepl(\"04\", tolower(description)) | grepl(\"cloud\", tolower(description))) {\n",
    "        return (\"cloudy\")\n",
    "    } else {\n",
    "        return (\"cloudy\")\n",
    "    }\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting Add meo to latest bj data...\n",
      "INFO: Add meo to latest bj data: 42.818 seconds elapsed\n"
     ]
    }
   ],
   "source": [
    "tic(\"Add meo to latest bj data\", quiet = FALSE, func.tic = my.msg.tic)\n",
    "# Add the meo features to the bj latest data\n",
    "for (s in 1:18) {\n",
    "    # First we have to find the 4 closest grid meo stations to the aq station\n",
    "    aqName <- bj_station_names_vec[s]\n",
    "    ULGridName <- bj_aq_grid$ULGridName[bj_aq_grid$station_id==aqName]\n",
    "    URGridName <- bj_aq_grid$URGridName[bj_aq_grid$station_id==aqName]\n",
    "    LLGridName <- bj_aq_grid$LLGridName[bj_aq_grid$station_id==aqName]\n",
    "    LRGridName <- bj_aq_grid$LRGridName[bj_aq_grid$station_id==aqName]\n",
    "    bj.aq.latest.filled[[s]]$utc_time <- as.POSIXct(bj.aq.latest.filled[[s]]$utc_time, origin=\"1970-01-01\", tz=\"UTC\")\n",
    "    \n",
    "    tempVec <- NULL\n",
    "    humidityVec <- NULL\n",
    "    pressureVec <- NULL\n",
    "    windSpeedVec <- NULL\n",
    "    windDirVec <- NULL\n",
    "    weatherVec <- NULL\n",
    "    \n",
    "    for (i in 1:nrow(bj.aq.latest.filled[[s]])) {\n",
    "        time <- bj.aq.latest.filled[[s]]$utc_time[i]\n",
    "        tempVec[i] <- mean(c(getTemp(bj.meo.latest, ULGridName, time), getTemp(bj.meo.latest, URGridName, time), getTemp(bj.meo.latest, LLGridName, time), getTemp(bj.meo.latest, LRGridName, time)))\n",
    "        pressureVec[i] <- mean(c(getPressure(bj.meo.latest, ULGridName, time), getPressure(bj.meo.latest, URGridName, time), getPressure(bj.meo.latest, LLGridName, time), getPressure(bj.meo.latest, LRGridName, time)))\n",
    "        humidityVec[i] <- mean(c(getHumidity(bj.meo.latest, ULGridName, time), getHumidity(bj.meo.latest, URGridName, time), getHumidity(bj.meo.latest, LLGridName, time), getHumidity(bj.meo.latest, LRGridName, time)))        \n",
    "        windSpeedVec[i] <- mean(c(getWindSpeed(bj.meo.latest, ULGridName, time), getWindSpeed(bj.meo.latest, URGridName, time), getWindSpeed(bj.meo.latest, LLGridName, time), getWindSpeed(bj.meo.latest, LRGridName, time)))\n",
    "        windDirVec[i] <- getWindDir(bj.meo.latest, ULGridName, time)\n",
    "        weatherVec[i] <- getWeatherDescription(bj.meo.latest$weather[bj.meo.latest$station_id==ULGridName & bj.meo.latest$time==time][1])\n",
    "    }\n",
    "    \n",
    "    bj.aq.latest.filled[[s]]$temperature <- tempVec\n",
    "    bj.aq.latest.filled[[s]]$pressure <- pressureVec\n",
    "    bj.aq.latest.filled[[s]]$humidity <- humidityVec\n",
    "    bj.aq.latest.filled[[s]]$windSpeed <- windSpeedVec\n",
    "    bj.aq.latest.filled[[s]]$windDir <- windDirVec\n",
    "    bj.aq.latest.filled[[s]]$weather <- weatherVec\n",
    "        \n",
    "}\n",
    "for (s in 19:35) {\n",
    "    # First we have to find the 4 closest grid meo stations to the aq station\n",
    "    aqName <- bj_station_names_vec[s]\n",
    "    ULGridName <- bj_aq_grid$ULGridName[bj_aq_grid$station_id==aqName]\n",
    "    URGridName <- bj_aq_grid$URGridName[bj_aq_grid$station_id==aqName]\n",
    "    LLGridName <- bj_aq_grid$LLGridName[bj_aq_grid$station_id==aqName]\n",
    "    LRGridName <- bj_aq_grid$LRGridName[bj_aq_grid$station_id==aqName]\n",
    "    bj.aq.latest.filled[[s]]$utc_time <- as.POSIXct(bj.aq.latest.filled[[s]]$utc_time, origin=\"1970-01-01\", tz=\"UTC\")\n",
    "    \n",
    "    tempVec <- NULL\n",
    "    humidityVec <- NULL\n",
    "    pressureVec <- NULL\n",
    "    windSpeedVec <- NULL\n",
    "    windDirVec <- NULL\n",
    "    weatherVec <- NULL\n",
    "    \n",
    "    for (i in 1:nrow(bj.aq.latest.filled[[s]])) {\n",
    "        time <- bj.aq.latest.filled[[s]]$utc_time[i]\n",
    "        tempVec[i] <- mean(c(getTemp(bj.meo.latest, ULGridName, time), getTemp(bj.meo.latest, URGridName, time), getTemp(bj.meo.latest, LLGridName, time), getTemp(bj.meo.latest, LRGridName, time)))\n",
    "        pressureVec[i] <- mean(c(getPressure(bj.meo.latest, ULGridName, time), getPressure(bj.meo.latest, URGridName, time), getPressure(bj.meo.latest, LLGridName, time), getPressure(bj.meo.latest, LRGridName, time)))\n",
    "        humidityVec[i] <- mean(c(getHumidity(bj.meo.latest, ULGridName, time), getHumidity(bj.meo.latest, URGridName, time), getHumidity(bj.meo.latest, LLGridName, time), getHumidity(bj.meo.latest, LRGridName, time)))        \n",
    "        windSpeedVec[i] <- mean(c(getWindSpeed(bj.meo.latest, ULGridName, time), getWindSpeed(bj.meo.latest, URGridName, time), getWindSpeed(bj.meo.latest, LLGridName, time), getWindSpeed(bj.meo.latest, LRGridName, time)))\n",
    "        windDirVec[i] <- getWindDir(bj.meo.latest, ULGridName, time)\n",
    "        weatherVec[i] <- getWeatherDescription(bj.meo.latest$weather[bj.meo.latest$station_id==ULGridName & bj.meo.latest$time==time][1])\n",
    "    }\n",
    "    \n",
    "    bj.aq.latest.filled[[s]]$temperature <- tempVec\n",
    "    bj.aq.latest.filled[[s]]$pressure <- pressureVec\n",
    "    bj.aq.latest.filled[[s]]$humidity <- humidityVec\n",
    "    bj.aq.latest.filled[[s]]$windSpeed <- windSpeedVec\n",
    "    bj.aq.latest.filled[[s]]$windDir <- windDirVec\n",
    "    bj.aq.latest.filled[[s]]$weather <- weatherVec\n",
    "        \n",
    "}\n",
    "toc(log = TRUE, quiet = FALSE, func.toc = my.msg.toc, info = \"INFO\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting Add meo to latest ld data...\n",
      "INFO: Add meo to latest ld data: 19.714 seconds elapsed\n"
     ]
    }
   ],
   "source": [
    "tic(\"Add meo to latest ld data\", quiet = FALSE, func.tic = my.msg.tic)\n",
    "# Add the meo features to the ld latest data\n",
    "for (s in 1:13) {\n",
    "    # First we have to find the 4 closest grid meo stations to the aq station\n",
    "    aqName <- ld_station_names_vec[s]\n",
    "    ULGridName <- ld_aq_grid$ULGridName[ld_aq_grid$station_id==aqName]\n",
    "    URGridName <- ld_aq_grid$URGridName[ld_aq_grid$station_id==aqName]\n",
    "    LLGridName <- ld_aq_grid$LLGridName[ld_aq_grid$station_id==aqName]\n",
    "    LRGridName <- ld_aq_grid$LRGridName[ld_aq_grid$station_id==aqName]\n",
    "    ld.aq.latest.filled[[s]]$utc_time <- as.POSIXct(ld.aq.latest.filled[[s]]$utc_time, origin=\"1970-01-01\", tz=\"UTC\")\n",
    "    \n",
    "    tempVec <- NULL\n",
    "    humidityVec <- NULL\n",
    "    pressureVec <- NULL\n",
    "    windSpeedVec <- NULL\n",
    "    windDirVec <- NULL\n",
    "    weatherVec <- NULL\n",
    "    \n",
    "    for (i in 1:nrow(ld.aq.latest.filled[[s]])) {\n",
    "        time <- ld.aq.latest.filled[[s]]$utc_time[i]\n",
    "        tempVec[i] <- mean(c(getTemp(ld.meo.latest, ULGridName, time), getTemp(ld.meo.latest, URGridName, time), getTemp(ld.meo.latest, LLGridName, time), getTemp(ld.meo.latest, LRGridName, time)))\n",
    "        pressureVec[i] <- mean(c(getPressure(ld.meo.latest, ULGridName, time), getPressure(ld.meo.latest, URGridName, time), getPressure(ld.meo.latest, LLGridName, time), getPressure(ld.meo.latest, LRGridName, time)))\n",
    "        humidityVec[i] <- mean(c(getHumidity(ld.meo.latest, ULGridName, time), getHumidity(ld.meo.latest, URGridName, time), getHumidity(ld.meo.latest, LLGridName, time), getHumidity(ld.meo.latest, LRGridName, time)))\n",
    "        windSpeedVec[i] <- mean(c(getWindSpeed(ld.meo.latest, ULGridName, time), getWindSpeed(ld.meo.latest, URGridName, time), getWindSpeed(ld.meo.latest, LLGridName, time), getWindSpeed(ld.meo.latest, LRGridName, time)))\n",
    "        windDirVec[i] <- getWindDir(ld.meo.latest, ULGridName, time)\n",
    "        weatherVec[i] <- weatherVec[i] <- getWeatherDescription(ld.meo.latest$weather[ld.meo.latest$station_id==ULGridName & ld.meo.latest$time==time][1])\n",
    "    }\n",
    "    \n",
    "    ld.aq.latest.filled[[s]]$temperature <- tempVec\n",
    "    ld.aq.latest.filled[[s]]$pressure <- pressureVec\n",
    "    ld.aq.latest.filled[[s]]$humidity <- humidityVec\n",
    "    ld.aq.latest.filled[[s]]$windSpeed <- windSpeedVec\n",
    "    ld.aq.latest.filled[[s]]$windDir <- windDirVec\n",
    "    ld.aq.latest.filled[[s]]$weather <- weatherVec\n",
    "        \n",
    "}\n",
    "toc(log = TRUE, quiet = FALSE, func.toc = my.msg.toc, info = \"INFO\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting Combine latest with Historical data...\n",
      "INFO: Combine latest with Historical data: 0.128 seconds elapsed\n"
     ]
    }
   ],
   "source": [
    "# combine BJ historical and latest\n",
    "tic(\"Combine latest with Historical data\", quiet = FALSE, func.tic = my.msg.tic)\n",
    "for (s in 1:35) {\n",
    "    bj.fullStationData[[s]]$utc_time <- as.POSIXct(bj.fullStationData[[s]]$utc_time, origin=\"1970-01-01\", format=\"%Y-%m-%d %H:%M:%S\", tz=\"UTC\")\n",
    "    bj.fullStationData[[s]] <- rbind(bj.fullStationData[[s]], bj.aq.latest.filled[[s]])\n",
    "    bj.fullStationData[[s]] <- bj.fullStationData[[s]][order(bj.fullStationData[[s]]$stationId, bj.fullStationData[[s]]$utc_time),]\n",
    "    bj.fullStationData[[s]] <- bj.fullStationData[[s]][!duplicated(bj.fullStationData[[s]][,c('utc_time')]),]   \n",
    "}\n",
    "\n",
    "toc(log = TRUE, quiet = FALSE, func.toc = my.msg.toc, info = \"INFO\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting Combine London latest with Historical data...\n",
      "INFO: Combine London latest with Historical data: 0.053 seconds elapsed\n"
     ]
    }
   ],
   "source": [
    "# combine London historical and latest\n",
    "tic(\"Combine London latest with Historical data\", quiet = FALSE, func.tic = my.msg.tic)\n",
    "for (s in 1:13) {\n",
    "    ld.fullStationData[[s]]$utc_time <- as.POSIXct(ld.fullStationData[[s]]$utc_time, origin=\"1970-01-01\", format=\"%Y-%m-%d %H:%M:%S\", tz=\"UTC\")\n",
    "    ld.fullStationData[[s]] <- rbind(ld.fullStationData[[s]], ld.aq.latest.filled[[s]])\n",
    "    ld.fullStationData[[s]] <- ld.fullStationData[[s]][order(ld.fullStationData[[s]]$stationId, ld.fullStationData[[s]]$utc_time),]\n",
    "    ld.fullStationData[[s]] <- ld.fullStationData[[s]][!duplicated(ld.fullStationData[[s]][,c('utc_time')]),]   \n",
    "}\n",
    "\n",
    "#rm(bj.aq.latest.filled)\n",
    "#rm(ld.aq.latest.filled)\n",
    "toc(log = TRUE, quiet = FALSE, func.toc = my.msg.toc, info = \"INFO\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting Convert categorical to factors...\n",
      "INFO: Convert categorical to factors: 0.15 seconds elapsed\n"
     ]
    }
   ],
   "source": [
    "tic(\"Convert categorical to factors\", quiet = FALSE, func.tic = my.msg.tic)\n",
    "# convert categorical variables to factors\n",
    "for (s in 1:35) {\n",
    "    #bj.fullStationData[[s]]$year <- as.factor(bj.fullStationData[[s]]$year)\n",
    "    #bj.fullStationData[[s]]$season <- as.factor(bj.fullStationData[[s]]$season)\n",
    "    bj.fullStationData[[s]]$month <- as.factor(bj.fullStationData[[s]]$month)\n",
    "    bj.fullStationData[[s]]$weekday <- as.factor(bj.fullStationData[[s]]$weekday)\n",
    "    bj.fullStationData[[s]]$hour <- as.factor(bj.fullStationData[[s]]$hour)\n",
    "    bj.fullStationData[[s]]$windDir <- as.factor(bj.fullStationData[[s]]$windDir)\n",
    "    bj.fullStationData[[s]]$weather <- factor(bj.fullStationData[[s]]$weather, levels=c(\"storm\",\"clear\",\"rainy\",\"snowy\",\"haze\",\"cloudy\"))\n",
    "}\n",
    "\n",
    "for (s in 1:13) {\n",
    "    #ld.fullStationData[[s]]$year <- as.factor(ld.fullStationData[[s]]$year)\n",
    "    #ld.fullStationData[[s]]$season <- as.factor(ld.fullStationData[[s]]$season)\n",
    "    ld.fullStationData[[s]]$month <- as.factor(ld.fullStationData[[s]]$month)\n",
    "    ld.fullStationData[[s]]$weekday <- as.factor(ld.fullStationData[[s]]$weekday)\n",
    "    ld.fullStationData[[s]]$hour <- as.factor(ld.fullStationData[[s]]$hour)\n",
    "    ld.fullStationData[[s]]$windDir <- as.factor(ld.fullStationData[[s]]$windDir)\n",
    "    ld.fullStationData[[s]]$weather <- factor(ld.fullStationData[[s]]$weather, levels=c(\"storm\",\"clear\",\"rainy\",\"snowy\",\"haze\",\"cloudy\"))\n",
    "}\n",
    "toc(log = TRUE, quiet = FALSE, func.toc = my.msg.toc, info = \"INFO\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting Remove all negative numbers and replace with 0...\n",
      "INFO: Remove all negative numbers and replace with 0: 2.324 seconds elapsed\n"
     ]
    }
   ],
   "source": [
    "# Remove all negative numbers and replace with 0\n",
    "tic(\"Remove all negative numbers and replace with 0\", quiet = FALSE, func.tic = my.msg.tic)\n",
    "for (s in 1:35) {\n",
    "    for (i in 1:nrow(bj.fullStationData[[s]])) {\n",
    "        if (bj.fullStationData[[s]]$PM2.5[i] < 0) {\n",
    "            bj.fullStationData[[s]]$PM2.5[i] <- 0\n",
    "        }\n",
    "        if (bj.fullStationData[[s]]$PM10[i] < 0) {\n",
    "            bj.fullStationData[[s]]$PM10[i] <- 0\n",
    "        }\n",
    "        if (bj.fullStationData[[s]]$O3[i] < 0) {\n",
    "            bj.fullStationData[[s]]$O3[i] <- 0\n",
    "        }\n",
    "        if (bj.fullStationData[[s]]$lwPM2.5[i] < 0) {\n",
    "            bj.fullStationData[[s]]$lwPM2.5[i] <- 0\n",
    "        }\n",
    "        if (bj.fullStationData[[s]]$lwPM10[i] < 0) {\n",
    "            bj.fullStationData[[s]]$lwPM10[i] <- 0\n",
    "        }\n",
    "        if (bj.fullStationData[[s]]$lwO3[i] < 0) {\n",
    "            bj.fullStationData[[s]]$lwO3[i] <- 0\n",
    "        }\n",
    "    }\n",
    "}\n",
    "\n",
    "for (s in 1:13) {\n",
    "    for (i in 1:nrow(ld.fullStationData[[s]])) {\n",
    "        if (ld.fullStationData[[s]]$PM2.5[i] < 0) {\n",
    "            ld.fullStationData[[s]]$PM2.5[i] <- 0\n",
    "        }\n",
    "        if (ld.fullStationData[[s]]$PM10[i] < 0) {\n",
    "            ld.fullStationData[[s]]$PM10[i] <- 0\n",
    "        }\n",
    "        if (ld.fullStationData[[s]]$lwPM2.5[i] < 0) {\n",
    "            ld.fullStationData[[s]]$lwPM2.5[i] <- 0\n",
    "        }\n",
    "        if (ld.fullStationData[[s]]$lwPM10[i] < 0) {\n",
    "            ld.fullStationData[[s]]$lwPM10[i] <- 0\n",
    "        }\n",
    "    }\n",
    "}\n",
    "toc(log = TRUE, quiet = FALSE, func.toc = my.msg.toc, info = \"INFO\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1] \"bj starting time: 2018-03-31 07:00:00\"\n",
      "[1] \"bj ending time: 2018-05-31 22:00:00\"\n",
      "[1] \"ld starting time: 2018-03-31 07:00:00\"\n",
      "[1] \"ld ending time: 2018-05-31 21:00:00\"\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th></th><th scope=col>used</th><th scope=col>(Mb)</th><th scope=col>gc trigger</th><th scope=col>(Mb)</th><th scope=col>limit (Mb)</th><th scope=col>max used</th><th scope=col>(Mb)</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><th scope=row>Ncells</th><td> 943777 </td><td>50.5    </td><td> 1869915</td><td>99.9    </td><td>   NA   </td><td>1869915 </td><td>99.9    </td></tr>\n",
       "\t<tr><th scope=row>Vcells</th><td>4234868 </td><td>32.4    </td><td>10146329</td><td>77.5    </td><td>16384   </td><td>8388608 </td><td>64.0    </td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|lllllll}\n",
       "  & used & (Mb) & gc trigger & (Mb) & limit (Mb) & max used & (Mb)\\\\\n",
       "\\hline\n",
       "\tNcells &  943777  & 50.5     &  1869915 & 99.9     &    NA    & 1869915  & 99.9    \\\\\n",
       "\tVcells & 4234868  & 32.4     & 10146329 & 77.5     & 16384    & 8388608  & 64.0    \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/markdown": [
       "\n",
       "| <!--/--> | used | (Mb) | gc trigger | (Mb) | limit (Mb) | max used | (Mb) | \n",
       "|---|---|\n",
       "| Ncells |  943777  | 50.5     |  1869915 | 99.9     |    NA    | 1869915  | 99.9     | \n",
       "| Vcells | 4234868  | 32.4     | 10146329 | 77.5     | 16384    | 8388608  | 64.0     | \n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "       used    (Mb) gc trigger (Mb) limit (Mb) max used (Mb)\n",
       "Ncells  943777 50.5  1869915   99.9    NA      1869915  99.9\n",
       "Vcells 4234868 32.4 10146329   77.5 16384      8388608  64.0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Check the time diff and nrow to be equal for the merged dataset\n",
    "bj_station_starting_times <- list()\n",
    "bj_station_ending_times <- list()\n",
    "bj_station_diff_times <- list()\n",
    "for (s in 1:35) {\n",
    "    nrow(bj.fullStationData[[s]])\n",
    "    bj_station_starting_times[[s]] <- head(bj.fullStationData[[s]]$utc_time,1)\n",
    "    bj_station_ending_times[[s]] <- tail(bj.fullStationData[[s]]$utc_time,1)\n",
    "\n",
    "    # Find the difference between the latest bj first and last record\n",
    "    bj_station_diff_times[[s]] <- as.integer(bj_station_ending_times[[s]] - bj_station_starting_times[[s]] + 1) * 24\n",
    "    - hour(bj_station_starting_times[[s]]) - (23 - hour(bj_station_ending_times[[s]]))\n",
    "}\n",
    "\n",
    "# Repeat for London\n",
    "ld_station_starting_times <- list()\n",
    "ld_station_ending_times <- list()\n",
    "ld_station_diff_times <- list()\n",
    "for (s in 1:13) {\n",
    "    nrow(ld.fullStationData[[s]])\n",
    "    ld_station_starting_times[[s]] <- head(ld.fullStationData[[s]]$utc_time,1)\n",
    "    ld_station_ending_times[[s]] <- tail(ld.fullStationData[[s]]$utc_time,1)\n",
    "    ld_station_diff_times[[s]] <- as.integer(ld_station_ending_times[[s]] - ld_station_starting_times[[s]] + 1) * 24\n",
    "    - hour(ld_station_starting_times[[s]]) - (23 - hour(ld_station_ending_times[[s]]))\n",
    "}\n",
    "\n",
    "bj_diff <- data.frame(matrix(unlist(bj_station_diff_times), nrow=35, byrow=T))\n",
    "ld_diff <- data.frame(matrix(unlist(ld_station_diff_times), nrow=13, byrow=T))\n",
    "\n",
    "if (sd(bj_diff[,1])!=0 || sd(ld_diff[,1]!=0)) {\n",
    "    print (\"Time mismatch detected between BJ stations\")\n",
    "}\n",
    "\n",
    "bj.startingTime <- bj_station_starting_times[[1]]\n",
    "print(paste(\"bj starting time:\",bj.startingTime))\n",
    "bj.endingTime <- bj_station_ending_times[[1]]\n",
    "print(paste(\"bj ending time:\",bj.endingTime))\n",
    "ld.startingTime <- ld_station_starting_times[[1]]\n",
    "print(paste(\"ld starting time:\",ld.startingTime))\n",
    "ld.endingTime <- ld_station_ending_times[[1]]\n",
    "print(paste(\"ld ending time:\",ld.endingTime))\n",
    "# Clear lists from memory\n",
    "rm(bj_station_starting_times)\n",
    "rm(bj_station_ending_times)\n",
    "rm(bj_station_diff_times)\n",
    "rm(ld_station_starting_times)\n",
    "rm(ld_station_ending_times)\n",
    "rm(ld_station_diff_times)\n",
    "rm(bj_diff)\n",
    "rm(ld_diff)\n",
    "gc()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "########################################################################\n",
    "## This is only for testing\n",
    "########################################################################\n",
    "# Create dataframe that will hold the last 48 hour results\n",
    "if (isValidation==1) {\n",
    "    tic(\"Remove two days from training set for validation\", quiet = FALSE, func.tic = my.msg.tic)\n",
    "    actual48Data <- as.data.frame(matrix(nrow = 2304, ncol = 4))\n",
    "    names(actual48Data) <- c(\"test_id\", \"PM2.5\", \"PM10\", \"O3\")\n",
    "    for (s in 1:35) {\n",
    "        last48rows <- tail(bj.fullStationData[[s]],48)\n",
    "        for (t in 1:48) {\n",
    "           actual48Data[(s-1)*48+t,] <- c(paste(bj_station_names_vec[s], \"#\", t-1, sep=\"\"),last48rows$PM2.5[t],\n",
    "                                        last48rows$PM10[t],last48rows$O3[t])\n",
    "        }\n",
    "        bj.fullStationData[[s]] <- head(bj.fullStationData[[s]],-48)\n",
    "    }\n",
    "\n",
    "    for (s in 1:13) {\n",
    "        last48rows <- tail(ld.fullStationData[[s]],48)\n",
    "        for (t in 1:48) {\n",
    "           actual48Data[(s-1)*48+t+1680,] <- c(paste(ld_station_names_vec[s], \"#\", t-1, sep=\"\"),last48rows$PM2.5[t],\n",
    "                                        last48rows$PM10[t],0)\n",
    "        }\n",
    "        ld.fullStationData[[s]] <- head(ld.fullStationData[[s]],-48)\n",
    "    }\n",
    "\n",
    "    write.csv(actual48Data, file = \"Validation48Data.csv\", row.names=FALSE)\n",
    "    toc(log = TRUE, quiet = FALSE, func.toc = my.msg.toc, info = \"INFO\")\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "############################################\n",
    "# Model Training Section\n",
    "############################################"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting Train RF for Beijing...\n",
      "INFO: Train RF for Beijing: 0.003 seconds elapsed\n"
     ]
    }
   ],
   "source": [
    "# Train the Random Forest models for each station\n",
    "tic(\"Train RF for Beijing\", quiet = FALSE, func.tic = my.msg.tic)\n",
    "\n",
    "if (trainRF == 1) \n",
    "{\n",
    "    \n",
    "for (s in 1:18) {\n",
    "    if (s==9) {\n",
    "        next #zhiwuyuan no data\n",
    "    }\n",
    "    #save each trained model to file\n",
    "    stationName <- bj.fullStationData[[s]]$stationId[1]\n",
    "    modelfile <- paste(\"rf_weather_models/\",stationName,\"_rf_PM25.RData\",sep=\"\")\n",
    "    rf_modelPM2.5 <- randomForest(bj.fullStationData[[s]][,c(\"hour\",\"month\",\"weekday\",\"temperature\",\"pressure\",\"humidity\",\"windSpeed\",\"windDir\",\"lwPM2.5\",\"tdPM2.5\",\"weather\")], y = bj.fullStationData[[s]]$PM2.5, ntree = ntree, mtry = mtry, \n",
    "                         nodesize=5, importance=TRUE)\n",
    "    save(rf_modelPM2.5, file = modelfile)\n",
    "    rm(rf_modelPM2.5)\n",
    "    \n",
    "    modelfile <- paste(\"rf_weather_models/\",stationName,\"_rf_PM10.RData\",sep=\"\")\n",
    "    rf_modelPM10 <- randomForest(bj.fullStationData[[s]][,c(\"hour\",\"month\",\"weekday\",\"temperature\",\"pressure\",\"humidity\",\"windSpeed\",\"windDir\",\"lwPM10\",\"tdPM10\",\"weather\")], y = bj.fullStationData[[s]]$PM10, ntree = ntree, mtry = mtry, \n",
    "                         nodesize=5, importance=TRUE)\n",
    "    save(rf_modelPM10, file = modelfile)\n",
    "    rm(rf_modelPM10)\n",
    "    \n",
    "    modelfile <- paste(\"rf_weather_models/\",stationName,\"_rf_O3.RData\",sep=\"\")\n",
    "    rf_modelO3 <- randomForest(bj.fullStationData[[s]][,c(\"hour\",\"month\",\"weekday\",\"temperature\",\"pressure\",\"humidity\",\"windSpeed\",\"windDir\",\"lwO3\",\"tdO3\",\"weather\")], y = bj.fullStationData[[s]]$O3, ntree = ntree, mtry = mtry, \n",
    "                         nodesize=5, importance=TRUE)  \n",
    "    save(rf_modelO3, file = modelfile)\n",
    "    rm(rf_modelO3)\n",
    "}\n",
    "\n",
    "for (s in 19:35) {\n",
    "    #save each trained model to file\n",
    "    stationName <- bj.fullStationData[[s]]$stationId[1]\n",
    "    modelfile <- paste(\"rf_weather_models/\",stationName,\"_rf_PM25.RData\",sep=\"\")\n",
    "    rf_modelPM2.5 <- randomForest(bj.fullStationData[[s]][,c(\"hour\",\"month\",\"weekday\",\"temperature\",\"pressure\",\"humidity\",\"windSpeed\",\"windDir\",\"lwPM2.5\",\"tdPM2.5\",\"weather\")], y = bj.fullStationData[[s]]$PM2.5, ntree = ntree, mtry = mtry, \n",
    "                         nodesize=5, importance=TRUE)\n",
    "    save(rf_modelPM2.5, file = modelfile)\n",
    "    rm(rf_modelPM2.5)\n",
    "    \n",
    "    modelfile <- paste(\"rf_weather_models/\",stationName,\"_rf_PM10.RData\",sep=\"\")\n",
    "    rf_modelPM10 <- randomForest(bj.fullStationData[[s]][,c(\"hour\",\"month\",\"weekday\",\"temperature\",\"pressure\",\"humidity\",\"windSpeed\",\"windDir\",\"lwPM10\",\"tdPM10\",\"weather\")], y = bj.fullStationData[[s]]$PM10, ntree = ntree, mtry = mtry, \n",
    "                         nodesize=5, importance=TRUE)\n",
    "    save(rf_modelPM10, file = modelfile)\n",
    "    rm(rf_modelPM10)\n",
    "    \n",
    "    modelfile <- paste(\"rf_weather_models/\",stationName,\"_rf_O3.RData\",sep=\"\")\n",
    "    rf_modelO3 <- randomForest(bj.fullStationData[[s]][,c(\"hour\",\"month\",\"weekday\",\"temperature\",\"pressure\",\"humidity\",\"windSpeed\",\"windDir\",\"lwO3\",\"tdO3\",\"weather\")], y = bj.fullStationData[[s]]$O3, ntree = ntree, mtry = mtry, \n",
    "                         nodesize=5, importance=TRUE)  \n",
    "    save(rf_modelO3, file = modelfile)\n",
    "    rm(rf_modelO3)\n",
    "}\n",
    "\n",
    "}\n",
    "toc(log = TRUE, quiet = FALSE, func.toc = my.msg.toc, info = \"INFO\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting Train RF for London...\n",
      "INFO: Train RF for London: 0.004 seconds elapsed\n"
     ]
    }
   ],
   "source": [
    "# Train the Random Forest models for each station\n",
    "tic(\"Train RF for London\", quiet = FALSE, func.tic = my.msg.tic)\n",
    "\n",
    "if (trainRF == 1) \n",
    "{\n",
    "    \n",
    "for (s in 1:13) {\n",
    "    #save each trained model to file\n",
    "    stationName <- ld.fullStationData[[s]]$stationId[1]\n",
    "    modelfile <- paste(\"rf_weather_models/\",stationName,\"_rf_PM25.RData\",sep=\"\")\n",
    "    rf_modelPM2.5 <- randomForest(ld.fullStationData[[s]][,c(\"hour\",\"month\",\"weekday\",\"temperature\",\"pressure\",\"humidity\",\"windSpeed\",\"windDir\",\"lwPM2.5\",\"tdPM2.5\",\"weather\")], y = ld.fullStationData[[s]]$PM2.5, ntree = ntree, mtry = mtry, \n",
    "                         nodesize=5, importance=TRUE)\n",
    "    save(rf_modelPM2.5, file = modelfile)\n",
    "    rm(rf_modelPM2.5)\n",
    "    \n",
    "    modelfile <- paste(\"rf_weather_models/\",stationName,\"_rf_PM10.RData\",sep=\"\")\n",
    "    rf_modelPM10 <- randomForest(ld.fullStationData[[s]][,c(\"hour\",\"month\",\"weekday\",\"temperature\",\"pressure\",\"humidity\",\"windSpeed\",\"windDir\",\"lwPM10\",\"tdPM10\",\"weather\")], y = ld.fullStationData[[s]]$PM10, ntree = ntree, mtry = mtry, \n",
    "                         nodesize=5, importance=TRUE)\n",
    "    save(rf_modelPM10, file = modelfile)\n",
    "    rm(rf_modelPM10)\n",
    "}\n",
    "    \n",
    "}\n",
    "\n",
    "toc(log = TRUE, quiet = FALSE, func.toc = my.msg.toc, info = \"INFO\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting Train SVR for Beijing...\n",
      "INFO: Train SVR for Beijing: 35.411 seconds elapsed\n"
     ]
    }
   ],
   "source": [
    "##################################\n",
    "# SVR Model\n",
    "#################################\n",
    "library(e1071)\n",
    "tic(\"Train SVR for Beijing\", quiet = FALSE, func.tic = my.msg.tic)\n",
    "if (trainSVR == 1) {\n",
    "    \n",
    "# Train the SVR models for each station\n",
    "\n",
    "for (s in 1:35) {\n",
    "    \n",
    "    if (s==9) {\n",
    "        next #zhiwuyuan no data\n",
    "    }\n",
    "    #save each trained model to file\n",
    "    stationName <- bj.fullStationData[[s]]$stationId[1]\n",
    "    modelfile <- paste(\"svr_weather_models/\",stationName,\"_svr_PM25.RData\",sep=\"\")\n",
    "    svr_modelPM2.5 <- svm(PM2.5 ~ hour+month+weekday+temperature+pressure+humidity+windSpeed+windDir+lwPM2.5+tdPM2.5+weather, data = bj.fullStationData[[s]], epsilon = 0.08, cost = 2^2, kernel=\"radial\", gamma = 0.2, type=\"eps-regression\")\n",
    "    save(svr_modelPM2.5, file = modelfile)\n",
    "    rm(svr_modelPM2.5)\n",
    "    \n",
    "    modelfile <- paste(\"svr_weather_models/\",stationName,\"_svr_PM10.RData\",sep=\"\")\n",
    "    svr_modelPM10 <- svm(PM10 ~ hour+month+weekday+temperature+pressure+humidity+windSpeed+windDir+lwPM10+tdPM10+weather, data = bj.fullStationData[[s]], epsilon = 0.08, cost = 2^2, kernel=\"radial\", gamma = 0.2, type=\"eps-regression\")\n",
    "    save(svr_modelPM10, file = modelfile)\n",
    "    rm(svr_modelPM10)\n",
    "    \n",
    "    modelfile <- paste(\"svr_weather_models/\",stationName,\"_svr_O3.RData\",sep=\"\")\n",
    "    svr_modelO3 <- svm(O3 ~ hour+month+weekday+temperature+pressure+humidity+windSpeed+windDir+lwO3+tdO3+weather, data = bj.fullStationData[[s]], epsilon = 0.08, cost = 2^2, kernel=\"radial\", gamma = 0.2, type=\"eps-regression\")\n",
    "    save(svr_modelO3, file = modelfile)\n",
    "    rm(svr_modelO3)\n",
    "}\n",
    "    \n",
    "}\n",
    "\n",
    "toc(log = TRUE, quiet = FALSE, func.toc = my.msg.toc, info = \"INFO\")    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting Train SVR for London...\n",
      "INFO: Train SVR for London: 9.032 seconds elapsed\n"
     ]
    }
   ],
   "source": [
    "tic(\"Train SVR for London\", quiet = FALSE, func.tic = my.msg.tic)\n",
    "if (trainSVR == 1) {\n",
    "    \n",
    "# Train the SVR models for each station\n",
    "for (s in 1:13) {\n",
    "    #save each trained model to file\n",
    "    stationName <- ld.fullStationData[[s]]$stationId[1]\n",
    "    modelfile <- paste(\"svr_weather_models/\",stationName,\"_svr_PM25.RData\",sep=\"\")\n",
    "    svr_modelPM2.5 <- svm(PM2.5 ~ hour+month+weekday+temperature+pressure+humidity+windSpeed+windDir+lwPM2.5+tdPM2.5+weather, data = ld.fullStationData[[s]], epsilon = 0.08, cost = 2^2, kernel=\"radial\", gamma = 0.2, type=\"eps-regression\")\n",
    "    save(svr_modelPM2.5, file = modelfile)\n",
    "    rm(svr_modelPM2.5)\n",
    "    \n",
    "    modelfile <- paste(\"svr_weather_models/\",stationName,\"_svr_PM10.RData\",sep=\"\")\n",
    "    svr_modelPM10 <- svm(PM10 ~ hour+month+weekday+temperature+pressure+humidity+windSpeed+windDir+lwPM10+tdPM10+weather, data = ld.fullStationData[[s]], epsilon = 0.08, cost = 2^2, kernel=\"radial\", gamma = 0.2, type=\"eps-regression\")\n",
    "    save(svr_modelPM10, file = modelfile)\n",
    "    rm(svr_modelPM10)\n",
    "}\n",
    "    \n",
    "}\n",
    "\n",
    "toc(log = TRUE, quiet = FALSE, func.toc = my.msg.toc, info = \"INFO\")  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "library(jsonlite)\n",
    "# Create dataframe that will hold the submission results\n",
    "submitData <- as.data.frame(matrix(nrow = 2304, ncol = 4))\n",
    "names(submitData) <- c(\"test_id\", \"PM2.5\", \"PM10\", \"O3\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "getWindDirLabel <- function(wind_dir) {\n",
    "    if (is.na(wind_dir)) {\n",
    "        return (\"NO\")\n",
    "    }\n",
    "    windDir <- as.numeric(wind_dir)\n",
    "    if (windDir > 337 & windDir<361 | windDir >=0 & windDir <= 22) {\n",
    "        windDir <- \"N\"\n",
    "    } else if (windDir > 22 & windDir <= 67) {\n",
    "        windDir <- \"NE\"\n",
    "    } else if (windDir > 67 & windDir <= 112) {\n",
    "        windDir <- \"E\"\n",
    "    }  else if (windDir > 112 & windDir <= 157) {\n",
    "        windDir <- \"SE\"\n",
    "    }  else if (windDir > 157 & windDir <= 202) {\n",
    "        windDir <- \"S\"\n",
    "    }  else if (windDir > 202 & windDir <= 247) {\n",
    "        windDir <- \"SW\"\n",
    "    }  else if (windDir > 247 & windDir <= 292) {\n",
    "        windDir <- \"W\"\n",
    "    }  else if (windDir > 292 & windDir <= 337) {\n",
    "        windDir <- \"NW\"\n",
    "    }  else {\n",
    "        windDir <- \"NO\"\n",
    "    }  \n",
    "    return (windDir)\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting Predict RF or SVR for Beijing...\n"
     ]
    },
    {
     "ename": "ERROR",
     "evalue": "Error in if (PM2.5 < 0) {: missing value where TRUE/FALSE needed\n",
     "output_type": "error",
     "traceback": [
      "Error in if (PM2.5 < 0) {: missing value where TRUE/FALSE needed\nTraceback:\n"
     ]
    }
   ],
   "source": [
    "##################################################\n",
    "# Prediction Section\n",
    "##################################################\n",
    "# Train the Random Forest models for each station\n",
    "tic(\"Predict RF or SVR for Beijing\", quiet = FALSE, func.tic = my.msg.tic)\n",
    "#data frame of 48 rows and number of features plus 3 prediction values of columns\n",
    "prediction_data <- as.data.frame(matrix(nrow = 48, ncol = 18)) \n",
    "names(prediction_data) <- c(\"PM2.5\", \"PM10\", \"O3\", \"hour\", \"month\", \"weekday\", \"lwPM2.5\", \"lwPM10\", \"lwO3\", \"temperature\", \"pressure\", \"humidity\", \"windSpeed\",\"windDir\",\"tdPM2.5\",\"tdPM10\",\"tdO3\",\"weather\")\n",
    "# Find the starting time of the next day (00:00 UTC)\n",
    "time <- tail(bj.fullStationData[[1]],1)$utc_time\n",
    "time <- time + 86400\n",
    "time <- time - hour(time)*3600 - minute(time)*60\n",
    "\n",
    "# Create the time and date values\n",
    "for (i in 1:48) {\n",
    "    hourly_time <- time + (i-1)*3600\n",
    "    prediction_data$hour[i] <- hour(hourly_time)\n",
    "    prediction_data$month[i] <- month(hourly_time)\n",
    "    prediction_data$weekday[i] <- wday(hourly_time)\n",
    "    #prediction_data$year[i] <- year(hourly_time)\n",
    "    #prediction_data$season[i] <- find_season(hourly_time)\n",
    "}\n",
    "\n",
    "# RF doesn't like NAs\n",
    "prediction_data$PM2.5 <- 0\n",
    "prediction_data$PM10 <- 0\n",
    "prediction_data$O3 <- 0\n",
    "#set categorical variables to match training data factors levels\n",
    "#prediction_data$year <- factor(prediction_data$year, levels=levels(bj.fullStationData[[1]]$year))\n",
    "#prediction_data$season <- factor(prediction_data$season, levels=levels(bj.fullStationData[[1]]$season))\n",
    "prediction_data$month <- factor(prediction_data$month, levels=levels(bj.fullStationData[[1]]$month))\n",
    "prediction_data$hour <- factor(prediction_data$hour, levels=levels(bj.fullStationData[[1]]$hour))\n",
    "prediction_data$weekday <- factor(prediction_data$weekday, levels=levels(bj.fullStationData[[1]]$weekday))\n",
    "\n",
    "# For every station:\n",
    "for (s in 1:35) {\n",
    "    \n",
    "    if (s==9) {\n",
    "      for (t in 1:48) {\n",
    "        submitData[(s-1)*48+t,] <- c(paste(bj_station_names_vec[s], \"#\", t-1, sep=\"\"),3,3,3)\n",
    "      }\n",
    "      next\n",
    "    }\n",
    "    \n",
    "    # First get the weather for the 48 hour period\n",
    "    lat <- bj_aq_grid$latitude[s]\n",
    "    lon <- bj_aq_grid$longitude[s]\n",
    "    URL<-paste(\"http://api.openweathermap.org/data/2.5/forecast?lat=\", lat, \"&lon=\", lon, \"&appid=d6f07a7bf09c7b5f2d743c38aa1999e2\", sep=\"\")\n",
    "    weather_data<-fromJSON(URL)\n",
    "    temp_list <- weather_data[[4]]$main[,1]\n",
    "    pressure_list <- weather_data[[4]]$main[,4]\n",
    "    humidity_list <- weather_data[[4]]$main[,7]\n",
    "    windSpeed_list <- weather_data[[4]]$wind[,1]\n",
    "    windDir_list <- weather_data[[4]]$wind[,2] \n",
    "    hour_list <- weather_data[[4]]$dt_txt\n",
    "    hour_list <- as.POSIXct(hour_list, origin=\"1970-01-01\", format=\"%Y-%m-%d %H:%M:%S\", tz=\"UTC\")\n",
    "    weather_description_list <- list()\n",
    "    for (i in 1:length(hour_list)) { weather_description_list[i] <- as.data.frame(weather_data[[4]]$weather[i])$icon }\n",
    "    index <- which(hour_list==time)\n",
    "    temperature17 <- temp_list[index:(index+16)]\n",
    "    pressure17 <- pressure_list[index:(index+16)]\n",
    "    humidity17 <- humidity_list[index:(index+16)]\n",
    "    windSpeed17 <- windSpeed_list[index:(index+16)]\n",
    "    windDir17 <- windDir_list[index:(index+16)]\n",
    "    weatherDescription17 <- weather_description_list[index:(index+16)]\n",
    "   \n",
    "    \n",
    "    for (i in 1:48) {\n",
    "        hourly_time <- time + (i-1)*3600\n",
    "        prediction_data$lwPM2.5[i] <- bj_find_last_week_val(s,hourly_time,\"PM2.5\")\n",
    "        prediction_data$lwPM10[i] <- bj_find_last_week_val(s,hourly_time,\"PM10\")\n",
    "        prediction_data$lwO3[i] <- bj_find_last_week_val(s,hourly_time,\"O3\")\n",
    "        prediction_data$tdPM2.5[i] <- bj_find_two_day_val(s,hourly_time,\"PM2.5\")\n",
    "        prediction_data$tdPM10[i] <- bj_find_two_day_val(s,hourly_time,\"PM10\")\n",
    "        prediction_data$tdO3[i] <- bj_find_two_day_val(s,hourly_time,\"O3\")\n",
    "        prediction_data$temperature[i] <- ifelse(i%%3==1, temperature17[(i+2)/3]-273.15, NA)\n",
    "        prediction_data$pressure[i] <- ifelse(i%%3==1, pressure17[(i+2)/3], NA)\n",
    "        prediction_data$humidity[i] <- ifelse(i%%3==1, humidity17[(i+2)/3], NA)\n",
    "        prediction_data$windSpeed[i] <- ifelse(i%%3==1, windSpeed17[(i+2)/3], NA)\n",
    "        if (i %% 3 == 1) {\n",
    "            windDirLabel <- getWindDirLabel(windDir17[(i+2)/3])\n",
    "            weatherDescription <- getWeatherDescription(weatherDescription17[(i+2)/3])\n",
    "        } else {\n",
    "            windDirLabel <- prediction_data$windDir[i-1]\n",
    "            weatherDescription <- prediction_data$weather[i-1]\n",
    "        }\n",
    "        prediction_data$windDir[i] <- windDirLabel\n",
    "        prediction_data$weather[i] <- weatherDescription\n",
    "    }\n",
    "    \n",
    "    #convert windir category to factor\n",
    "    prediction_data$windDir <- factor(prediction_data$windDir, levels=levels(bj.fullStationData[[s]]$windDir))\n",
    "    prediction_data$weather <- factor(prediction_data$weather, levels=levels(bj.fullStationData[[s]]$weather))\n",
    "    \n",
    "    prediction_data$temperature <- na.kalman(prediction_data$temperature)\n",
    "    prediction_data$pressure <- na.kalman(prediction_data$pressure)\n",
    "    prediction_data$humidity <- na.kalman(prediction_data$humidity)\n",
    "    prediction_data$windSpeed <- na.kalman(prediction_data$windSpeed)\n",
    "    \n",
    "    # Load the saved RF or SVR model\n",
    "    stationName <- bj.fullStationData[[s]]$stationId[1]\n",
    "    if (toPredict == \"RF\") {       \n",
    "        model_name <- paste(\"rf_weather_models/\",stationName,\"_rf_PM25.RData\",sep=\"\")\n",
    "        rf_modelPM2.5 <- get(load(model_name))\n",
    "        pred_testPM2.5 <- predict(rf_modelPM2.5, newdata=prediction_data, type='response')\n",
    "    \n",
    "        model_name <- paste(\"rf_weather_models/\",stationName,\"_rf_PM10.RData\",sep=\"\")\n",
    "        rf_modelPM10 <- get(load(model_name))\n",
    "        pred_testPM10 <- predict(rf_modelPM10, newdata=prediction_data, type='response')\n",
    "    \n",
    "        model_name <- paste(\"rf_weather_models/\",stationName,\"_rf_O3.RData\",sep=\"\")\n",
    "        rf_modelO3 <- get(load(model_name))\n",
    "        pred_testO3 <- predict(rf_modelO3, newdata=prediction_data, type='response')\n",
    "    } \n",
    "    else {\n",
    "        model_name <- paste(\"svr_weather_models/\",stationName,\"_svr_PM25.RData\",sep=\"\")\n",
    "        svr_modelPM2.5 <- get(load(model_name))\n",
    "        pred_testPM2.5 <- predict(svr_modelPM2.5, prediction_data)\n",
    "    \n",
    "        model_name <- paste(\"svr_weather_models/\",stationName,\"_svr_PM10.RData\",sep=\"\")\n",
    "        svr_modelPM10 <- get(load(model_name))\n",
    "        pred_testPM10 <- predict(svr_modelPM10, prediction_data)\n",
    "    \n",
    "        model_name <- paste(\"svr_weather_models/\",stationName,\"_svr_O3.RData\",sep=\"\")\n",
    "        svr_modelO3 <- get(load(model_name))\n",
    "        pred_testO3 <- predict(svr_modelO3, prediction_data)     \n",
    "    }\n",
    "\n",
    "    for (t in 1:48) {\n",
    "        PM2.5 <- pred_testPM2.5[t]\n",
    "        if (PM2.5<0) {\n",
    "            PM2.5 <- 0\n",
    "        }\n",
    "        PM10 <- pred_testPM10[t]\n",
    "        if (PM10<0) {\n",
    "            PM10 <- 0\n",
    "        }\n",
    "        O3 <- pred_testO3[t]\n",
    "        if (O3<0) {\n",
    "            O3 <- 0\n",
    "        }\n",
    "        submitData[(s-1)*48+t,] <- c(paste(bj_station_names_vec[s], \"#\", t-1, sep=\"\"),PM2.5,PM10,O3)\n",
    "    }\n",
    "\n",
    "}\n",
    "\n",
    "toc(log = TRUE, quiet = FALSE, func.toc = my.msg.toc, info = \"INFO\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Predict RF model for London stations\n",
    "tic(\"Predict RF or SVR for London\", quiet = FALSE, func.tic = my.msg.tic)\n",
    "for (s in 1:13) {\n",
    "    \n",
    "    # First get the weather for the 48 hour period\n",
    "    lat <- ld_aq_grid$latitude[s]\n",
    "    lon <- ld_aq_grid$longitude[s]\n",
    "    URL<-paste(\"http://api.openweathermap.org/data/2.5/forecast?lat=\", lat, \"&lon=\", lon, \"&appid=d6f07a7bf09c7b5f2d743c38aa1999e2\", sep=\"\")\n",
    "    weather_data<-fromJSON(URL)\n",
    "    temp_list <- weather_data[[4]]$main[,1]\n",
    "    pressure_list <- weather_data[[4]]$main[,4]\n",
    "    humidity_list <- weather_data[[4]]$main[,7]\n",
    "    windSpeed_list <- weather_data[[4]]$wind[,1]\n",
    "    windDir_list <- weather_data[[4]]$wind[,2]\n",
    "    hour_list <- weather_data[[4]]$dt_txt\n",
    "    hour_list <- as.POSIXct(hour_list, origin=\"1970-01-01\", format=\"%Y-%m-%d %H:%M:%S\", tz=\"UTC\")\n",
    "    weather_description_list <- list()\n",
    "    for (i in 1:length(hour_list)) { weather_description_list[i] <- as.data.frame(weather_data[[4]]$weather[i])$icon }\n",
    "    index <- which(hour_list==time)\n",
    "    temperature17 <- temp_list[index:(index+16)]\n",
    "    pressure17 <- pressure_list[index:(index+16)]\n",
    "    humidity17 <- humidity_list[index:(index+16)]\n",
    "    windSpeed17 <- windSpeed_list[index:(index+16)]\n",
    "    windDir17 <- windDir_list[index:(index+16)]\n",
    "    weatherDescription17 <- weather_description_list[index:(index+16)]\n",
    "    \n",
    "    for (i in 1:48) {\n",
    "        hourly_time <- time + (i-1)*3600\n",
    "        prediction_data$lwPM2.5[i] <- ld_find_last_week_val(s,t,\"PM2.5\")\n",
    "        prediction_data$lwPM10[i] <- ld_find_last_week_val(s,t,\"PM10\")\n",
    "        prediction_data$tdPM2.5[i] <- ld_find_two_day_val(s,hourly_time,\"PM2.5\")\n",
    "        prediction_data$tdPM10[i] <- ld_find_two_day_val(s,hourly_time,\"PM10\")\n",
    "        prediction_data$temperature[i] <- ifelse(i%%3==1, temperature17[(i+2)/3]-273.15, NA)\n",
    "        prediction_data$pressure[i] <- ifelse(i%%3==1, pressure17[(i+2)/3], NA)\n",
    "        prediction_data$humidity[i] <- ifelse(i%%3==1, humidity17[(i+2)/3], NA)\n",
    "        prediction_data$windSpeed[i] <- ifelse(i%%3==1, windSpeed17[(i+2)/3], NA)\n",
    "        if (i %% 3 == 1) {\n",
    "            windDirLabel <- getWindDirLabel(windDir17[(i+2)/3])\n",
    "            weatherDescription <- getWeatherDescription(weatherDescription17[(i+2)/3])\n",
    "        } else {\n",
    "            windDirLabel <- prediction_data$windDir[i-1]\n",
    "            weatherDescription <- prediction_data$weather[i-1]\n",
    "        }\n",
    "        prediction_data$windDir[i] <- windDirLabel\n",
    "        prediction_data$weather[i] <- weatherDescription\n",
    "    }\n",
    "    \n",
    "    #convert windir category to factor\n",
    "    prediction_data$windDir <- factor(prediction_data$windDir, levels=levels(ld.fullStationData[[s]]$windDir))\n",
    "    prediction_data$weather <- factor(prediction_data$weather, levels=levels(ld.fullStationData[[s]]$weather))\n",
    "    \n",
    "    prediction_data$temperature <- na.kalman(prediction_data$temperature)\n",
    "    prediction_data$pressure <- na.kalman(prediction_data$pressure)\n",
    "    prediction_data$humidity <- na.kalman(prediction_data$humidity)\n",
    "    prediction_data$windSpeed <- na.kalman(prediction_data$windSpeed)\n",
    "    \n",
    "    # Load the saved RF or SVR model\n",
    "    stationName <- ld.fullStationData[[s]]$stationId[1]\n",
    "    if (toPredict == \"RF\") {       \n",
    "        model_name <- paste(\"rf_weather_models/\",stationName,\"_rf_PM25.RData\",sep=\"\")\n",
    "        rf_modelPM2.5 <- get(load(model_name))\n",
    "        pred_testPM2.5 <- predict(rf_modelPM2.5, newdata=prediction_data, type='response')\n",
    "    \n",
    "        model_name <- paste(\"rf_weather_models/\",stationName,\"_rf_PM10.RData\",sep=\"\")\n",
    "        rf_modelPM10 <- get(load(model_name))\n",
    "        pred_testPM10 <- predict(rf_modelPM10, newdata=prediction_data, type='response')\n",
    "    } \n",
    "    else {\n",
    "        model_name <- paste(\"svr_weather_models/\",stationName,\"_svr_PM25.RData\",sep=\"\")\n",
    "        svr_modelPM2.5 <- get(load(model_name))\n",
    "        pred_testPM2.5 <- predict(svr_modelPM2.5, prediction_data)\n",
    "    \n",
    "        model_name <- paste(\"svr_weather_models/\",stationName,\"_svr_PM10.RData\",sep=\"\")\n",
    "        svr_modelPM10 <- get(load(model_name))\n",
    "        pred_testPM10 <- predict(svr_modelPM10, prediction_data)\n",
    "   }\n",
    "        \n",
    "    for (t in 1:48) {\n",
    "        PM2.5 <- pred_testPM2.5[t]\n",
    "        if (PM2.5<0) {\n",
    "            PM2.5 <- 0\n",
    "        }\n",
    "        PM10 <- pred_testPM10[t]\n",
    "        if (PM10<0) {\n",
    "            PM10 <- 0\n",
    "        }\n",
    "        submitData[(s-1)*48+t+1680,] <- c(paste(ld_station_names_vec[s], \"#\", t-1, sep=\"\"),PM2.5,PM10,0)\n",
    "    }\n",
    "    \n",
    "}\n",
    "toc(log = TRUE, quiet = FALSE, func.toc = my.msg.toc, info = \"INFO\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "filename <- paste(\"UWDS420LiangTiaoMalax_submission_\",toPredict,\"_weather.csv\",sep=\"\")\n",
    "write.csv(submitData, file = filename, row.names=FALSE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ################################################\n",
    "# Starting ETS Section\n",
    "#################################################\n",
    "# Find how many hours till midnight and add them to the ETS prediction length\n",
    "bj.hoursUntilMidnight <- 23 - hour(bj.endingTime)\n",
    "bj.hoursUntilMidnight\n",
    "ld.hoursUntilMidnight <- 23 - hour(ld.endingTime)\n",
    "ld.hoursUntilMidnight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#################################################################\n",
    "# For Validation only\n",
    "#################################################################\n",
    "if (isValidation==1) {\n",
    "    bj.endingTime <- bj.endingTime - 48*3600\n",
    "    ld.endingTime <- ld.endingTime - 48*3600\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create time series model for ETS or ARIMA prediction\n",
    "tic(\"Create ETS time series for BJ\", quiet = FALSE, func.tic = my.msg.tic)\n",
    "bj_time_index <- seq(from = bj.startingTime, to = bj.endingTime, by = \"hour\")\n",
    "\n",
    "for (s in 1:35) {\n",
    "    bj_time_index <- seq(from = bj.startingTime, to = bj.endingTime, by = \"hour\") #2018-03-17 18:00\n",
    "    set.seed(1)\n",
    "    valuePM2.5 <- bj.fullStationData[[s]]$PM2.5\n",
    "    valuePM10 <- bj.fullStationData[[s]]$PM10\n",
    "    valueO3 <- bj.fullStationData[[s]]$O3\n",
    "    \n",
    "    eventdataPM2.5 <- xts(valuePM2.5, order.by = bj_time_index)\n",
    "    eventdataPM10 <- xts(valuePM10, order.by = bj_time_index)\n",
    "    eventdataO3 <- xts(valueO3, order.by = bj_time_index)\n",
    "    \n",
    "    ets_dataPM2.5 <- ets(eventdataPM2.5, model=\"AAN\")\n",
    "    ets_dataPM10 <- ets(eventdataPM10, model=\"AAN\")\n",
    "    ets_dataO3 <- ets(eventdataO3, model=\"AAN\")\n",
    "      \n",
    "    bj.PM2.5.forecast = forecast(ets_dataPM2.5, h=bj.hoursUntilMidnight+48, method=\"ets\")\n",
    "    bj.PM10.forecast = forecast(ets_dataPM10, h=bj.hoursUntilMidnight+48, method=\"ets\")\n",
    "    bj.O3.forecast = forecast(ets_dataO3, h=bj.hoursUntilMidnight+48, method=\"ets\")\n",
    "\n",
    "    for (t in 1:48) {\n",
    "       submitData[(s-1)*48+t,] <- c(paste(bj_station_names_vec[s], \"#\", t-1, sep=\"\"),abs(bj.PM2.5.forecast$mean[t+bj.hoursUntilMidnight]),\n",
    "                                    abs(bj.PM10.forecast$mean[t+bj.hoursUntilMidnight]),abs(bj.O3.forecast$mean[t+bj.hoursUntilMidnight]))\n",
    "    }\n",
    "    \n",
    "}\n",
    "\n",
    "toc(log = TRUE, quiet = FALSE, func.toc = my.msg.toc, info = \"INFO\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tic(\"Create ETS time series for LD\", quiet = FALSE, func.tic = my.msg.tic)\n",
    "ld_time_index <- seq(from = ld.startingTime, to = ld.endingTime, by = \"hour\")\n",
    "\n",
    "for (s in 1:13) {\n",
    "    ld_time_index <- seq(from = ld.startingTime, to = ld.endingTime, by = \"hour\") #2018-03-17 18:00\n",
    "    set.seed(1)\n",
    "    valuePM2.5 <- ld.fullStationData[[s]]$PM2.5\n",
    "    valuePM10 <- ld.fullStationData[[s]]$PM10\n",
    "    \n",
    "    eventdataPM2.5 <- xts(valuePM2.5, order.by = ld_time_index)\n",
    "    eventdataPM10 <- xts(valuePM10, order.by = ld_time_index)\n",
    "    \n",
    "    ets_dataPM2.5 <- ets(eventdataPM2.5, model=\"AAN\")\n",
    "    ets_dataPM10 <- ets(eventdataPM10, model=\"AAN\")\n",
    "       \n",
    "    ld.PM2.5.forecast = forecast(ets_dataPM2.5, h=ld.hoursUntilMidnight+48, method=\"ets\")\n",
    "    ld.PM10.forecast = forecast(ets_dataPM10, h=ld.hoursUntilMidnight+48, method=\"ets\")\n",
    "\n",
    "    for (t in 1:48) {\n",
    "       submitData[(s-1)*48+t+1680,] <- c(paste(ld_station_names_vec[s], \"#\", t-1, sep=\"\"),abs(ld.PM2.5.forecast$mean[t+ld.hoursUntilMidnight]),\n",
    "                                    abs(ld.PM10.forecast$mean[t+ld.hoursUntilMidnight]),0)\n",
    "    }\n",
    "    \n",
    "}\n",
    "\n",
    "toc(log = TRUE, quiet = FALSE, func.toc = my.msg.toc, info = \"INFO\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "write.csv(submitData, file = \"UWDS420LiangTiaoMalax_submission_ETS_weather.csv\", row.names=FALSE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#########################################\n",
    "# Update Historical Datasets - Write to Files\n",
    "########################################\n",
    "if (updateHistorical == 1) {\n",
    "\n",
    "    tic(\"Update historical set\", quiet = FALSE, func.tic = my.msg.tic)\n",
    "    # Write Beijing new Historical\n",
    "    for (s in 1:35) {\n",
    "        outfile <- paste(writeHistoricalFolderName,\"/\",bj.fullStationData[[s]]$stationId[1],\"_historical.csv\",sep=\"\")\n",
    "        write.csv(bj.fullStationData[[s]], file = outfile, row.names=FALSE)\n",
    "    }\n",
    "    \n",
    "    # Write London new Historical\n",
    "    for (s in 1:13) {\n",
    "        outfile <- paste(writeHistoricalFolderName,\"/\",ld.fullStationData[[s]]$stationId[1],\"_historical.csv\",sep=\"\")\n",
    "        write.csv(ld.fullStationData[[s]], file = outfile, row.names=FALSE)\n",
    "    }\n",
    "    \n",
    "    toc(log = TRUE, quiet = FALSE, func.toc = my.msg.toc, info = \"INFO\")\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "toc(log = TRUE, quiet = FALSE, func.toc = my.msg.toc, info = \"INFO\")\n",
    "log.txt <- tic.log(format = TRUE)\n",
    "log.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "R",
   "language": "R",
   "name": "ir"
  },
  "language_info": {
   "codemirror_mode": "r",
   "file_extension": ".r",
   "mimetype": "text/x-r-source",
   "name": "R",
   "pygments_lexer": "r",
   "version": "3.5.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}