QueryBuilder_v1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hideCode": true,
    "hideOutput": false,
    "hidePrompt": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<h3 style=\"color:#42A5F5;\">Run this cell and click the button to initialize.</h3>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%html\n",
    "<!-- P0 -->\n",
    "<div class=\"aegis\">\n",
    "    <button type=\"button\" class=\"jupyter-button jupyter-widgets mod-primary widget-button\" onclick=\"initQB()\">Initialise\n",
    "        QB</button>\n",
    "</div>\n",
    "<script>\n",
    "    $.ajaxSetup({\n",
    "        headers: { \"Authorization\": localStorage.getItem(\"token\") }\n",
    "    });\n",
    "    function initQB() {\n",
    "        Jupyter.notebook.execute_cells([1]);\n",
    "        Jupyter.notebook.execute_cells([2, 3, 4, 5, 6, 7, 8]);\n",
    "    }\n",
    "\n",
    "</script>\n",
    "\n",
    "<style>\n",
    "    .aegis .control-label {\n",
    "        padding-right: 10px;\n",
    "        font-weight: bold;\n",
    "    }\n",
    "\n",
    "    .aegis .widget-dropdown {\n",
    "        width: 100%;\n",
    "        height: inherit;\n",
    "    }\n",
    "\n",
    "    .aegis .widget-dropdown>select {\n",
    "        margin-left: 0;\n",
    "        margin-right: 0;\n",
    "    }\n",
    "\n",
    "    .aegis .input-group .jupyter-button {\n",
    "        padding: 6px 12px;\n",
    "        line-height: 1.42857143;\n",
    "        /*height: 32px;*/\n",
    "    }\n",
    "\n",
    "    .aegis .form-control {\n",
    "        padding-top: 0;\n",
    "        padding-bottom: 0;\n",
    "        height: 28px;\n",
    "        border: 1px solid #9E9E9E;\n",
    "        border-radius: 0;\n",
    "        padding-left: 8px;\n",
    "        -webkit-box-shadow: none;\n",
    "        box-shadow: none;\n",
    "        color: rgba(0, 0, 0, .8);\n",
    "    }\n",
    "\n",
    "    .aegis select[multiple],\n",
    "    .aegis select[size] {\n",
    "        height: auto;\n",
    "    }\n",
    "\n",
    "    .aegis #fileBrowser option[dir=\"true\"] {\n",
    "        font-weight: bold;\n",
    "    }\n",
    "\n",
    "    .aegis input[type=checkbox] {\n",
    "        height: 28px;\n",
    "    }\n",
    "\n",
    "    .output pre {\n",
    "        white-space: pre;\n",
    "    }\n",
    "\n",
    "    #list2 {\n",
    "        width: 320px;\n",
    "    }\n",
    "\n",
    "    #list2 ol {\n",
    "        font-size: 18px;\n",
    "        color: #337ab7;\n",
    "    }\n",
    "\n",
    "    #list2 ol li p {\n",
    "        padding: 8px;\n",
    "        font-style: normal;\n",
    "        font-size: 13px;\n",
    "        color: black;\n",
    "        border-left: 1px solid #999;\n",
    "    }\n",
    "\n",
    "    #list2 ol li.applied p em {\n",
    "        color: lightgray;\n",
    "    }\n",
    "\n",
    "    #list2 ol li p em {\n",
    "        display: block;\n",
    "    }\n",
    "\n",
    "    #filter-application-buttons {\n",
    "        text-align: center;\n",
    "    }\n",
    "\n",
    "    .aegis h4 {\n",
    "        margin: 9px 0px;\n",
    "    }\n",
    "\n",
    "    .aegis h4 span {\n",
    "        font-style: italic;\n",
    "        font-weight: initial;\n",
    "    }\n",
    "    \n",
    "    .output_scroll .output_subarea.output_text {\n",
    "        max-width: inherit;\n",
    "    }\n",
    "</style>\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hideCode": true,
    "hideOutput": false,
    "hidePrompt": false
   },
   "outputs": [],
   "source": [
    "#P1\n",
    "\n",
    "from pyspark.sql.functions import col, isnan, isnull, quarter, ntile, month, hour, minute, dayofmonth, dayofyear, to_date, unix_timestamp, trunc, datediff, months_between, sqrt, when\n",
    "from pyspark.sql.window import Window\n",
    "from pyspark.sql import Row, functions as F\n",
    "from pyspark.sql.functions import unix_timestamp\n",
    "from  pyspark.sql.functions import abs as cabs\n",
    "from pyspark.sql.functions import round as pyround\n",
    "import json\n",
    "import requests\n",
    "from hops import hdfs\n",
    "import pydoop.hdfs as phdfs\n",
    "from  pyspark.sql.functions import year, ltrim, rtrim\n",
    "from pyspark.sql.functions import coalesce, lit\n",
    "\n",
    "myhdfs = phdfs.hdfs(user = hdfs.project_user())\n",
    "\n",
    "tempDF = None\n",
    "masterDF = None\n",
    "hdfspath = hdfs.project_path()\n",
    "g=None\n",
    "query_init=\"from pyspark.sql.functions import col, isnan, isnull, quarter, ntile, month, hour, minute, dayofmonth, dayofyear, to_date, unix_timestamp, trunc, datediff, months_between, sqrt, when, unix_timestamp, year, ltrim, rtrim, coalesce, lit \\nfrom pyspark.sql.window import Window \\nfrom pyspark.sql import Row, functions as F\\nfrom  pyspark.sql.functions import abs as cabs \\nfrom pyspark.sql.functions import round as pyround\\n\"\n",
    "query = query_init\n",
    "\n",
    "def apply_filters(applied_filters):\n",
    "    global tempDF\n",
    "    global masterDF\n",
    "    applied_filters = json.loads(applied_filters)\n",
    "    global g\n",
    "    global query\n",
    "    g = applied_filters\n",
    "    filters_success = []\n",
    "    for i in range(1,len(applied_filters)+1):\n",
    "        index = str(i)\n",
    "        filter_type = applied_filters[index][\"type\"]\n",
    "        if applied_filters[index][\"executed\"]:\n",
    "            continue  \n",
    "        try:\n",
    "            if filter_type==\"filter\":\n",
    "                applied_filters[index][\"parameters\"][0] = \"`\"+applied_filters[index][\"parameters\"][0]+\"`\"\n",
    "                if applied_filters[index][\"parameters\"][1]==\"like\":\n",
    "                    tempDF = tempDF.filter(applied_filters[index][\"parameters\"][0]+\" \"+applied_filters[index][\"parameters\"][1]+\" \\'\"+applied_filters[index][\"parameters\"][2][:-1]+\"%\\'\")\n",
    "                    query+=\"tempDF = tempDF.filter(\\\"\"+applied_filters[index][\"parameters\"][0]+\" \"+applied_filters[index][\"parameters\"][1]+\" \\'\"+applied_filters[index][\"parameters\"][2][:-1]+\"%\\'\\\")\\n\"\n",
    "                else:\n",
    "                    try:\n",
    "                        float(applied_filters[index][\"parameters\"][2])\n",
    "                        tempDF = tempDF.filter(\" \".join(applied_filters[index][\"parameters\"]))\n",
    "                        query+=\"tempDF = tempDF.filter(\\\"\"+\" \".join(applied_filters[index][\"parameters\"])+\"\\\")\\n\"\n",
    "                    except ValueError as ver:\n",
    "                        applied_filters[index][\"parameters\"][2] = \"'\"+applied_filters[index][\"parameters\"][2]+\"'\"\n",
    "                        tempDF = tempDF.filter(\" \".join(applied_filters[index][\"parameters\"]))\n",
    "                        query+=\"tempDF = tempDF.filter(\\\"\"+\" \".join(applied_filters[index][\"parameters\"])+\"\\\")\\n\"\n",
    "            if filter_type==\"filterOr\":\n",
    "                applied_filters[index][\"parameters\"][0] = \"`\"+applied_filters[index][\"parameters\"][0]+\"`\"\n",
    "                if applied_filters[index][\"parameters\"][1]==\"like\":\n",
    "                    cond1 = applied_filters[index][\"parameters\"][0]+\" \"+applied_filters[index][\"parameters\"][1]+\" \\'\"+applied_filters[index][\"parameters\"][2][:-1]+\"%\\'\"\n",
    "                else:\n",
    "                    try:\n",
    "                        float(applied_filters[index][\"parameters\"][2])\n",
    "                    except ValueError as ver:       \n",
    "                        applied_filters[index][\"parameters\"][2] = \"'\"+applied_filters[index][\"parameters\"][2]+\"'\"\n",
    "                    cond1 = \" \".join(applied_filters[index][\"parameters\"][:3])\n",
    "\n",
    "                applied_filters[index][\"parameters\"][3] = \"`\"+applied_filters[index][\"parameters\"][3]+\"`\"\n",
    "                if applied_filters[index][\"parameters\"][4]==\"like\":\n",
    "                    cond2 = applied_filters[index][\"parameters\"][3]+\" \"+applied_filters[index][\"parameters\"][4]+\" \\'\"+applied_filters[index][\"parameters\"][5][:-1]+\"%\\'\"\n",
    "                else:\n",
    "                    try:\n",
    "                        float(applied_filters[index][\"parameters\"][5])\n",
    "                    except ValueError as ver:       \n",
    "                        applied_filters[index][\"parameters\"][5] = \"'\"+applied_filters[index][\"parameters\"][5]+\"'\"\n",
    "                    cond2 = \" \".join(applied_filters[index][\"parameters\"][3:6])\n",
    "\n",
    "                tempDF = tempDF.filter(cond1 + \" or \" + cond2)\n",
    "                query+=\"tempDF = tempDF.filter(\\\"\"+cond1 + \" or \" + cond2+\"\\\")\\n\"\n",
    "            elif filter_type==\"join\":\n",
    "                paramsList = [x for x in applied_filters[index][\"parameters\"]]\n",
    "                keep = [x for x in paramsList[1:] if (None not in x)]\n",
    "                jhow = str(paramsList[0])\n",
    "                if len(keep) == 1:\n",
    "                    tempDF = masterDF.alias('masterDF').join(tempDF.alias('tempDF'),on=[col('masterDF.'+keep[0][0])==col('tempDF.'+keep[0][1])],how=jhow)\n",
    "                elif len(keep) == 2:\n",
    "                    tempDF = masterDF.alias('masterDF').join(tempDF.alias('tempDF'),on=[col('masterDF.'+keep[0][0])==col('tempDF.'+keep[0][1]),col('masterDF.'+keep[1][0])==col('tempDF.'+keep[1][1])],how=jhow)\n",
    "                elif len(keep) == 3:\n",
    "                    tempDF = masterDF.alias('masterDF').join(tempDF.alias('tempDF'),on=[col('masterDF.'+keep[0][0])==col('tempDF.'+keep[0][1]),col('masterDF.'+keep[1][0])==col('tempDF.'+keep[1][1]),col('masterDF.'+keep[2][0])==col('tempDF.'+keep[2][1])],how=jhow)\n",
    "            elif filter_type==\"year\":\n",
    "                applied_filters[index][\"parameters\"][0] = \"`\"+applied_filters[index][\"parameters\"][0]+\"`\"\n",
    "                applied_filters[index][\"parameters\"][1] = applied_filters[index][\"parameters\"][1].replace(\" \",\"_\")\n",
    "                tempDF = tempDF.withColumn(applied_filters[index][\"parameters\"][1],year(tempDF[applied_filters[index][\"parameters\"][0]]))\n",
    "                query+= \"tempDF = tempDF.withColumn(\\\"\"+applied_filters[index][\"parameters\"][1]+\"\\\",year(tempDF[\\\"\"+applied_filters[index][\"parameters\"][0]+\"\\\"]))\\n\"\n",
    "            elif filter_type==\"quarter\":\n",
    "                applied_filters[index][\"parameters\"][0] = \"`\"+applied_filters[index][\"parameters\"][0]+\"`\"\n",
    "                applied_filters[index][\"parameters\"][1] = applied_filters[index][\"parameters\"][1].replace(\" \",\"_\")\n",
    "                tempDF = tempDF.withColumn(applied_filters[index][\"parameters\"][1],quarter(tempDF[applied_filters[index][\"parameters\"][0]]))\n",
    "                query+= \"tempDF = tempDF.withColumn(\\\"\"+applied_filters[index][\"parameters\"][1]+\"\\\",quarter(tempDF[\\\"\"+applied_filters[index][\"parameters\"][0]+\"\\\"]))\\n\"\n",
    "            elif filter_type==\"month\":\n",
    "                applied_filters[index][\"parameters\"][0] = \"`\"+applied_filters[index][\"parameters\"][0]+\"`\"\n",
    "                applied_filters[index][\"parameters\"][1] = applied_filters[index][\"parameters\"][1].replace(\" \",\"_\")\n",
    "                tempDF = tempDF.withColumn(applied_filters[index][\"parameters\"][1],month(tempDF[applied_filters[index][\"parameters\"][0]]))\n",
    "                query+= \"tempDF = tempDF.withColumn(\\\"\"+applied_filters[index][\"parameters\"][1]+\"\\\",month(tempDF[\\\"\"+applied_filters[index][\"parameters\"][0]+\"\\\"]))\\n\"\n",
    "            elif filter_type==\"hour\":\n",
    "                applied_filters[index][\"parameters\"][0] = \"`\"+applied_filters[index][\"parameters\"][0]+\"`\"\n",
    "                applied_filters[index][\"parameters\"][1] = applied_filters[index][\"parameters\"][1].replace(\" \",\"_\")\n",
    "                tempDF = tempDF.withColumn(applied_filters[index][\"parameters\"][1],hour(tempDF[applied_filters[index][\"parameters\"][0]]))\n",
    "                query+= \"tempDF = tempDF.withColumn(\\\"\"+applied_filters[index][\"parameters\"][1]+\"\\\",hour(tempDF[\\\"\"+applied_filters[index][\"parameters\"][0]+\"\\\"]))\\n\"\n",
    "            elif filter_type==\"minute\":\n",
    "                applied_filters[index][\"parameters\"][0] = \"`\"+applied_filters[index][\"parameters\"][0]+\"`\"\n",
    "                applied_filters[index][\"parameters\"][1] = applied_filters[index][\"parameters\"][1].replace(\" \",\"_\")\n",
    "                tempDF = tempDF.withColumn(applied_filters[index][\"parameters\"][1],minute(tempDF[applied_filters[index][\"parameters\"][0]]))\n",
    "                query+= \"tempDF = tempDF.withColumn(\\\"\"+applied_filters[index][\"parameters\"][1]+\"\\\",minute(tempDF[\\\"\"+applied_filters[index][\"parameters\"][0]+\"\\\"]))\\n\"\n",
    "            elif filter_type==\"dayofmonth\":\n",
    "                applied_filters[index][\"parameters\"][0] = \"`\"+applied_filters[index][\"parameters\"][0]+\"`\"\n",
    "                applied_filters[index][\"parameters\"][1] = applied_filters[index][\"parameters\"][1].replace(\" \",\"_\")\n",
    "                tempDF = tempDF.withColumn(applied_filters[index][\"parameters\"][1],dayofmonth(tempDF[applied_filters[index][\"parameters\"][0]]))\n",
    "                query+= \"tempDF = tempDF.withColumn(\\\"\"+applied_filters[index][\"parameters\"][1]+\"\\\",dayofmonth(tempDF[\\\"\"+applied_filters[index][\"parameters\"][0]+\"\\\"]))\\n\"\n",
    "            elif filter_type==\"dayofyear\":\n",
    "                applied_filters[index][\"parameters\"][0] = \"`\"+applied_filters[index][\"parameters\"][0]+\"`\"\n",
    "                applied_filters[index][\"parameters\"][1] = applied_filters[index][\"parameters\"][1].replace(\" \",\"_\")\n",
    "                tempDF = tempDF.withColumn(applied_filters[index][\"parameters\"][1],dayofyear(tempDF[applied_filters[index][\"parameters\"][0]]))\n",
    "                query+= \"tempDF = tempDF.withColumn(\\\"\"+applied_filters[index][\"parameters\"][1]+\"\\\",dayofyear(tempDF[\\\"\"+applied_filters[index][\"parameters\"][0]+\"\\\"]))\\n\"\n",
    "            elif filter_type==\"to_date\":\n",
    "                applied_filters[index][\"parameters\"][0] = \"`\"+applied_filters[index][\"parameters\"][0]+\"`\"\n",
    "                applied_filters[index][\"parameters\"][1] = applied_filters[index][\"parameters\"][1].replace(\" \",\"_\")\n",
    "                tempDF = tempDF.withColumn(applied_filters[index][\"parameters\"][1],to_date(tempDF[applied_filters[index][\"parameters\"][0]]))\n",
    "                query+= \"tempDF = tempDF.withColumn(\\\"\"+applied_filters[index][\"parameters\"][1]+\"\\\",to_date(tempDF[\\\"\"+applied_filters[index][\"parameters\"][0]+\"\\\"]))\\n\"\n",
    "            elif filter_type==\"unix_timestamp\":\n",
    "                applied_filters[index][\"parameters\"][0] = \"`\"+applied_filters[index][\"parameters\"][0]+\"`\"\n",
    "                applied_filters[index][\"parameters\"][1] = applied_filters[index][\"parameters\"][1].replace(\" \",\"_\")\n",
    "                tempDF = tempDF.withColumn(applied_filters[index][\"parameters\"][1],unix_timestamp(tempDF[applied_filters[index][\"parameters\"][0]]))\n",
    "                query+= \"tempDF = tempDF.withColumn(\\\"\"+applied_filters[index][\"parameters\"][1]+\"\\\",to_timestamp(tempDF[\\\"\"+applied_filters[index][\"parameters\"][0]+\"\\\"]))\\n\"\n",
    "            elif filter_type==\"ytrunc\":\n",
    "                applied_filters[index][\"parameters\"][0] = \"`\"+applied_filters[index][\"parameters\"][0]+\"`\"\n",
    "                applied_filters[index][\"parameters\"][1] = applied_filters[index][\"parameters\"][1].replace(\" \",\"_\")\n",
    "                tempDF = tempDF.withColumn(applied_filters[index][\"parameters\"][1],trunc(tempDF[applied_filters[index][\"parameters\"][0]],\"year\"))\n",
    "                query+= \"tempDF = tempDF.withColumn(\\\"\"+applied_filters[index][\"parameters\"][1]+\"\\\",trunc(tempDF[\\\"\"+applied_filters[index][\"parameters\"][0]+\"\\\"],\\\"year\\\"))\\n\"\n",
    "            elif filter_type==\"mtrunc\":\n",
    "                applied_filters[index][\"parameters\"][0] = \"`\"+applied_filters[index][\"parameters\"][0]+\"`\"\n",
    "                applied_filters[index][\"parameters\"][1] = applied_filters[index][\"parameters\"][1].replace(\" \",\"_\")\n",
    "                tempDF = tempDF.withColumn(applied_filters[index][\"parameters\"][1],trunc(tempDF[applied_filters[index][\"parameters\"][0]],\"month\"))\n",
    "                query+= \"tempDF = tempDF.withColumn(\\\"\"+applied_filters[index][\"parameters\"][1]+\"\\\",trunc(tempDF[\\\"\"+applied_filters[index][\"parameters\"][0]+\"\\\"],\\\"month\\\"))\\n\"\n",
    "            elif filter_type==\"sqrt\":\n",
    "                applied_filters[index][\"parameters\"][0] = \"`\"+applied_filters[index][\"parameters\"][0]+\"`\"\n",
    "                applied_filters[index][\"parameters\"][1] = applied_filters[index][\"parameters\"][1].replace(\" \",\"_\")\n",
    "                tempDF = tempDF.withColumn(applied_filters[index][\"parameters\"][1],sqrt(tempDF[applied_filters[index][\"parameters\"][0]]))\n",
    "                query+= \"tempDF = tempDF.withColumn(\\\"\"+applied_filters[index][\"parameters\"][1]+\"\\\",sqrt(tempDF[\\\"\"+applied_filters[index][\"parameters\"][0]+\"\\\"]))\\n\"\n",
    "            elif filter_type==\"round\":\n",
    "                applied_filters[index][\"parameters\"][0] = \"`\"+applied_filters[index][\"parameters\"][0]+\"`\"\n",
    "                applied_filters[index][\"parameters\"][1] = int(applied_filters[index][\"parameters\"][1])\n",
    "                applied_filters[index][\"parameters\"][2] = applied_filters[index][\"parameters\"][2].replace(\" \",\"_\")\n",
    "                tempDF = tempDF.withColumn(applied_filters[index][\"parameters\"][2],pyround(tempDF[applied_filters[index][\"parameters\"][0]],applied_filters[index][\"parameters\"][1]))\n",
    "                query += \"tempDF = tempDF.withColumn(\\\"\"+applied_filters[index][\"parameters\"][2]+\"\\\",pyround(tempDF[\\\"\"+applied_filters[index][\"parameters\"][0]+\"\\\"],\"+str(applied_filters[index][\"parameters\"][1])+\"))\\n\"\n",
    "            elif filter_type==\"when_1\":\n",
    "                then_value = None\n",
    "                else_value = None\n",
    "                new_col_name = applied_filters[index][\"parameters\"][5].replace(\" \",\"_\")\n",
    "                applied_filters[index][\"parameters\"][0] = \"tempDF['`\"+applied_filters[index][\"parameters\"][0]+\"`']\"\n",
    "                applied_filters[index][\"parameters\"][2] = \"'\"+applied_filters[index][\"parameters\"][2]+\"'\"\n",
    "                condition = \" \".join(applied_filters[index][\"parameters\"][0:3])\n",
    "                evaluated_condition = eval(condition)\n",
    "                try:\n",
    "                    then_value = float(applied_filters[index][\"parameters\"][3])\n",
    "                except ValueError as ver:\n",
    "                    then_value = applied_filters[index][\"parameters\"][3]\n",
    "                try:\n",
    "                    else_value = float(applied_filters[index][\"parameters\"][4])\n",
    "                except ValueError as ver:\n",
    "                    else_value = applied_filters[index][\"parameters\"][4]\n",
    "                tempDF = tempDF.withColumn(new_col_name,when(evaluated_condition,then_value).otherwise(else_value))\n",
    "                query+=\"tempDF = tempDF.withColumn(\\\"\"+new_col_name+\"\\\",when(\"+condition+\",\\\"\"+str(then_value)+\"\\\").otherwise(\\\"\"+str(else_value)+\"\\\"))\\n\"\n",
    "            elif filter_type==\"when_2\":\n",
    "                new_col_name = applied_filters[index][\"parameters\"][9].replace(\" \",\"_\")\n",
    "                applied_filters[index][\"parameters\"][0] = \"tempDF['`\"+applied_filters[index][\"parameters\"][0]+\"`']\"\n",
    "                applied_filters[index][\"parameters\"][2] = \"'\"+applied_filters[index][\"parameters\"][2]+\"'\"\n",
    "                condition = \" \".join(applied_filters[index][\"parameters\"][0:3])\n",
    "                evaluated_condition = eval(condition)\n",
    "                applied_filters[index][\"parameters\"][3] = \"tempDF['`\"+applied_filters[index][\"parameters\"][3]+\"`']\"\n",
    "                applied_filters[index][\"parameters\"][5] = \"'\"+applied_filters[index][\"parameters\"][5]+\"'\"\n",
    "                then_condition = \" \".join(applied_filters[index][\"parameters\"][3:6])\n",
    "                evaluated_then_condition = eval(then_condition)\n",
    "                applied_filters[index][\"parameters\"][6] = \"tempDF['`\"+applied_filters[index][\"parameters\"][6]+\"`']\"\n",
    "                applied_filters[index][\"parameters\"][8] = \"'\"+applied_filters[index][\"parameters\"][8]+\"'\"\n",
    "                else_condition = \" \".join(applied_filters[index][\"parameters\"][6:9])\n",
    "                evaluated_else_condition = eval(else_condition)\n",
    "                tempDF = tempDF.withColumn(new_col_name,when(evaluated_condition,evaluated_then_condition).otherwise(evaluated_else_condition))\n",
    "                query+= \"tempDF = tempDF.withColumn(\\\"\"+new_col_name+\"\\\",when(\"+condition+\",\"+then_condition+\").otherwise(\"+else_condition+\"))\\n\"\n",
    "            elif filter_type==\"when_3\":\n",
    "                new_col_name = applied_filters[index][\"parameters\"][9].replace(\" \",\"_\")\n",
    "                applied_filters[index][\"parameters\"][0] = \"tempDF['`\"+applied_filters[index][\"parameters\"][0]+\"`']\"\n",
    "                applied_filters[index][\"parameters\"][2] = \"'\"+applied_filters[index][\"parameters\"][2]+\"'\"\n",
    "                condition = \" \".join(applied_filters[index][\"parameters\"][0:3])\n",
    "                evaluated_condition = eval(condition)\n",
    "                applied_filters[index][\"parameters\"][3] = \"tempDF['`\"+applied_filters[index][\"parameters\"][3]+\"`']\"\n",
    "                applied_filters[index][\"parameters\"][5] = \"tempDF['`\"+applied_filters[index][\"parameters\"][5]+\"`']\"\n",
    "                then_condition = \" \".join(applied_filters[index][\"parameters\"][3:6])\n",
    "                evaluated_then_condition = eval(then_condition)\n",
    "                applied_filters[index][\"parameters\"][6] = \"tempDF['`\"+applied_filters[index][\"parameters\"][6]+\"`']\"\n",
    "                applied_filters[index][\"parameters\"][8] = \"tempDF['`\"+applied_filters[index][\"parameters\"][8]+\"`']\"\n",
    "                else_condition = \" \".join(applied_filters[index][\"parameters\"][6:9])\n",
    "                evaluated_else_condition = eval(else_condition)\n",
    "                tempDF = tempDF.withColumn(new_col_name,when(evaluated_condition,evaluated_then_condition).otherwise(evaluated_else_condition))\n",
    "                query+= \"tempDF = tempDF.withColumn(\\\"\"+new_col_name+\"\\\",when(\"+condition+\",\"+then_condition+\").otherwise(\"+else_condition+\"))\\n\"\n",
    "            elif filter_type==\"col_operation_1\":\n",
    "                new_col_name = applied_filters[index][\"parameters\"][3].replace(\" \",\"_\")\n",
    "                applied_filters[index][\"parameters\"][0] = \"tempDF['`\"+applied_filters[index][\"parameters\"][0]+\"`']\"\n",
    "                applied_filters[index][\"parameters\"][2] = \"'\"+applied_filters[index][\"parameters\"][2]+\"'\"\n",
    "                expr = \" \".join(applied_filters[index][\"parameters\"][0:3])\n",
    "                evaluated_expr = eval(expr)\n",
    "                tempDF = tempDF.withColumn(new_col_name,evaluated_expr)\n",
    "                query+=\"tempDF = tempDF.withColumn(\\\"\"+new_col_name+\"\\\",\"+expr+\")\\n\"\n",
    "            elif filter_type==\"col_operation_2\":\n",
    "                new_col_name = applied_filters[index][\"parameters\"][3].replace(\" \",\"_\")\n",
    "                applied_filters[index][\"parameters\"][0] = \"tempDF['`\"+applied_filters[index][\"parameters\"][0]+\"`']\"\n",
    "                applied_filters[index][\"parameters\"][2] = \"tempDF['`\"+applied_filters[index][\"parameters\"][2]+\"`']\"\n",
    "                expr = \" \".join(applied_filters[index][\"parameters\"][0:3])\n",
    "                evaluated_expr = eval(expr)\n",
    "                tempDF = tempDF.withColumn(new_col_name,evaluated_expr)\n",
    "                query+=\"tempDF = tempDF.withColumn(\\\"\"+new_col_name+\"\\\",\"+expr+\")\\n\"\n",
    "            elif filter_type==\"datediff\":\n",
    "                applied_filters[index][\"parameters\"][0] = \"`\"+applied_filters[index][\"parameters\"][0]+\"`\"\n",
    "                applied_filters[index][\"parameters\"][1] = \"`\"+applied_filters[index][\"parameters\"][1]+\"`\"\n",
    "                applied_filters[index][\"parameters\"][2] = applied_filters[index][\"parameters\"][2].replace(\" \",\"_\")\n",
    "                tempDF = tempDF.withColumn(applied_filters[index][\"parameters\"][2],datediff(tempDF[applied_filters[index][\"parameters\"][0]],tempDF[applied_filters[index][\"parameters\"][1]]))\n",
    "                query+=\"tempDF = tempDF.withColumn(\\\"\"+applied_filters[index][\"parameters\"][2]+\"\\\",datediff(tempDF[\\\"\"+applied_filters[index][\"parameters\"][0]+\"\\\"],tempDF[\\\"\"+applied_filters[index][\"parameters\"][1]+\"\\\"]))\\n\"\n",
    "            elif filter_type==\"months_between\":\n",
    "                applied_filters[index][\"parameters\"][0] = \"`\"+applied_filters[index][\"parameters\"][0]+\"`\"\n",
    "                applied_filters[index][\"parameters\"][1] = \"`\"+applied_filters[index][\"parameters\"][1]+\"`\"\n",
    "                applied_filters[index][\"parameters\"][2] = applied_filters[index][\"parameters\"][2].replace(\" \",\"_\")\n",
    "                tempDF = tempDF.withColumn(applied_filters[index][\"parameters\"][2],months_between(tempDF[applied_filters[index][\"parameters\"][0]],tempDF[applied_filters[index][\"parameters\"][1]]))\n",
    "                query+=\"tempDF = tempDF.withColumn(\\\"\"+applied_filters[index][\"parameters\"][2]+\"\\\",months_between(tempDF[\\\"\"+applied_filters[index][\"parameters\"][0]+\"\\\"],tempDF[\\\"\"+applied_filters[index][\"parameters\"][1]+\"\\\"]))\\n\"\n",
    "            elif filter_type==\"orderBy\":\n",
    "                paramsList = [x for x in applied_filters[index][\"parameters\"]]\n",
    "                l = [\"`\"+x+\"`\" for x in list(paramsList[0])]\n",
    "                tempDF = tempDF.orderBy(l,ascending = list(paramsList[1]))\n",
    "                query+=\"tempDF = tempDF.orderBy(\\\"\"+l[0]+\"\\\",ascending = \"+str(list(paramsList[1]))+\")\\n\"\n",
    "            elif filter_type==\"drop\":\n",
    "                tempDF = tempDF.drop(*applied_filters[index][\"parameters\"][0])\n",
    "                query+=\"tempDF = tempDF.drop(*\"+str(applied_filters[index][\"parameters\"][0])+\")\\n\"\n",
    "            elif filter_type==\"withColumnRenamed\":\n",
    "                tempDF = tempDF.withColumnRenamed(applied_filters[index][\"parameters\"][0],applied_filters[index][\"parameters\"][1])\n",
    "                query+=\"tempDF = tempDF.withColumnRenamed(\\\"\"+applied_filters[index][\"parameters\"][0]+\"\\\",\\\"\"+applied_filters[index][\"parameters\"][1]+\"\\\")\\n\"\n",
    "            elif filter_type==\"select\":\n",
    "                l = [\"`\"+x+\"`\" for x in applied_filters[index][\"parameters\"][0]]\n",
    "                tempDF = tempDF.select(*l)\n",
    "                query+=\"tempDF = tempDF.select(*\"+str(l)+\")\\n\"\n",
    "            elif filter_type==\"dropDuplicates\":\n",
    "                tempDF = tempDF.dropDuplicates()\n",
    "                query+=\"tempDF = tempDF.dropDuplicates()\\n\"\n",
    "            elif filter_type==\"fillna\":\n",
    "                fillDict = dict(applied_filters[index][\"parameters\"][0])\n",
    "                tempDF = tempDF.fillna(fillDict)\n",
    "                query+=\"tempDF = tempDF.fillna(\"+str(fillDict)+\")\\n\"\n",
    "            elif filter_type==\"replace\":\n",
    "                paramsList = [x for x in applied_filters[index][\"parameters\"]]\n",
    "                tempDF = tempDF.replace(paramsList[0],paramsList[1],str(paramsList[2]))\n",
    "                query+=\"tempDF = tempDF.replace(list(\"+str(paramsList[0])+\"),list(\"+str(paramsList[1])+\"),(\\\"\"+str(paramsList[2])+\"\\\"))\\n\"\n",
    "            elif filter_type==\"min\":\n",
    "                paramsList = [x for x in applied_filters[index][\"parameters\"]]\n",
    "                colsp = list(paramsList[1])\n",
    "                cols = [\"`\"+x+\"`\" for x in colsp]\n",
    "                gby = [\"`\"+y+\"`\" for y in list(paramsList[0])]\n",
    "                tempDF = tempDF.groupBy(gby).min(*cols)\n",
    "                query+=\"tempDF = tempDF.groupBy(\\\"\"+gby[0]+\"\\\").min(*\"+str(cols)+\")\\n\"\n",
    "            elif filter_type==\"max\":\n",
    "                paramsList = [x for x in applied_filters[index][\"parameters\"]]\n",
    "                colsp = list(paramsList[1])\n",
    "                cols = [\"`\"+x+\"`\" for x in colsp]\n",
    "                gby = [\"`\"+y+\"`\" for y in list(paramsList[0])]\n",
    "                tempDF = tempDF.groupBy(gby).max(*cols)\n",
    "                query+=\"tempDF = tempDF.groupBy(\\\"\"+gby[0]+\"\\\").max(*\"+str(cols)+\")\\n\"\n",
    "            elif filter_type==\"avg\":\n",
    "                paramsList = [x for x in applied_filters[index][\"parameters\"]]\n",
    "                colsp = list(paramsList[1])\n",
    "                cols = [\"`\"+x+\"`\" for x in colsp]\n",
    "                gby = [\"`\"+y+\"`\" for y in list(paramsList[0])]\n",
    "                tempDF = tempDF.groupBy(gby).avg(*cols)\n",
    "                query+=\"tempDF = tempDF.groupBy(\\\"\"+gby[0]+\"\\\").avg(*\"+str(cols)+\")\\n\"\n",
    "            elif filter_type==\"sum\":\n",
    "                paramsList = [x for x in applied_filters[index][\"parameters\"]]\n",
    "                colsp = list(paramsList[1])\n",
    "                cols = [\"`\"+x+\"`\" for x in colsp]\n",
    "                gby = [\"`\"+y+\"`\" for y in list(paramsList[0])]\n",
    "                tempDF = tempDF.groupBy(gby).sum(*cols)\n",
    "                query+=\"tempDF = tempDF.groupBy(\\\"\"+gby[0]+\"\\\").sum(*\"+str(cols)+\")\\n\"\n",
    "            elif filter_type==\"count\":\n",
    "                paramsList = [x for x in applied_filters[index][\"parameters\"]]\n",
    "                l = [\"`\"+x+\"`\" for x in paramsList[0]]\n",
    "                tempDF = tempDF.groupBy(list(l)).count()\n",
    "                query+=\"tempDF.groupBy(\\\"\"+list(l)[0]+\"\\\").count()\\n\"\n",
    "            elif filter_type==\"isnan\":\n",
    "                cols = \"`\"+applied_filters[index][\"parameters\"][0]+\"`\"\n",
    "                tempDF = tempDF.withColumn(applied_filters[index][\"parameters\"][0]+\"_isnan\",isnan(tempDF[cols]))\n",
    "                query+= \"tempDF = tempDF.withColumn(\\\"\"+applied_filters[index][\"parameters\"][0]+\"_isnan\\\",isnan(tempDF[\\\"\"+cols+\"\\\"]))\\n\"\n",
    "            elif filter_type==\"isnull\":\n",
    "                cols = \"`\"+applied_filters[index][\"parameters\"][0]+\"`\"\n",
    "                tempDF = tempDF.withColumn(applied_filters[index][\"parameters\"][0]+\"_isnull\",isnull(tempDF[cols]))\n",
    "                query+= \"tempDF = tempDF.withColumn(\\\"\"+applied_filters[index][\"parameters\"][0]+\"_isnull\\\",isnull(tempDF[\\\"\"+cols+\"\\\"]))\\n\"\n",
    "            elif filter_type==\"ltrim\":\n",
    "                cols = \"`\"+applied_filters[index][\"parameters\"][0]+\"`\"\n",
    "                tempDF = tempDF.withColumn(applied_filters[index][\"parameters\"][0]+\"_ltrimmed\",ltrim(tempDF[cols]))\n",
    "                query+= \"tempDF = tempDF.withColumn(\\\"\"+applied_filters[index][\"parameters\"][0]+\"_ltrimmed\\\",ltrim(tempDF[\\\"\"+cols+\"\\\"]))\\n\"\n",
    "            elif filter_type==\"rtrim\":\n",
    "                cols = \"`\"+applied_filters[index][\"parameters\"][0]+\"`\"\n",
    "                tempDF = tempDF.withColumn(applied_filters[index][\"parameters\"][0]+\"_rtrimmed\",rtrim(tempDF[cols]))\n",
    "                query+= \"tempDF = tempDF.withColumn(\\\"\"+applied_filters[index][\"parameters\"][0]+\"_rtrimmed\\\",rtrim(tempDF[\\\"\"+cols+\"\\\"]))\\n\"\n",
    "            elif filter_type==\"joinApproxTime\":\n",
    "                paramsList = [x for x in applied_filters[index][\"parameters\"]]\n",
    "                keep = [x for x in paramsList[1:-1] if (None not in x)]\n",
    "                jhow = str(paramsList[0])\n",
    "                time_columns = paramsList[-1]\n",
    "                allcols = tempDF.columns\n",
    "                allcols.extend(masterDF.columns)\n",
    "                if len(keep) == 1:\n",
    "                    joinedDF = masterDF.alias('masterDF').join(tempDF.alias('tempDF'),on=[col('masterDF.'+keep[0][0])==col('tempDF.'+keep[0][1])],how=jhow)\n",
    "                    additional_col = joinedDF.withColumn(\"time_diff\",  cabs(unix_timestamp(str(time_columns[0])) - unix_timestamp(str(time_columns[1]))))\n",
    "                    partDF = additional_col.select(F.row_number().over(Window.partitionBy(str(keep[0][0])).orderBy(\"time_diff\")).alias(\"rank\"), \"time_diff\",*allcols ).filter(col(\"rank\") == 1)\n",
    "                    tempDF = partDF\n",
    "                elif len(keep) == 2:\n",
    "                    joinedDF = masterDF.alias('masterDF').join(tempDF.alias('tempDF'),on=[col('masterDF.'+keep[0][0])==col('tempDF.'+keep[0][1]),col('masterDF.'+keep[1][0])==col('tempDF.'+keep[1][1])],how=jhow)\n",
    "                    additional_col = joinedDF.withColumn(\"time_diff\",  cabs(unix_timestamp(str(time_columns[0])) - unix_timestamp(str(time_columns[1]))))\n",
    "                    partDf = additional_col.select( F.row_number().over(Window.partitionBy(str(keep[0][0]),str(keep[1][0])).orderBy(\"time_diff\")).alias(\"rank\"), *allcols ).filter(col(\"rank\") == 1).drop('rank')\n",
    "                    tempDF = partDF\n",
    "                elif len(keep) == 3:\n",
    "                    joinedDF = masterDF.alias('masterDF').join(tempDF.alias('tempDF'),on=[col('masterDF.'+keep[0][0])==col('tempDF.'+keep[0][1]),col('masterDF.'+keep[1][0])==col('tempDF.'+keep[1][1]),col('masterDF.'+keep[2][0])==col('tempDF.'+keep[2][1])],how=jhow)\n",
    "                    additional_col = joinedDF.withColumn(\"time_diff\",  cabs(unix_timestamp(str(time_columns[0])) - unix_timestamp(str(time_columns[1]))))\n",
    "                    partDf = additional_col.select( F.row_number().over(Window.partitionBy(str(keep[0][0]),str(keep[1][0]),str(keep[2][0])).orderBy(\"time_diff\")).alias(\"rank\"), *allcols).filter(col(\"rank\") == 1).drop('rank')\n",
    "                    tempDF = partDF\n",
    "            filters_success.append(index)\n",
    "        except Exception as e:\n",
    "            g = e\n",
    "            return\n",
    "    return filters_success\n",
    "\n",
    "def read_dataset(selectedFile, separator, encoding):\n",
    "    global tempDF\n",
    "    global query\n",
    "    global g\n",
    "    if selectedFile.startswith('/'):\n",
    "        selectedFile = \"hdfs://\" + myhdfs.host + \":\" + str(myhdfs.port) + selectedFile\n",
    "    try:\n",
    "        if encoding:\n",
    "            tempDF = spark.read.csv(selectedFile, header=True, inferSchema=True, sep=separator, encoding=encoding)\n",
    "        else:\n",
    "            tempDF = spark.read.csv(selectedFile, header=True, inferSchema=True, sep=separator)\n",
    "    except Exception as e:\n",
    "        g=e\n",
    "    query=query_init\n",
    "    return tempDF.columns\n",
    "\n",
    "def getTempColumns():\n",
    "    global tempDF\n",
    "    return tempDF.columns\n",
    "\n",
    "def tempDFtoMasterDF():\n",
    "    global masterDF\n",
    "    global tempDF\n",
    "    masterDF = tempDF\n",
    "    return masterDF.columns\n",
    "\n",
    "def saveMasterAsCSV(savepath, withheader):\n",
    "    global masterDF\n",
    "    global hdfspath\n",
    "    global g\n",
    "    hdfs_start_path = \"hdfs://\"+myhdfs.host+\":\"+str(myhdfs.port)\n",
    "    completepath = hdfs_start_path + savepath\n",
    "    if withheader == 1:\n",
    "        headerflag = True\n",
    "    else:\n",
    "        headerflag = False\n",
    "    try:\n",
    "        masterDF.coalesce(1).write.csv(path=completepath, mode='append', header=headerflag)\n",
    "        return 200\n",
    "    except Exception as e:\n",
    "        g=e\n",
    "\n",
    "def getCSVfiles():\n",
    "    fileSet = set()\n",
    "    for f in myhdfs.walk(hdfs.project_path()):\n",
    "        fileSet.add(f['name'])\n",
    "    available_files = [x for x in list(fileSet) if x.endswith(\".csv\")]\n",
    "    return available_files\n",
    "    \n",
    "def find_current_projectID(pDict):\n",
    "    projectsDict = json.loads(pDict)\n",
    "    pname = hdfs.project_name()\n",
    "    return projectsDict[pname] #KeyError should not be possible here, so no try-except...\n",
    "    \n",
    "def walk_dataset(dataset_path):\n",
    "    fileSet = set()\n",
    "    hdfs_start_path = \"hdfs://\"+myhdfs.host+\":\"+str(myhdfs.port)\n",
    "    for f in myhdfs.walk(hdfs_start_path + dataset_path):\n",
    "        fileSet.add(f['name'])\n",
    "    available_files = [x for x in list(fileSet) if x.endswith(\".csv\")]\n",
    "    return available_files\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hideCode": true,
    "hideOutput": false,
    "hidePrompt": false
   },
   "outputs": [],
   "source": [
    "%%html\n",
    "<!-- P2 -->\n",
    "<div class=\"aegis\">\n",
    "    <div id=\"cbrowser\">\n",
    "        <div class=\"form-group row form-horizontal\">\n",
    "            <label class=\"col-sm-2 control-label\">File:</label>\n",
    "            <div class=\"col-sm-10\">\n",
    "                <div class=\"input-group widget-dropdown\">\n",
    "                    <select id=\"fileBrowser\" class=\"form-control\">\n",
    "                    </select>\n",
    "                    <span class=\"input-group-btn\">\n",
    "                        <button type=\"button\" style=\"display:none\" class=\"btn jupyter-button mod-primary\" onclick=\"fileBrowserOpen()\">\n",
    "                            Open\n",
    "                        </button>\n",
    "                        <button type=\"button\" class=\"btn jupyter-button mod-primary\" onclick=\"forceFileBrowserRefresh()\">\n",
    "                            Refresh list\n",
    "                        </button>\n",
    "                    </span>\n",
    "                </div>\n",
    "            </div>\n",
    "        </div>\n",
    "        <div class=\"form-group row form-horizontal\" style=\"display:none\">\n",
    "            <label class=\"col-sm-2 control-label\">Available Datasets:</label>\n",
    "            <div class=\"col-sm-10\">\n",
    "                <div class=\"input-group widget-dropdown\">\n",
    "                    <select id=\"datasetSelect\" class=\"form-control\">\n",
    "                    </select>\n",
    "                    <span class=\"input-group-btn\">\n",
    "                        <button type=\"button\" class=\"btn jupyter-button mod-primary\" onclick=\"selectDataset()\"> Open\n",
    "                            dataset\n",
    "                        </button>\n",
    "                        <button type=\"button\" class=\"btn jupyter-button mod-primary\" onclick=\"showCurrentBrowser()\">\n",
    "                            Refresh list\n",
    "                        </button>\n",
    "                    </span>\n",
    "                </div>\n",
    "            </div>\n",
    "        </div>\n",
    "        <div class=\"form-group row form-horizontal\" style=\"display:none\">\n",
    "            <label class=\"col-sm-2 control-label\">Available Files:</label>\n",
    "            <div class=\"col-sm-10\">\n",
    "                <div class=\"widget-dropdown\">\n",
    "                    <select id=\"fileSelect\" class=\"form-control\">\n",
    "                    </select>\n",
    "                </div>\n",
    "            </div>\n",
    "        </div>\n",
    "        <div class=\"form-group row form-horizontal\">\n",
    "            <label class=\"col-sm-2 control-label\">CSV Separator:</label>\n",
    "            <div class=\"col-sm-4\">\n",
    "                <div class=\"widget-dropdown\">\n",
    "                    <select id=\"sepSelect\" class=\"form-control\">\n",
    "                        <option value=\",\">Comma (,)</option>\n",
    "                        <option value=\";\">Semicolon (;)</option>\n",
    "                    </select>\n",
    "                </div>\n",
    "            </div>\n",
    "            <label class=\"col-sm-2 control-label\">Encoding:</label>\n",
    "            <div class=\"col-sm-4\">\n",
    "                <div class=\"widget-dropdown\">\n",
    "                    <select id=\"encSelect\" class=\"form-control\">\n",
    "                        <option value=\"\">Auto</option>\n",
    "                        <option value=\"ascii\">ASCII</option>\n",
    "                        <option value=\"utf_8\">UTF-8</option>\n",
    "                        <option value=\"utf_8_sig\">UTF-8 with BOM</option>\n",
    "                        <option value=\"cp1252\">Windows-1252</option>\n",
    "                        <option value=\"cp1253\">Windows-1253</option>\n",
    "                    </select>\n",
    "                </div>\n",
    "            </div>\n",
    "        </div>\n",
    "        <div class=\"form-group row form-horizontal\">\n",
    "            <div class=\"col-sm-offset-2 col-sm-10\">\n",
    "                <button type=\"button\" class=\"jupyter-button jupyter-widgets mod-primary widget-button\" style=\"display:none\"\n",
    "                    onclick=\"selectDatafile()\"> Open file </button>\n",
    "                <button id=\"fileBrowserOpenBtn\" type=\"button\" class=\"btn jupyter-button mod-primary\" onclick=\"fileBrowserOpen()\">\n",
    "                    Open\n",
    "                </button>\n",
    "            </div>\n",
    "        </div>\n",
    "    </div>\n",
    "    <hr>\n",
    "\n",
    "    <div id=\"filters\">\n",
    "        <div class=\"form-group row form-horizontal\" id=\"null-filterType\" style=\"display:none;\">\n",
    "            <label class=\"col-sm-2 control-label\">Available actions:</label>\n",
    "            <div class=\"col-sm-10\">\n",
    "                <div class=\"widget-dropdown\">\n",
    "                    <select id=\"filterType\" class=\"form-control\" id=\"filterType\" onchange=\"divVisibilityChanged()\">\n",
    "                        <option value=\"\"></option>\n",
    "                        <option disabled>-------- Basic --------</option>\n",
    "                        <option value=\"filter\">Filter</option>\n",
    "                        <option value=\"filterOr\">Filter OR</option>\n",
    "                        <option value=\"orderBy\">Order By</option>\n",
    "                        <option value=\"select\">Select column(s)</option>\n",
    "                        <option value=\"drop\">Drop column(s)</option>\n",
    "                        <option value=\"withColumnRenamed\">Rename Column</option>\n",
    "                        <option disabled>------ Aggregations ------</option>\n",
    "                        <option value=\"count\">Count</option>\n",
    "                        <option value=\"min\">Min</option>\n",
    "                        <option value=\"max\">Max</option>\n",
    "                        <option value=\"avg\">Average</option>\n",
    "                        <option value=\"sum\">Sum</option>\n",
    "                        <option disabled>------- Cleansing -------</option>\n",
    "                        <option value=\"fillna\">Fill N/A with value</option>\n",
    "                        <option value=\"isnan\">Check if NaN</option>\n",
    "                        <option value=\"isnull\">Check if null</option>\n",
    "                        <option value=\"replace\">Replace value</option>\n",
    "                        <option value=\"ltrim\">Left Trim a Column</option>\n",
    "                        <option value=\"rtrim\">Right Trim a Column</option>\n",
    "                        <option disabled value=\"linear_interp\">Fill N/A with linear interpolation</option>\n",
    "                        <option disabled>--------- Joins ---------</option>\n",
    "                        <option value=\"join\">Join with master</option>\n",
    "                        <option value=\"joinApproxTime\">Join with master on approximate time</option>\n",
    "                        <option disabled>------- Time Conversions ------</option>\n",
    "                        <option value=\"year\">Add Year Colum</option>\n",
    "                        <option value=\"quarter\">Add Quarter Column</option>\n",
    "                        <option value=\"month\">Add Month Column</option>\n",
    "                        <option value=\"hour\">Add Hour Column</option>\n",
    "                        <option value=\"minute\">Add Minute Column</option>\n",
    "                        <option value=\"to_date\">Convert to date</option>\n",
    "                        <option value=\"unix_timestamp\">Convert to UNIX timestamp</option>\n",
    "                        <option value=\"ytrunc\">Truncate date to year</option>\n",
    "                        <option value=\"mtrunc\">Truncate date to month</option>\n",
    "                        <option value=\"dayofyear\">Add dayOfYear column</option>\n",
    "                        <option value=\"dayofmonth\">Add dayOfMonth column</option>\n",
    "                        <option value=\"datediff\">Number of days between dates</option>\n",
    "                        <option value=\"months_between\">Number of months between dates</option>\n",
    "                        <option disabled>--------- Math ---------</option>\n",
    "                        <option value=\"sqrt\">Add Square Root Column</option>\n",
    "                        <option value=\"round\">Add Round Column</option>\n",
    "                        <option disabled>--------- Other ---------</option>\n",
    "                        <option value=\"col_operation_1\">Add column - based on other column </option>\n",
    "                        <option value=\"col_operation_2\">Add column - combine two columns </option>\n",
    "                        <option value=\"when_1\">Add column - column-based condition</option>\n",
    "                        <option value=\"when_2\">Add column - column-based condition & value</option>\n",
    "                        <option value=\"when_3\">Add column - column-based condition & columns-combination value</option>\n",
    "                        <option disabled value=\"replace_extract\">Replace based on regexp</option>\n",
    "                    </select>\n",
    "                </div>\n",
    "            </div>\n",
    "        </div>\n",
    "\n",
    "        <div id=\"filterType_filter\" style=\"display:none\">\n",
    "            <h3>Filter</h3>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Column:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <div class=\"widget-dropdown\">\n",
    "                        <select class=\"form-control single-column-selector temp-column\">\n",
    "                        </select>\n",
    "                    </div>\n",
    "                </div>\n",
    "            </div>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Operator:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <div class=\"widget-dropdown\">\n",
    "                        <select class=\"form-control operator-selector\">\n",
    "                            <option></option>\n",
    "                            <option value=\"<\">&lt;</option>\n",
    "                            <option value=\"==\">=</option>\n",
    "                            <option value=\">\">&gt;</option>\n",
    "                            <option value=\"!=\">!=</option>\n",
    "                            <option value=\"like\">starts with</option>\n",
    "                        </select>\n",
    "                    </div>\n",
    "                </div>\n",
    "            </div>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Value:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <input class=\"form-control user-defined-value\" />\n",
    "                </div>\n",
    "            </div>\n",
    "        </div>\n",
    "\n",
    "        <div id=\"filterType_filterOr\" style=\"display:none\">\n",
    "                <h3>Filter OR</h3>\n",
    "                <div class=\"form-group row form-horizontal\">\n",
    "                    <label class=\"col-sm-2 control-label\">Column 1:</label>\n",
    "                    <div class=\"col-sm-10\">\n",
    "                        <div class=\"widget-dropdown\">\n",
    "                            <select class=\"form-control single-column-selector temp-column col1_colname\">\n",
    "                            </select>\n",
    "                        </div>\n",
    "                    </div>\n",
    "                </div>\n",
    "                <div class=\"form-group row form-horizontal\">\n",
    "                    <label class=\"col-sm-2 control-label\">Operator:</label>\n",
    "                    <div class=\"col-sm-10\">\n",
    "                        <div class=\"widget-dropdown\">\n",
    "                            <select class=\"form-control operator-selector col1_operator\">\n",
    "                                <option></option>\n",
    "                                <option value=\"<\">&lt;</option>\n",
    "                                <option value=\"==\">=</option>\n",
    "                                <option value=\">\">&gt;</option>\n",
    "                                <option value=\"!=\">!=</option>\n",
    "                                <option value=\"like\">starts with</option>\n",
    "                            </select>\n",
    "                        </div>\n",
    "                    </div>\n",
    "                </div>\n",
    "                <div class=\"form-group row form-horizontal\">\n",
    "                    <label class=\"col-sm-2 control-label\">Value:</label>\n",
    "                    <div class=\"col-sm-10\">\n",
    "                        <input class=\"form-control user-defined-value col1_value\" />\n",
    "                    </div>\n",
    "                </div>\n",
    "                <div class=\"form-group row form-horizontal\">\n",
    "                        <label class=\"col-sm-2 control-label\">Column 2:</label>\n",
    "                        <div class=\"col-sm-10\">\n",
    "                            <div class=\"widget-dropdown\">\n",
    "                                <select class=\"form-control single-column-selector temp-column col2_colname\">\n",
    "                                </select>\n",
    "                            </div>\n",
    "                        </div>\n",
    "                    </div>\n",
    "                    <div class=\"form-group row form-horizontal\">\n",
    "                        <label class=\"col-sm-2 control-label\">Operator:</label>\n",
    "                        <div class=\"col-sm-10\">\n",
    "                            <div class=\"widget-dropdown\">\n",
    "                                <select class=\"form-control operator-selector col2_operator\">\n",
    "                                    <option></option>\n",
    "                                    <option value=\"<\">&lt;</option>\n",
    "                                    <option value=\"==\">=</option>\n",
    "                                    <option value=\">\">&gt;</option>\n",
    "                                    <option value=\"!=\">!=</option>\n",
    "                                    <option value=\"like\">starts with</option>\n",
    "                                </select>\n",
    "                            </div>\n",
    "                        </div>\n",
    "                    </div>\n",
    "                    <div class=\"form-group row form-horizontal\">\n",
    "                        <label class=\"col-sm-2 control-label\">Value:</label>\n",
    "                        <div class=\"col-sm-10\">\n",
    "                            <input class=\"form-control user-defined-value col2_value\" />\n",
    "                        </div>\n",
    "                    </div>\n",
    "            </div>\n",
    "\n",
    "        <div id=\"filterType_col_operation_1\" style=\"display:none\">\n",
    "            <h3>Create column from an existing</h3>\n",
    "            <h4>Compute new value <span>-only numeric operations supported</span></h4>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Column:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <div class=\"widget-dropdown\">\n",
    "                        <select class=\"form-control single-column-selector temp-column\">\n",
    "                        </select>\n",
    "                    </div>\n",
    "                </div>\n",
    "            </div>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Operator:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <div class=\"widget-dropdown\">\n",
    "                        <select class=\"form-control operator-selector\">\n",
    "                            <option></option>\n",
    "                            <option value=\"+\">+</option>\n",
    "                            <option value=\"-\">-</option>\n",
    "                            <option value=\"*\">*</option>\n",
    "                            <option value=\"/\">/</option>\n",
    "                        </select>\n",
    "                    </div>\n",
    "                </div>\n",
    "            </div>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Value:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <input class=\"form-control user-defined-value operation-value\" />\n",
    "                </div>\n",
    "            </div>\n",
    "            <h4>New Column Name</h4>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Name:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <input class=\"form-control user-defined-value new-col-name\" />\n",
    "                </div>\n",
    "            </div>\n",
    "        </div>\n",
    "\n",
    "        <div class=\"form-horizontal\" id=\"filterType_col_operation_2\" style=\"display:none\">\n",
    "            <h3>Create column from two existing columns</h3>\n",
    "            <h4>Compute new value <span>-only numeric operations supported</span></h4>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Column:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <select class=\"form-control single-column-selector temp-column temp-column-A\">\n",
    "                    </select>\n",
    "                </div>\n",
    "            </div>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Operator:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <div class=\"widget-dropdown\">\n",
    "                        <select class=\"form-control operator-selector\">\n",
    "                            <option></option>\n",
    "                            <option value=\"+\">+</option>\n",
    "                            <option value=\"-\">-</option>\n",
    "                            <option value=\"*\">*</option>\n",
    "                            <option value=\"/\">/</option>\n",
    "                        </select>\n",
    "                    </div>\n",
    "                </div>\n",
    "            </div>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Column:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <select class=\"form-control single-column-selector temp-column temp-column-B\">\n",
    "                    </select>\n",
    "                </div>\n",
    "            </div>\n",
    "            <h4>New Column Name</h4>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Name:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <input class=\"form-control user-defined-value new-col-name\" />\n",
    "                </div>\n",
    "            </div>\n",
    "        </div>\n",
    "\n",
    "        <div class=\"form-horizontal\" id=\"filterType_when_1\" style=\"display:none\">\n",
    "            <h3>When-Then-Else</h3>\n",
    "            <h4>Condition (When)</h4>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Column:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <div class=\"widget-dropdown\">\n",
    "                        <select class=\"form-control single-column-selector temp-column when-div\">\n",
    "                        </select>\n",
    "                    </div>\n",
    "                </div>\n",
    "            </div>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Operator:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <div class=\"widget-dropdown\">\n",
    "                        <select class=\"form-control operator-selector when-div\">\n",
    "                            <option></option>\n",
    "                            <option value=\"<\">&lt;</option>\n",
    "                            <option value=\"==\">=</option>\n",
    "                            <option value=\">\">&gt;</option>\n",
    "                            <option value=\"!=\">!=</option>\n",
    "                        </select>\n",
    "                    </div>\n",
    "                </div>\n",
    "            </div>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Value:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <input class=\"form-control user-defined-value when-div\" />\n",
    "                </div>\n",
    "            </div>\n",
    "            <h4>Value (Then)</h4>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Value:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <input class=\"form-control user-defined-value then-div\" />\n",
    "                </div>\n",
    "            </div>\n",
    "            <h4>Alternative Value (Else)</h4>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Alt Value:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <input class=\"form-control user-defined-value else-div\" />\n",
    "                </div>\n",
    "            </div>\n",
    "            <h4>New Column Name</h4>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Name:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <input class=\"form-control user-defined-value new-col-name\" />\n",
    "                </div>\n",
    "            </div>\n",
    "        </div>\n",
    "\n",
    "        <div class=\"form-horizontal\" id=\"filterType_when_2\" style=\"display:none\">\n",
    "            <h3>When-Then-Else</h3>\n",
    "            <h4>Condition (When)</h4>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Column:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <div class=\"widget-dropdown\">\n",
    "                        <select class=\"form-control single-column-selector temp-column when-div temp-column-A\">\n",
    "                        </select>\n",
    "                    </div>\n",
    "                </div>\n",
    "            </div>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Operator:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <div class=\"widget-dropdown\">\n",
    "                        <select class=\"form-control operator-selector when-div\">\n",
    "                            <option></option>\n",
    "                            <option value=\"<\">&lt;</option>\n",
    "                            <option value=\"==\">=</option>\n",
    "                            <option value=\">\">&gt;</option>\n",
    "                            <option value=\"!=\">!=</option>\n",
    "                        </select>\n",
    "                    </div>\n",
    "                </div>\n",
    "            </div>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Value:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <input class=\"form-control user-defined-value when-div\" />\n",
    "                </div>\n",
    "            </div>\n",
    "            <h4>Value (Then) <span>-only numeric operations supported</span></h4>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Column:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <div class=\"widget-dropdown\">\n",
    "                        <select class=\"form-control single-column-selector temp-column then-div temp-column-A\">\n",
    "                        </select>\n",
    "                    </div>\n",
    "                </div>\n",
    "            </div>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Operator:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <div class=\"widget-dropdown\">\n",
    "                        <select class=\"form-control operator-selector then-div\">\n",
    "                            <option></option>\n",
    "                            <option value=\"+\">+</option>\n",
    "                            <option value=\"-\">-</option>\n",
    "                            <option value=\"*\">*</option>\n",
    "                            <option value=\"/\">/</option>\n",
    "                        </select>\n",
    "                    </div>\n",
    "                </div>\n",
    "            </div>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Value:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <input class=\"form-control user-defined-value then-div\" />\n",
    "                </div>\n",
    "            </div>\n",
    "            <h4>Alternative Value (Else) <span>-only numeric operations supported</span></h4>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Column:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <div class=\"widget-dropdown\">\n",
    "                        <select class=\"form-control single-column-selector temp-column else-div temp-column-A\">\n",
    "                        </select>\n",
    "                    </div>\n",
    "                </div>\n",
    "            </div>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Operator:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <div class=\"widget-dropdown\">\n",
    "                        <select class=\"form-control operator-selector else-div\">\n",
    "                            <option></option>\n",
    "                            <option value=\"+\">+</option>\n",
    "                            <option value=\"-\">-</option>\n",
    "                            <option value=\"*\">*</option>\n",
    "                            <option value=\"/\">/</option>\n",
    "                        </select>\n",
    "                    </div>\n",
    "                </div>\n",
    "            </div>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Alt Value:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <input class=\"form-control user-defined-value else-div\" />\n",
    "                </div>\n",
    "            </div>\n",
    "            <h4>New Column Name</h4>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Name:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <input class=\"form-control user-defined-value new-col-name\" />\n",
    "                </div>\n",
    "            </div>\n",
    "        </div>\n",
    "\n",
    "        <div class=\"form-horizontal\" id=\"filterType_when_3\" style=\"display:none\">\n",
    "            <h3>When-Then-Else</h3>\n",
    "            <h4>Condition (When)</h4>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Column:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <div class=\"widget-dropdown\">\n",
    "                        <select class=\"form-control single-column-selector temp-column when-div\">\n",
    "                        </select>\n",
    "                    </div>\n",
    "                </div>\n",
    "            </div>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Operator:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <div class=\"widget-dropdown\">\n",
    "                        <select class=\"form-control operator-selector when-div\">\n",
    "                            <option></option>\n",
    "                            <option value=\"<\">&lt;</option>\n",
    "                            <option value=\"==\">=</option>\n",
    "                            <option value=\">\">&gt;</option>\n",
    "                            <option value=\"!=\">!=</option>\n",
    "                        </select>\n",
    "                    </div>\n",
    "                </div>\n",
    "            </div>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Value:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <input class=\"form-control user-defined-value when-div\" />\n",
    "                </div>\n",
    "            </div>\n",
    "            <h4>Value (Then) <span>-only numeric operations supported</span></h4>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Column:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <div class=\"widget-dropdown\">\n",
    "                        <select class=\"form-control single-column-selector temp-column then-div temp-column-A\">\n",
    "                        </select>\n",
    "                    </div>\n",
    "                </div>\n",
    "            </div>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Operator:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <div class=\"widget-dropdown\">\n",
    "                        <select class=\"form-control operator-selector then-div\">\n",
    "                            <option></option>\n",
    "                            <option value=\"+\">+</option>\n",
    "                            <option value=\"-\">-</option>\n",
    "                            <option value=\"*\">*</option>\n",
    "                            <option value=\"/\">/</option>\n",
    "                        </select>\n",
    "                    </div>\n",
    "                </div>\n",
    "            </div>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Column:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <div class=\"widget-dropdown\">\n",
    "                        <select class=\"form-control single-column-selector temp-column then-div temp-column-B\">\n",
    "                        </select>\n",
    "                    </div>\n",
    "                </div>\n",
    "            </div>\n",
    "            <h4>Alternative Value (Else) <span>-only numeric operations supported</span></h4>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Column:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <div class=\"widget-dropdown\">\n",
    "                        <select class=\"form-control single-column-selector temp-column else-div temp-column-A\">\n",
    "                        </select>\n",
    "                    </div>\n",
    "                </div>\n",
    "            </div>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Operator:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <div class=\"widget-dropdown\">\n",
    "                        <select class=\"form-control operator-selector else-div\">\n",
    "                            <option></option>\n",
    "                            <option value=\"+\">+</option>\n",
    "                            <option value=\"-\">-</option>\n",
    "                            <option value=\"*\">*</option>\n",
    "                            <option value=\"/\">/</option>\n",
    "                        </select>\n",
    "                    </div>\n",
    "                </div>\n",
    "            </div>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Column:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <div class=\"widget-dropdown\">\n",
    "                        <select class=\"form-control single-column-selector temp-column else-div temp-column-B\">\n",
    "                        </select>\n",
    "                    </div>\n",
    "                </div>\n",
    "            </div>\n",
    "            <h4>New Column Name</h4>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Name:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <input class=\"form-control user-defined-value new-col-name\" />\n",
    "                </div>\n",
    "            </div>\n",
    "        </div>\n",
    "\n",
    "        <div class=\"form-horizontal\" id=\"filterType_orderBy\" style=\"display:none\">\n",
    "            <h3>Order By</h3>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Column:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <div class=\"widget-dropdown\">\n",
    "                        <select class=\"form-control single-column-selector temp-column\">\n",
    "                        </select>\n",
    "                    </div>\n",
    "                </div>\n",
    "            </div>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Sort:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <div class=\"widget-dropdown\">\n",
    "                        <select class=\"form-control operator-selector\">\n",
    "                            <option value=\"1\">Asc</option>\n",
    "                            <option value=\"0\">Desc</option>\n",
    "                        </select>\n",
    "                    </div>\n",
    "                </div>\n",
    "            </div>\n",
    "        </div>\n",
    "\n",
    "        <div class=\"form-horizontal\" id=\"filterType_count\" style=\"display:none\">\n",
    "            <h3>Count</h3>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Group By:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <div class=\"widget-dropdown\">\n",
    "                        <select class=\"form-control single-column-selector temp-column\">\n",
    "                        </select>\n",
    "                    </div>\n",
    "                </div>\n",
    "            </div>\n",
    "        </div>\n",
    "\n",
    "        <div class=\"form-horizontal\" id=\"filterType_ltrim\" style=\"display:none\">\n",
    "            <h3>Left Trim</h3>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Column:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <div class=\"widget-dropdown\">\n",
    "                        <select class=\"form-control single-column-selector temp-column\">\n",
    "                        </select>\n",
    "                    </div>\n",
    "                </div>\n",
    "            </div>\n",
    "        </div>\n",
    "\n",
    "        <div class=\"form-horizontal\" id=\"filterType_rtrim\" style=\"display:none\">\n",
    "            <h3>Right Trim</h3>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Column:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <div class=\"widget-dropdown\">\n",
    "                        <select class=\"form-control single-column-selector temp-column\">\n",
    "                        </select>\n",
    "                    </div>\n",
    "                </div>\n",
    "            </div>\n",
    "        </div>\n",
    "\n",
    "        <div class=\"form-horizontal\" id=\"filterType_isnan\" style=\"display:none\">\n",
    "            <h3>Is NaN</h3>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Column:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <div class=\"widget-dropdown\">\n",
    "                        <select class=\"form-control single-column-selector temp-column\">\n",
    "                        </select>\n",
    "                    </div>\n",
    "                </div>\n",
    "            </div>\n",
    "        </div>\n",
    "\n",
    "        <div class=\"form-horizontal\" id=\"filterType_isnull\" style=\"display:none\">\n",
    "            <h3>Is Null</h3>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Column:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <div class=\"widget-dropdown\">\n",
    "                        <select class=\"form-control single-column-selector temp-column\">\n",
    "                        </select>\n",
    "                    </div>\n",
    "                </div>\n",
    "            </div>\n",
    "        </div>\n",
    "\n",
    "        <div class=\"form-horizontal\" id=\"filterType_min\" style=\"display:none\">\n",
    "            <h3>Min</h3>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Columns:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <div class=\"widget-dropdown\">\n",
    "                        <select multiple class=\"form-control multiple-column-selector temp-column\">\n",
    "                        </select>\n",
    "                    </div>\n",
    "                </div>\n",
    "            </div>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Group By:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <div class=\"widget-dropdown\">\n",
    "                        <select class=\"form-control single-column-selector temp-column\">\n",
    "                        </select>\n",
    "                    </div>\n",
    "                </div>\n",
    "            </div>\n",
    "\n",
    "        </div>\n",
    "\n",
    "        <div class=\"form-horizontal\" id=\"filterType_max\" style=\"display:none\">\n",
    "            <h3>Max</h3>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Columns:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <div class=\"widget-dropdown\">\n",
    "                        <select multiple class=\"form-control multiple-column-selector temp-column\">\n",
    "                        </select>\n",
    "                    </div>\n",
    "                </div>\n",
    "            </div>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Group By:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <div class=\"widget-dropdown\">\n",
    "                        <select class=\"form-control single-column-selector temp-column\">\n",
    "                        </select>\n",
    "                    </div>\n",
    "                </div>\n",
    "            </div>\n",
    "        </div>\n",
    "\n",
    "        <div class=\"form-horizontal\" id=\"filterType_avg\" style=\"display:none\">\n",
    "            <h3>Average</h3>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Columns:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <div class=\"widget-dropdown\">\n",
    "                        <select multiple class=\"form-control multiple-column-selector temp-column\">\n",
    "                        </select>\n",
    "                    </div>\n",
    "                </div>\n",
    "            </div>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Group By:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <div class=\"widget-dropdown\">\n",
    "                        <select class=\"form-control single-column-selector temp-column\">\n",
    "                        </select>\n",
    "                    </div>\n",
    "                </div>\n",
    "            </div>\n",
    "        </div>\n",
    "\n",
    "        <div class=\"form-horizontal\" id=\"filterType_sum\" style=\"display:none\">\n",
    "            <h3>Sum</h3>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Columns:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <div class=\"widget-dropdown\">\n",
    "                        <select multiple class=\"form-control multiple-column-selector temp-column\">\n",
    "                        </select>\n",
    "                    </div>\n",
    "                </div>\n",
    "            </div>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Group By:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <div class=\"widget-dropdown\">\n",
    "                        <select class=\"form-control single-column-selector temp-column\">\n",
    "                        </select>\n",
    "                    </div>\n",
    "                </div>\n",
    "            </div>\n",
    "        </div>\n",
    "\n",
    "        <div class=\"form-horizontal\" id=\"filterType_year\" style=\"display:none\">\n",
    "            <h3>Create Year Column</h3>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Date Column:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <div class=\"widget-dropdown\">\n",
    "                        <select class=\"form-control single-column-selector temp-column\">\n",
    "                        </select>\n",
    "                    </div>\n",
    "                </div>\n",
    "            </div>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Name of New Column:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <input class=\"form-control user-defined-value\" />\n",
    "                </div>\n",
    "            </div>\n",
    "        </div>\n",
    "\n",
    "        <div class=\"form-horizontal\" id=\"filterType_quarter\" style=\"display:none\">\n",
    "            <h3>Create Quarter Column</h3>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Date Column:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <div class=\"widget-dropdown\">\n",
    "                        <select class=\"form-control single-column-selector temp-column\">\n",
    "                        </select>\n",
    "                    </div>\n",
    "                </div>\n",
    "            </div>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Name of New Column:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <input class=\"form-control user-defined-value\" />\n",
    "                </div>\n",
    "            </div>\n",
    "        </div>\n",
    "\n",
    "        <div class=\"form-horizontal\" id=\"filterType_month\" style=\"display:none\">\n",
    "            <h3>Create Month Column</h3>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Date Column:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <div class=\"widget-dropdown\">\n",
    "                        <select class=\"form-control single-column-selector temp-column\">\n",
    "                        </select>\n",
    "                    </div>\n",
    "                </div>\n",
    "            </div>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Name of New Column:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <input class=\"form-control user-defined-value\" />\n",
    "                </div>\n",
    "            </div>\n",
    "        </div>\n",
    "\n",
    "        <div class=\"form-horizontal\" id=\"filterType_hour\" style=\"display:none\">\n",
    "            <h3>Create Hour Column</h3>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Date Column:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <div class=\"widget-dropdown\">\n",
    "                        <select class=\"form-control single-column-selector temp-column\">\n",
    "                        </select>\n",
    "                    </div>\n",
    "                </div>\n",
    "            </div>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Name of New Column:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <input class=\"form-control user-defined-value\" />\n",
    "                </div>\n",
    "            </div>\n",
    "        </div>\n",
    "\n",
    "        <div class=\"form-horizontal\" id=\"filterType_minute\" style=\"display:none\">\n",
    "            <h3>Create Minute Column</h3>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Date Column:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <div class=\"widget-dropdown\">\n",
    "                        <select class=\"form-control single-column-selector temp-column\">\n",
    "                        </select>\n",
    "                    </div>\n",
    "                </div>\n",
    "            </div>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Name of New Column:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <input class=\"form-control user-defined-value\" />\n",
    "                </div>\n",
    "            </div>\n",
    "        </div>\n",
    "\n",
    "        <div class=\"form-horizontal\" id=\"filterType_dayofmonth\" style=\"display:none\">\n",
    "            <h3>Create Day of Month Column</h3>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Date Column:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <div class=\"widget-dropdown\">\n",
    "                        <select class=\"form-control single-column-selector temp-column\">\n",
    "                        </select>\n",
    "                    </div>\n",
    "                </div>\n",
    "            </div>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Name of New Column:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <input class=\"form-control user-defined-value\" />\n",
    "                </div>\n",
    "            </div>\n",
    "        </div>\n",
    "\n",
    "        <div class=\"form-horizontal\" id=\"filterType_dayofyear\" style=\"display:none\">\n",
    "            <h3>Create Day of Year Column</h3>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Date Column:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <div class=\"widget-dropdown\">\n",
    "                        <select class=\"form-control single-column-selector temp-column\">\n",
    "                        </select>\n",
    "                    </div>\n",
    "                </div>\n",
    "            </div>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Name of New Column:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <input class=\"form-control user-defined-value\" />\n",
    "                </div>\n",
    "            </div>\n",
    "        </div>\n",
    "\n",
    "        <div class=\"form-horizontal\" id=\"filterType_to_date\" style=\"display:none\">\n",
    "            <h3>Create Date Column</h3>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Date Column:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <div class=\"widget-dropdown\">\n",
    "                        <select class=\"form-control single-column-selector temp-column\">\n",
    "                        </select>\n",
    "                    </div>\n",
    "                </div>\n",
    "            </div>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Name of New Column:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <input class=\"form-control user-defined-value\" />\n",
    "                </div>\n",
    "            </div>\n",
    "        </div>\n",
    "\n",
    "        <div class=\"form-horizontal\" id=\"filterType_unix_timestamp\" style=\"display:none\">\n",
    "            <h3>Create UNIX Timestamp Column</h3>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Date Column:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <div class=\"widget-dropdown\">\n",
    "                        <select class=\"form-control single-column-selector temp-column\">\n",
    "                        </select>\n",
    "                    </div>\n",
    "                </div>\n",
    "            </div>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Name of New Column:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <input class=\"form-control user-defined-value\" />\n",
    "                </div>\n",
    "            </div>\n",
    "        </div>\n",
    "\n",
    "        <div class=\"form-horizontal\" id=\"filterType_ytrunc\" style=\"display:none\">\n",
    "            <h3>Truncate Date Column to Year</h3>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Date Column:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <div class=\"widget-dropdown\">\n",
    "                        <select class=\"form-control single-column-selector temp-column\">\n",
    "                        </select>\n",
    "                    </div>\n",
    "                </div>\n",
    "            </div>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Name of New Column:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <input class=\"form-control user-defined-value\" />\n",
    "                </div>\n",
    "            </div>\n",
    "        </div>\n",
    "\n",
    "        <div class=\"form-horizontal\" id=\"filterType_mtrunc\" style=\"display:none\">\n",
    "            <h3>Truncate Date Column to Month</h3>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Date Column:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <div class=\"widget-dropdown\">\n",
    "                        <select class=\"form-control single-column-selector temp-column\">\n",
    "                        </select>\n",
    "                    </div>\n",
    "                </div>\n",
    "            </div>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Name of New Column:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <input class=\"form-control user-defined-value\" />\n",
    "                </div>\n",
    "            </div>\n",
    "        </div>\n",
    "\n",
    "        <div class=\"form-horizontal\" id=\"filterType_datediff\" style=\"display:none\">\n",
    "            <h3>Create DateDiff Column</h3>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Date Column A:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <div class=\"widget-dropdown\">\n",
    "                        <select class=\"form-control single-column-selector temp-column temp-column-A\">\n",
    "                        </select>\n",
    "                    </div>\n",
    "                </div>\n",
    "            </div>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Date Column B:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <div class=\"widget-dropdown\">\n",
    "                        <select class=\"form-control single-column-selector temp-column temp-column-B\">\n",
    "                        </select>\n",
    "                    </div>\n",
    "                </div>\n",
    "            </div>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Name of New Column:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <input class=\"form-control user-defined-value\" />\n",
    "                </div>\n",
    "            </div>\n",
    "        </div>\n",
    "\n",
    "        <div class=\"form-horizontal\" id=\"filterType_months_between\" style=\"display:none\">\n",
    "            <h3>Create MonthsDiff Column</h3>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Date Column A:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <div class=\"widget-dropdown\">\n",
    "                        <select class=\"form-control single-column-selector temp-column temp-column-A\">\n",
    "                        </select>\n",
    "                    </div>\n",
    "                </div>\n",
    "            </div>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Date Column B:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <div class=\"widget-dropdown\">\n",
    "                        <select class=\"form-control single-column-selector temp-column temp-column-B\">\n",
    "                        </select>\n",
    "                    </div>\n",
    "                </div>\n",
    "            </div>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Name of New Column:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <input class=\"form-control user-defined-value\" />\n",
    "                </div>\n",
    "            </div>\n",
    "        </div>\n",
    "\n",
    "        <div class=\"form-horizontal\" id=\"filterType_sqrt\" style=\"display:none\">\n",
    "            <h3>Create SQRT Column</h3>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Select Column:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <div class=\"widget-dropdown\">\n",
    "                        <select class=\"form-control single-column-selector temp-column\">\n",
    "                        </select>\n",
    "                    </div>\n",
    "                </div>\n",
    "            </div>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Name of New Column:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <input class=\"form-control user-defined-value\" />\n",
    "                </div>\n",
    "            </div>\n",
    "        </div>\n",
    "\n",
    "        <div class=\"form-horizontal\" id=\"filterType_round\" style=\"display:none\">\n",
    "            <h3>Create Rounded Column</h3>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Select Column:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <div class=\"widget-dropdown\">\n",
    "                        <select class=\"form-control single-column-selector temp-column\">\n",
    "                        </select>\n",
    "                    </div>\n",
    "                </div>\n",
    "            </div>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Name of New Column:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <input class=\"form-control user-defined-value col-name\" />\n",
    "                </div>\n",
    "            </div>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Number of Decimal Places:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <input type=\"number\" step=\"1\" class=\"form-control user-defined-value int-input\" />\n",
    "                </div>\n",
    "            </div>\n",
    "        </div>\n",
    "\n",
    "        <div class=\"form-horizontal\" id=\"filterType_select\" style=\"display:none\">\n",
    "            <h3>Select Columns</h3>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Columns:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <div class=\"widget-dropdown\">\n",
    "                        <select multiple class=\"form-control multiple-column-selector temp-column\">\n",
    "                        </select>\n",
    "                    </div>\n",
    "                </div>\n",
    "            </div>\n",
    "        </div>\n",
    "\n",
    "        <div class=\"form-horizontal\" id=\"filterType_drop\" style=\"display:none\">\n",
    "            <h3>Select Columns</h3>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Columns:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <div class=\"widget-dropdown\">\n",
    "                        <select multiple class=\"form-control multiple-column-selector temp-column\">\n",
    "                        </select>\n",
    "                    </div>\n",
    "                </div>\n",
    "            </div>\n",
    "        </div>\n",
    "\n",
    "        <div class=\"form-horizontal\" id=\"filterType_fillna\" style=\"display:none\">\n",
    "            <h3>Fill N/A</h3>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Column:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <div class=\"widget-dropdown\">\n",
    "                        <select class=\"form-control single-column-selector temp-column\">\n",
    "                        </select>\n",
    "                    </div>\n",
    "                </div>\n",
    "            </div>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Value:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <input class=\"form-control user-defined-value\" />\n",
    "                </div>\n",
    "            </div>\n",
    "        </div>\n",
    "\n",
    "        <div class=\"form-horizontal\" id=\"filterType_replace\" style=\"display:none\">\n",
    "            <h3>Replace Value</h3>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Column:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <div class=\"widget-dropdown\">\n",
    "                        <select class=\"form-control single-column-selector temp-column\">\n",
    "                        </select>\n",
    "                    </div>\n",
    "                </div>\n",
    "            </div>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Current Value:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <input class=\"form-control user-defined-value existing-value\" />\n",
    "                </div>\n",
    "            </div>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">To-Be Value:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <input class=\"form-control user-defined-value to-be-value\" />\n",
    "                </div>\n",
    "            </div>\n",
    "        </div>\n",
    "\n",
    "        <div class=\"form-horizontal\" id=\"filterType_withColumnRenamed\" style=\"display:none\">\n",
    "            <h3>Rename Column</h3>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Column:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <div class=\"widget-dropdown\">\n",
    "                        <select class=\"form-control single-column-selector temp-column\">\n",
    "                        </select>\n",
    "                    </div>\n",
    "                </div>\n",
    "            </div>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">New Name:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <input class=\"form-control user-defined-value\" />\n",
    "                </div>\n",
    "            </div>\n",
    "        </div>\n",
    "\n",
    "        <div class=\"form-horizontal\" id=\"filterType_join\" style=\"display:none\">\n",
    "            <h3>Join</h3>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Join Type:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <div class=\"widget-dropdown\">\n",
    "                        <select class=\"form-control operator-selector\">\n",
    "                            <option value=\"inner\">inner</option>\n",
    "                            <option value=\"outer\">outer</option>\n",
    "                            <option value=\"left_outer\">left_outer</option>\n",
    "                            <option value=\"right_outer\">right_outer</option>\n",
    "                        </select>\n",
    "                    </div>\n",
    "                </div>\n",
    "            </div>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Master Column 1:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <div class=\"widget-dropdown\">\n",
    "                        <select class=\"form-control single-column-selector master-column master-col-1\">\n",
    "                        </select>\n",
    "                    </div>\n",
    "                </div>\n",
    "            </div>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Temp Column 1:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <div class=\"widget-dropdown\">\n",
    "                        <select class=\"form-control single-column-selector temp-col-1 temp-column empty-option-allowed\">\n",
    "                        </select>\n",
    "                    </div>\n",
    "                </div>\n",
    "            </div>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Master Column 2 (opt) :</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <div class=\"widget-dropdown\">\n",
    "                        <select class=\"form-control single-column-selector master-column master-col-2\">\n",
    "                        </select>\n",
    "                    </div>\n",
    "                </div>\n",
    "            </div>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Temp Column 2 (opt) :</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <div class=\"widget-dropdown\">\n",
    "                        <select class=\"form-control single-column-selector temp-col-2 temp-column empty-option-allowed\">\n",
    "                        </select>\n",
    "                    </div>\n",
    "                </div>\n",
    "            </div>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Master Column 3 (opt) :</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <div class=\"widget-dropdown\">\n",
    "                        <select class=\"form-control single-column-selector master-column master-col-3\">\n",
    "                        </select>\n",
    "                    </div>\n",
    "                </div>\n",
    "            </div>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Temp Column 3 (opt) :</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <div class=\"widget-dropdown\">\n",
    "                        <select class=\"form-control single-column-selector temp-col-3 temp-column empty-option-allowed\">\n",
    "                        </select>\n",
    "                    </div>\n",
    "                </div>\n",
    "            </div>\n",
    "        </div>\n",
    "\n",
    "        <div class=\"form-horizontal\" id=\"filterType_joinApproxTime\" style=\"display:none\">\n",
    "            <h3>Join on closest time</h3>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Join Type:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <div class=\"widget-dropdown\">\n",
    "                        <select class=\"form-control operator-selector\">\n",
    "                            <option value=\"inner\">inner</option>\n",
    "                            <option value=\"outer\">outer</option>\n",
    "                            <option value=\"left_outer\">left_outer</option>\n",
    "                            <option value=\"right_outer\">right_outer</option>\n",
    "                        </select>\n",
    "                    </div>\n",
    "                </div>\n",
    "            </div>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Master Time Column:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <div class=\"widget-dropdown\">\n",
    "                        <select class=\"form-control single-column-selector master-column master-time-col\">\n",
    "                        </select>\n",
    "                    </div>\n",
    "                </div>\n",
    "            </div>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Temp Time Column:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <div class=\"widget-dropdown\">\n",
    "                        <select class=\"form-control single-column-selector temp-time-col temp-column empty-option-allowed\">\n",
    "                        </select>\n",
    "                    </div>\n",
    "                </div>\n",
    "            </div>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Master Column 1:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <div class=\"widget-dropdown\">\n",
    "                        <select class=\"form-control single-column-selector master-column master-col-1\">\n",
    "                        </select>\n",
    "                    </div>\n",
    "                </div>\n",
    "            </div>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Temp Column 1:</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <div class=\"widget-dropdown\">\n",
    "                        <select class=\"form-control single-column-selector temp-col-1 temp-column empty-option-allowed\">\n",
    "                        </select>\n",
    "                    </div>\n",
    "                </div>\n",
    "            </div>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Master Column 2 (opt) :</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <div class=\"widget-dropdown\">\n",
    "                        <select class=\"form-control single-column-selector master-column master-col-2\">\n",
    "                        </select>\n",
    "                    </div>\n",
    "                </div>\n",
    "            </div>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Temp Column 2 (opt) :</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <div class=\"widget-dropdown\">\n",
    "                        <select class=\"form-control single-column-selector temp-col-2 temp-column empty-option-allowed\">\n",
    "                        </select>\n",
    "                    </div>\n",
    "                </div>\n",
    "            </div>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Master Column 3 (opt) :</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <div class=\"widget-dropdown\">\n",
    "                        <select class=\"form-control single-column-selector master-column master-col-3\">\n",
    "                        </select>\n",
    "                    </div>\n",
    "                </div>\n",
    "            </div>\n",
    "            <div class=\"form-group row form-horizontal\">\n",
    "                <label class=\"col-sm-2 control-label\">Temp Column 3 (opt) :</label>\n",
    "                <div class=\"col-sm-10\">\n",
    "                    <div class=\"widget-dropdown\">\n",
    "                        <select class=\"form-control single-column-selector temp-col-3 temp-column empty-option-allowed\">\n",
    "                        </select>\n",
    "                    </div>\n",
    "                </div>\n",
    "            </div>\n",
    "        </div>\n",
    "    </div>\n",
    "    <div id=\"filter-control-buttons\" style=\"display:none\">\n",
    "        <div class=\"form-group row form-horizontal\">\n",
    "            <div class=\"col-sm-offset-2 col-sm-10\">\n",
    "                <button class=\"jupyter-button jupyter-widgets mod-primary widget-button\" onclick=\"addFilter();\"> OK\n",
    "                </button>\n",
    "                <button class=\"jupyter-button jupyter-widgets widget-button\" onclick=\"cancelFilter()\"> Cancel </button>\n",
    "            </div>\n",
    "        </div>\n",
    "    </div>\n",
    "\n",
    "    <div id=\"list2\">\n",
    "        <ol id=\"show-filters\"></ol>\n",
    "    </div>\n",
    "\n",
    "    <div id=\"filter-application-buttons\" style=\"display:none\">\n",
    "        <hr>\n",
    "        <div class=\"form-group row form-horizontal\">\n",
    "            <button type=\"button\" class=\"jupyter-button jupyter-widgets mod-primary widget-button\"\n",
    "                onclick=\"applyAllFilters();\"> Apply Filters </button>\n",
    "            <button type=\"button\" class=\"jupyter-button jupyter-widgets mod-primary widget-button\"\n",
    "                onclick=\"Jupyter.notebook.execute_cells([4]);\"> Refresh\n",
    "                temp </button>\n",
    "            <button type=\"button\" class=\"jupyter-button jupyter-widgets mod-warning widget-button\"\n",
    "                onclick=\"tempToMaster();\">Save temp as master</button>\n",
    "            <button type=\"button\" class=\"jupyter-button jupyter-widgets mod-primary widget-button\"\n",
    "                onclick=\"Jupyter.notebook.execute_cells([6]);\"> Refresh\n",
    "                master </button>\n",
    "            <button type=\"button\" class=\"jupyter-button jupyter-widgets mod-primary widget-button\"\n",
    "                onclick=\"document.getElementById('save_conf').style.display = 'block';\">Save master to CSV</button>\n",
    "            <button type=\"button\" class=\"jupyter-button jupyter-widgets mod-primary widget-button\"\n",
    "                onclick=\"Jupyter.notebook.execute_cells([8]);\"> Update\n",
    "                Query Output </button>\n",
    "        </div>\n",
    "    </div>\n",
    "\n",
    "    <div id=\"save_conf\" style=\"display:none\">\n",
    "        <hr>\n",
    "        <div class=\"form-group row form-horizontal\">\n",
    "            <h3>Save Configuration</h3>\n",
    "        </div>\n",
    "        <div class=\"form-group row form-horizontal\">\n",
    "            <label class=\"col-sm-2 control-label\">Select Dataset:</label>\n",
    "            <div class=\"col-sm-10\">\n",
    "                <div class=\"widget-dropdown\">\n",
    "                    <select id=\"datasetSaveSelect\" class=\"form-control\">\n",
    "                    </select>\n",
    "                </div>\n",
    "            </div>\n",
    "        </div>\n",
    "        <div class=\"form-group row form-horizontal\">\n",
    "            <label class=\"col-sm-2 control-label\">Result Folder Name:</label>\n",
    "            <div class=\"col-sm-10\">\n",
    "                <input id=\"folderNameSelect\" class=\"form-control\"></input>\n",
    "            </div>\n",
    "        </div>\n",
    "        <div class=\"form-group row form-horizontal\">\n",
    "            <label class=\"col-sm-2 control-label\">Include Header:</label>\n",
    "            <div class=\"col-sm-10\">\n",
    "                <input type=\"checkbox\" id=\"saveHeaderCheckbox\" value=\"with\" checked>\n",
    "                </input>\n",
    "            </div>\n",
    "        </div>\n",
    "        <div class=\"form-group row form-horizontal\">\n",
    "            <div class=\"col-sm-offset-2 col-sm-10\">\n",
    "                <button type=\"button\" class=\"jupyter-button jupyter-widgets mod-primary widget-button\"\n",
    "                    onclick=\"exportMaster();document.getElementById('save_conf').style.display = 'none';\"> Save\n",
    "                </button>\n",
    "                <button type=\"button\" class=\"jupyter-button jupyter-widgets widget-button\"\n",
    "                    onclick=\"document.getElementById('save_conf').style.display = 'none';\"> Cancel </button>\n",
    "            </div>\n",
    "        </div>\n",
    "    </div>\n",
    "\n",
    "</div>\n",
    "<script>\n",
    "    $(document).ready(function () {\n",
    "        $(\"#fileBrowser\").on('change', function() {\n",
    "            updateFileBrowserOpenBtn();\n",
    "        });\n",
    "        paramDataset = new URL(window.location).searchParams.get(\"dataset\");\n",
    "        if (paramDataset !== null) {\n",
    "            paramDataset = \"/\" + paramDataset;\n",
    "            QB.currentPath = paramDataset.substring(0, paramDataset.lastIndexOf(\"/\")+1);\n",
    "        }\n",
    "        showCurrentBrowser();\n",
    "        findCurrentProjectID()\n",
    "            .done(function (projectId) {\n",
    "                QB.currentProjectId = projectId;\n",
    "            })\n",
    "            .done(function () {\n",
    "                fileBrowserRefresh(paramDataset);\n",
    "            });\n",
    "    });\n",
    "    var QB = {\n",
    "        currentProjectId: null,\n",
    "        currentPath: \"/\",\n",
    "        browser: {\"/\":[]}\n",
    "    }\n",
    "    var tempDF_columns = []\n",
    "    var datafile_options = []\n",
    "    var filter_divs = [\"null-filterType\", \"filterType_joinApproxTime\", \"filterType_join\",\n",
    "        \"filterType_year\", \"filterType_withColumnRenamed\", \"filterType_replace\",\n",
    "        \"filterType_fillna\", \"filterType_orderBy\", \"filterType_count\",\n",
    "        \"filterType_min\", \"filterType_max\", \"filterType_sum\", \"filterType_ltrim\",\n",
    "        \"filterType_rtrim\", \"filterType_avg\", \"filterType_filter\", \"filterType_filterOr\",\n",
    "        \"filterType_select\", \"filterType_drop\", \"filterType_isnan\",\n",
    "        \"filterType_isnull\", \"filterType_quarter\", \"filterType_month\",\n",
    "        \"filterType_minute\", \"filterType_hour\", \"filterType_dayofmonth\", \"filterType_dayofyear\",\n",
    "        \"filterType_to_date\", \"filterType_unix_timestamp\", \"filterType_ytrunc\", \"filterType_mtrunc\",\n",
    "        \"filterType_datediff\", \"filterType_months_between\", \"filterType_sqrt\", \"filterType_round\",\n",
    "        \"filterType_when_1\", \"filterType_when_2\", \"filterType_when_3\", \"filterType_col_operation_2\",\n",
    "        \"filterType_col_operation_1\"]\n",
    "    var visibleDivId = null;\n",
    "    var applied_filters = {}\n",
    "    var filter_count = 0\n",
    "    var successfulFilters = []\n",
    "    var successfulFiltersPermanent = [] //< !--temporary solution, to be removed -->\n",
    "\n",
    "    function show_filters() {\n",
    "        $(\"#show-filters\").empty()\n",
    "        $(\"#show-filters\").append(\"<label class='qblabel'>Selected Filters</label>\")\n",
    "        $.each(applied_filters, function (key, val) {\n",
    "            if (successfulFiltersPermanent.includes(key)) {\n",
    "                $(\"#show-filters\").append(\"<li id='\" + key + \"' class='applied'><p><em>\" + val[\"descr\"] + \"</em></p></li>\");\n",
    "            } else {\n",
    "                $(\"#show-filters\").append(\"<li id='\" + key + \"' class='notapplied'><p><em>\" + val[\"descr\"] + \"</em></p></li>\");\n",
    "            }\n",
    "        });\n",
    "    }\n",
    "\n",
    "    function exportMaster() {\n",
    "        var kernel = IPython.notebook.kernel;\n",
    "        var callbacks = {\n",
    "            iopub: {\n",
    "                output: csvExported,\n",
    "            }\n",
    "        }\n",
    "        var folder = document.getElementById('folderNameSelect').value;\n",
    "        var dataset = document.getElementById('datasetSaveSelect').value;\n",
    "        var withHeader = 0;\n",
    "        if ($(\"#saveHeaderCheckbox\").is(\":checked\")) {\n",
    "            withHeader = 1;\n",
    "        }\n",
    "        path = dataset + \"/\" + folder\n",
    "        var msg_id = kernel.execute('saveMasterAsCSV(\"' + path + '\",' + withHeader + ')', callbacks);\n",
    "\n",
    "    }\n",
    "\n",
    "    // call python to read file from hdfs, reset filter dropdown and populate column dropdowns -->\n",
    "    function selectDatafile() {\n",
    "        dataset = document.getElementById('fileSelect').value;\n",
    "        separator = document.getElementById('sepSelect').value;\n",
    "        encoding = document.getElementById('encSelect').value;\n",
    "        loadDataFile(dataset, separator, encoding);\n",
    "    }\n",
    "\n",
    "    function loadDataFile(path, separator, encoding) {\n",
    "        var kernel = IPython.notebook.kernel;\n",
    "        var callbacks = {\n",
    "            iopub: {\n",
    "                output: handle_output,\n",
    "            }\n",
    "        }\n",
    "        var msg_id = kernel.execute('read_dataset(\"' + path + '\",\"' + separator + '\",\"' + encoding + '\")', callbacks);\n",
    "        divVisibility(\"null-filterType\");\n",
    "        document.getElementById(\"filter-application-buttons\").style.display = \"block\"; //because now we have a tempDF\n",
    "        visibleDivId = null;\n",
    "        applied_filters = {}\n",
    "        filter_count = 0\n",
    "        $(\"#show-filters\").empty()\n",
    "        successfulFilters = [];\n",
    "        successfulFiltersPermanent = [];\n",
    "    }\n",
    "\n",
    "    function selectDataset() {\n",
    "        var kernel = IPython.notebook.kernel;\n",
    "        var callbacks = {\n",
    "            iopub: {\n",
    "                output: getCSVfilesFromFS,\n",
    "            }\n",
    "        }\n",
    "        dataset = document.getElementById('datasetSelect').value;\n",
    "        var msg_id = kernel.execute('walk_dataset(\"' + dataset + '\")', callbacks);\n",
    "    }\n",
    "\n",
    "    function getCSVfilesFromFS(data) {\n",
    "        datafile_options = data.content.text;\n",
    "        datafilesObtained(JSON.parse(datafile_options.replace(/u\\'/g, '\"').replace(/\\'/g, '\"')))\n",
    "    }\n",
    "\n",
    "    //add tempDF columns to the dropdown selectors\n",
    "    function handle_output(data) {\n",
    "        tempDF_columns = data.content.text;\n",
    "        addColumnFields(tempDF_columns);\n",
    "    }\n",
    "\n",
    "    function filters_applied_result(data) {\n",
    "        successfulFilters = JSON.parse(data.content.text.replace(/\\'/g, '\"'))\n",
    "        for (var i = 0; i < successfulFilters.length; i++) {\n",
    "            var index = successfulFilters[i];\n",
    "            applied_filters[index][\"executed\"] = true;\n",
    "            successfulFiltersPermanent.push(index);\n",
    "        }\n",
    "        show_filters();\n",
    "        var kernel = IPython.notebook.kernel;\n",
    "        var callbacks = {\n",
    "            iopub: {\n",
    "                output: handle_output,\n",
    "            }\n",
    "        }\n",
    "        var msg_id = kernel.execute('getTempColumns()', callbacks);\n",
    "    }\n",
    "\n",
    "    function tempToMaster() {\n",
    "        var kernel = IPython.notebook.kernel;\n",
    "        var callbacks = {\n",
    "            iopub: {\n",
    "                output: masterDFupdated,\n",
    "            }\n",
    "        }\n",
    "        var msg_id = kernel.execute('tempDFtoMasterDF()', callbacks);\n",
    "    }\n",
    "\n",
    "    function csvExported() {\n",
    "        alert(\"CSV ready\")\n",
    "    }\n",
    "\n",
    "    function masterDFupdated(data) {\n",
    "        masterDF_columns = data.content.text;\n",
    "        addMasterColumnFields(masterDF_columns);\n",
    "        alert(\"The master Dataframe (masterDF) was changed successfully!\")\n",
    "    }\n",
    "\n",
    "    function populateAvailableDatasetsList(data) {\n",
    "        $.get({\n",
    "            url: \"/hopsworks-api/api/project/\" + data.content.text + \"/dataset/getContent\", \n",
    "            headers: { \"Authorization\": localStorage.getItem(\"token\") }}, function (data, status) {\n",
    "            var user_datasets = []\n",
    "\n",
    "            skipthem = [\"Jupyter\", \"Logs\", \"Models\", \"notebook\", \"Resources\"]\n",
    "            data.forEach(function (element) {\n",
    "                if (skipthem.includes(element.name) || element.name.endsWith(\".db\")) {\n",
    "                    return;\n",
    "                }\n",
    "                user_datasets.push(element.path)\n",
    "            });\n",
    "            var dropl = document.getElementById(\"datasetSelect\");\n",
    "            dropl.options.length = 0;\n",
    "            for (var i = 0; i < user_datasets.length; i++) {\n",
    "                dropl.options[dropl.options.length] = new Option(user_datasets[i].substring(user_datasets[i].lastIndexOf(\"/\") + 1), user_datasets[i]);\n",
    "            }\n",
    "            var droplsave = document.getElementById(\"datasetSaveSelect\");\n",
    "            droplsave.options.length = 0;\n",
    "            for (var i = 0; i < user_datasets.length; i++) {\n",
    "                droplsave.options[droplsave.options.length] = new Option(user_datasets[i].substring(user_datasets[i].lastIndexOf(\"/\") + 1), user_datasets[i]);\n",
    "            }\n",
    "        })\n",
    "    }\n",
    "\n",
    "    function fileBrowserOpen() {\n",
    "        var dropl = document.getElementById(\"fileBrowser\");\n",
    "        if (dropl.options[dropl.selectedIndex].getAttribute(\"dir\") === \"true\") {\n",
    "            QB.currentPath = dropl.value;\n",
    "            fileBrowserRefresh();\n",
    "        } else {\n",
    "            console.log(\"Open file: \" + dropl.options[dropl.selectedIndex].getAttribute(\"path\"));\n",
    "            dataset = dropl.options[dropl.selectedIndex].getAttribute(\"path\");\n",
    "            separator = document.getElementById('sepSelect').value;\n",
    "            encoding = document.getElementById('encSelect').value;\n",
    "            loadDataFile(dataset, separator, encoding);\n",
    "        }\n",
    "    }\n",
    "\n",
    "    function updateFileBrowserOpenBtn() {\n",
    "        var dropl = document.getElementById(\"fileBrowser\");\n",
    "        if (dropl.options[dropl.selectedIndex].getAttribute(\"dir\") === \"true\") {\n",
    "            $(\"#fileBrowserOpenBtn\").text(\"Open Folder\");\n",
    "        } else {\n",
    "            $(\"#fileBrowserOpenBtn\").text(\"Open File\");\n",
    "        }\n",
    "    }\n",
    "\n",
    "    function forceFileBrowserRefresh() {\n",
    "        var path = QB.currentPath || \"/\";\n",
    "        if (path.substr(-1) !== \"/\") path = path + \"/\";\n",
    "\n",
    "        QB.browser[path] = [];\n",
    "        console.log(\"force reload files \" + path);\n",
    "        fileBrowserRefresh();\n",
    "    }\n",
    "\n",
    "    function fileBrowserRefresh(selectedDataset) {\n",
    "        var path = QB.currentPath || \"/\";\n",
    "        if (path.substr(-1) !== \"/\") path = path + \"/\";\n",
    "\n",
    "        if (!QB.browser[path]) {\n",
    "            QB.browser[path] = [];\n",
    "        }\n",
    "\n",
    "        var b = QB.browser[path];\n",
    "\n",
    "        if (QB.browser[path].length > 0) {\n",
    "            console.log(\"retrieve cached folder: \" + path)\n",
    "            populateFileBrowser(b, selectedDataset);\n",
    "            return;\n",
    "        }\n",
    "        console.log(\"retrieve folder: \" + path)\n",
    "        $.get({\n",
    "            url: `/hopsworks-api/api/project/${QB.currentProjectId}/dataset/getContent${path}`, \n",
    "            headers: { \"Authorization\": localStorage.getItem(\"token\") }}, function (data, status) {\n",
    "            if (path !== \"/\") {\n",
    "                var relativePath = path.split(\"/\");\n",
    "                relativePath.pop();\n",
    "                relativePath.pop();\n",
    "                b.push({\n",
    "                    label: \"..\",\n",
    "                    relativePath: relativePath.join(\"/\") + \"/\",\n",
    "                    dir: true \n",
    "                })\n",
    "            }\n",
    "\n",
    "            var shared = path.indexOf(\"::\") > 0;\n",
    "            skipthem = [\"Jupyter\", \"Logs\", \"Models\", \"notebook\", \"Resources\"]\n",
    "            data.forEach(function (element) {\n",
    "                if (element.dir && (skipthem.includes(element.name) || element.name.endsWith(\".db\"))) {\n",
    "                    return;\n",
    "                }\n",
    "                if (!element.dir && element.name.substr(-4) !== \".csv\") {\n",
    "                    return;\n",
    "                }\n",
    "                b.push({\n",
    "                    label: element.name,\n",
    "                    path: element.path,\n",
    "                    relativePath: `${path}${element.name}`,\n",
    "                    dir: element.dir\n",
    "                });\n",
    "            });\n",
    "            b.sort(browserSort);\n",
    "            populateFileBrowser(b, selectedDataset);\n",
    "        });\n",
    "    }\n",
    "    function populateFileBrowser(b, selectedDataset) {\n",
    "        var dropl = document.getElementById(\"fileBrowser\");\n",
    "        dropl.options.length = 0;\n",
    "        b.forEach(function(bb) {\n",
    "            dropl.options.add(new Option(bb.label, bb.relativePath));\n",
    "            dropl.options[dropl.options.length-1].setAttribute(\"dir\", bb.dir);\n",
    "            if (!bb.dir)\n",
    "                dropl.options[dropl.options.length-1].setAttribute(\"path\", bb.path);\n",
    "            if (selectedDataset && selectedDataset === bb.relativePath)\n",
    "                dropl.options[dropl.options.length-1].selected = true;\n",
    "        });\n",
    "        updateFileBrowserOpenBtn();\n",
    "    }\n",
    "    // Will sort firt the directories, then the files, sorted by label.\n",
    "    function browserSort(a, b) {\n",
    "        if (~a.dir < ~b.dir) {\n",
    "            return -1;\n",
    "        }\n",
    "        if (~a.dir > ~b.dir) {\n",
    "            return 1;\n",
    "        }\n",
    "        if (a.label.toLowerCase() < b.label.toLowerCase()) {\n",
    "            return -1;\n",
    "        }\n",
    "        if (a.label.toLowerCase() > b.label.toLowerCase()) {\n",
    "            return 1;\n",
    "        }\n",
    "        return 0;\n",
    "    }\n",
    "    function findCurrentProjectID() {\n",
    "        var deferred = jQuery.Deferred();\n",
    "        $.get({\n",
    "            url: \"/hopsworks-api/api/project\", \n",
    "            headers: { \"Authorization\": localStorage.getItem(\"token\") }}, function (data, status) {\n",
    "            var pdict = {}\n",
    "            data.forEach(function (e) {\n",
    "                pdict[e.project.name] = e.project.id;\n",
    "            });\n",
    "            var kernel = IPython.notebook.kernel;\n",
    "            var callbacks = {\n",
    "                iopub: {\n",
    "                    output: function(data) {\n",
    "                        var projectId = data.content.text;\n",
    "                        deferred.resolve(projectId);\n",
    "                    },\n",
    "                }\n",
    "            }\n",
    "            var pdict_parameter = (JSON.stringify(pdict))\n",
    "            var msg_id = kernel.execute('find_current_projectID(\\'' + pdict_parameter + '\\')', callbacks);\n",
    "        });\n",
    "        return deferred.promise();\n",
    "    }\n",
    "\n",
    "    function showCurrentBrowser() {\n",
    "        $.get({\n",
    "            url: \"/hopsworks-api/api/project\", \n",
    "            headers: { \"Authorization\": localStorage.getItem(\"token\") }}, function (data, status) {\n",
    "            var pdict = {}\n",
    "            data.forEach(function (e) {\n",
    "                pdict[e.project.name] = e.project.id;\n",
    "            });\n",
    "            var kernel = IPython.notebook.kernel;\n",
    "            var callbacks = {\n",
    "                iopub: {\n",
    "                    output: populateAvailableDatasetsList,\n",
    "                }\n",
    "            }\n",
    "            var pdict_parameter = (JSON.stringify(pdict))\n",
    "            var msg_id = kernel.execute('find_current_projectID(\\'' + pdict_parameter + '\\')', callbacks);\n",
    "        })\n",
    "    }\n",
    "\n",
    "    function datafilesObtained(options) {\n",
    "        var dropl = document.getElementById(\"fileSelect\");\n",
    "        dropl.options.length = 0;\n",
    "        for (opt in options) {\n",
    "            dropl.options[dropl.options.length] = new Option(options[opt].substring(options[opt].lastIndexOf(\"/\") + 1), options[opt]);\n",
    "        }\n",
    "    }\n",
    "\n",
    "    function divVisibilityChanged() {\n",
    "        enabled_filter = document.getElementById(\"filterType\").value;\n",
    "        divid = \"filterType_\".concat(enabled_filter);\n",
    "        divVisibility(divid);\n",
    "    }\n",
    "\n",
    "    //choose which of the forms will be visible\n",
    "    function divVisibility(divId) {\n",
    "        visibleDivId = divId;\n",
    "        hideNonVisibleDivs();\n",
    "    }\n",
    "\n",
    "    //helper:called to choose which of the forms will be visible\n",
    "    function hideNonVisibleDivs() {\n",
    "        var i, divId, div;\n",
    "        for (i = 0; i < filter_divs.length; i++) {\n",
    "            divId = filter_divs[i];\n",
    "            div = document.getElementById(divId);\n",
    "            if (visibleDivId === divId) {\n",
    "                div.style.display = \"block\";\n",
    "            } else {\n",
    "                div.style.display = \"none\";\n",
    "            }\n",
    "        }\n",
    "        if ((visibleDivId != null) && (visibleDivId != \"null-filterType\")) {\n",
    "            document.getElementById(\"filter-control-buttons\").style.display = \"block\";\n",
    "        }\n",
    "        else {\n",
    "            document.getElementById(\"filter-control-buttons\").style.display = \"none\";\n",
    "        }\n",
    "\n",
    "    }\n",
    "\n",
    "    //populate column dropdowns\n",
    "    function addColumnFields(columns) {\n",
    "        arraycols = JSON.parse(columns.replace(/\\'/g, '\"'))\n",
    "        $('.temp-column').each(function (index) {\n",
    "            if ($(this).hasClass(\"empty-option-allowed\")) {\n",
    "                $(this).children('option:not(:first)').remove();\n",
    "            }\n",
    "            else {\n",
    "                $(this).children('option').remove();\n",
    "            }\n",
    "\n",
    "            for (var i = 0; i < arraycols.length; i++) {\n",
    "                optionGenerator($(this), arraycols[i].trim());\n",
    "            }\n",
    "\n",
    "        });\n",
    "\n",
    "    }\n",
    "\n",
    "    //populate master column dropdowns\n",
    "    function addMasterColumnFields(columns) {\n",
    "        masterarraycols = JSON.parse(columns.replace(/\\'/g, '\"'))\n",
    "        $('.master-column').each(function (index) {\n",
    "            $(this).children('option:not(:first)').remove();\n",
    "            for (var i = 0; i < masterarraycols.length; i++) {\n",
    "                optionGenerator($(this), masterarraycols[i].trim());\n",
    "            }\n",
    "\n",
    "        });\n",
    "    }\n",
    "\n",
    "    function applyAllFilters() {\n",
    "        var kernel = IPython.notebook.kernel;\n",
    "        var callbacks = {\n",
    "            iopub: {\n",
    "                output: filters_applied_result,\n",
    "            }\n",
    "        }\n",
    "        agh = (JSON.stringify(applied_filters))\n",
    "        lala = {}\n",
    "        var msg_id = kernel.execute('apply_filters(\\'' + agh + '\\')', callbacks);\n",
    "    }\n",
    "\n",
    "    function cancelFilter() {\n",
    "        divVisibility('null-filterType');\n",
    "        $(\"#filterType\")[0].selectedIndex = 0;\n",
    "    }\n",
    "\n",
    "    function addFilter() {\n",
    "        enabled_filter = document.getElementById(\"filterType\").value;\n",
    "        filterObj = {}\n",
    "        filter_count++;\n",
    "        switch (enabled_filter) {\n",
    "            case \"filter\": {\n",
    "                filterObj[\"type\"] = enabled_filter\n",
    "                filterObj[\"executed\"] = false\n",
    "                parameters = []\n",
    "                parameters.push($(\"#filterType_filter .single-column-selector\").find(\":selected\").val())\n",
    "                parameters.push($(\"#filterType_filter .operator-selector\").find(\":selected\").val())\n",
    "                parameters.push($(\"#filterType_filter .user-defined-value\").val())\n",
    "                filterObj[\"parameters\"] = parameters\n",
    "                filterObj[\"descr\"] = \"Filter \" + parameters.toString();\n",
    "                applied_filters[filter_count.toString()] = filterObj;\n",
    "                break;\n",
    "            }\n",
    "            case \"filterOr\": {\n",
    "                filterObj[\"type\"] = enabled_filter;\n",
    "                filterObj[\"executed\"] = false;\n",
    "                parameters = [];\n",
    "                parameters.push($(\"#filterType_filterOr .single-column-selector.col1_colname\").find(\":selected\").val());\n",
    "                parameters.push($(\"#filterType_filterOr .operator-selector.col1_operator\").find(\":selected\").val());\n",
    "                parameters.push($(\"#filterType_filterOr .user-defined-value.col1_value\").val());\n",
    "                parameters.push($(\"#filterType_filterOr .single-column-selector.col2_colname\").find(\":selected\").val());\n",
    "                parameters.push($(\"#filterType_filterOr .operator-selector.col2_operator\").find(\":selected\").val());\n",
    "                parameters.push($(\"#filterType_filterOr .user-defined-value.col2_value\").val());\n",
    "                filterObj[\"parameters\"] = parameters;\n",
    "                filterObj[\"descr\"] = \"Filter \" + parameters.slice(0,3).toString() + \" OR \" + parameters.slice(3).toString();\n",
    "                applied_filters[filter_count.toString()] = filterObj;\n",
    "                break;\n",
    "            }\n",
    "            case \"orderBy\": {\n",
    "                filterObj[\"type\"] = enabled_filter\n",
    "                filterObj[\"executed\"] = false\n",
    "                parameters = []\n",
    "                parameters.push([$(\"#filterType_orderBy .single-column-selector\").find(\":selected\").val()])\n",
    "                parameters.push([Number($(\"#filterType_orderBy .operator-selector\").find(\":selected\").val())])\n",
    "                filterObj[\"parameters\"] = parameters\n",
    "                filterObj[\"descr\"] = parameters[0][0] + '(' + $(\"#filterType_orderBy .operator-selector\").find(\":selected\").text() + ')'\n",
    "                applied_filters[filter_count.toString()] = filterObj;\n",
    "                break;\n",
    "            }\n",
    "            case \"count\": {\n",
    "                filterObj[\"type\"] = enabled_filter\n",
    "                filterObj[\"executed\"] = false\n",
    "                parameters = []\n",
    "                parameters.push([$(\"#filterType_count .single-column-selector\").find(\":selected\").val()])\n",
    "                filterObj[\"parameters\"] = parameters\n",
    "                filterObj[\"descr\"] = \"Count [OrderBy:\" + parameters[0][0] + \"]\"\n",
    "                applied_filters[filter_count.toString()] = filterObj;\n",
    "                break;\n",
    "            }\n",
    "            case \"ltrim\": {\n",
    "                filterObj[\"type\"] = enabled_filter\n",
    "                filterObj[\"executed\"] = false\n",
    "                parameters = []\n",
    "                parameters.push($(\"#filterType_ltrim .single-column-selector\").find(\":selected\").val())\n",
    "                filterObj[\"parameters\"] = parameters\n",
    "                filterObj[\"descr\"] = \"Left trim [Column:\" + parameters[0] + \"]\"\n",
    "                applied_filters[filter_count.toString()] = filterObj;\n",
    "                break;\n",
    "            }\n",
    "            case \"rtrim\": {\n",
    "                filterObj[\"type\"] = enabled_filter\n",
    "                filterObj[\"executed\"] = false\n",
    "                parameters = []\n",
    "                parameters.push($(\"#filterType_rtrim .single-column-selector\").find(\":selected\").val())\n",
    "                filterObj[\"parameters\"] = parameters\n",
    "                filterObj[\"descr\"] = \"Right trim [Column:\" + parameters[0] + \"]\"\n",
    "                applied_filters[filter_count.toString()] = filterObj;\n",
    "                break;\n",
    "            }\n",
    "            case \"isnan\": {\n",
    "                filterObj[\"type\"] = enabled_filter\n",
    "                filterObj[\"executed\"] = false\n",
    "                parameters = []\n",
    "                parameters.push($(\"#filterType_isnan .single-column-selector\").find(\":selected\").val())\n",
    "                filterObj[\"parameters\"] = parameters\n",
    "                filterObj[\"descr\"] = \"Is NaN [Column:\" + parameters[0] + \"]\"\n",
    "                applied_filters[filter_count.toString()] = filterObj;\n",
    "                break;\n",
    "            }\n",
    "            case \"isnull\": {\n",
    "                filterObj[\"type\"] = enabled_filter\n",
    "                filterObj[\"executed\"] = false\n",
    "                parameters = []\n",
    "                parameters.push($(\"#filterType_isnull .single-column-selector\").find(\":selected\").val())\n",
    "                filterObj[\"parameters\"] = parameters\n",
    "                filterObj[\"descr\"] = \"Is Null [Column:\" + parameters[0] + \"]\"\n",
    "                applied_filters[filter_count.toString()] = filterObj;\n",
    "                break;\n",
    "            }\n",
    "            case \"select\": {\n",
    "                filterObj[\"type\"] = enabled_filter\n",
    "                filterObj[\"executed\"] = false\n",
    "                parameters = []\n",
    "                param_array = $(\"#filterType_select .multiple-column-selector option:selected\").map(function () { return this.value }).get();\n",
    "                parameters.push(param_array);\n",
    "                filterObj[\"parameters\"] = parameters\n",
    "                filterObj[\"descr\"] = \"Select [\" + parameters.toString() + \"]\"\n",
    "                applied_filters[filter_count.toString()] = filterObj;\n",
    "                break;\n",
    "            }\n",
    "            case \"drop\": {\n",
    "                filterObj[\"type\"] = enabled_filter\n",
    "                filterObj[\"executed\"] = false\n",
    "                parameters = []\n",
    "                param_array = $(\"#filterType_drop .multiple-column-selector option:selected\").map(function () { return this.value }).get();\n",
    "                parameters.push(param_array);\n",
    "                filterObj[\"parameters\"] = parameters\n",
    "                filterObj[\"descr\"] = \"Drop [\" + parameters.toString() + \"]\"\n",
    "                applied_filters[filter_count.toString()] = filterObj;\n",
    "                break;\n",
    "            }\n",
    "            case \"min\": {\n",
    "                filterObj[\"type\"] = enabled_filter\n",
    "                filterObj[\"executed\"] = false\n",
    "                parameters = []\n",
    "                parameters.push([$(\"#filterType_min .single-column-selector\").find(\":selected\").val()])\n",
    "                cols_array = $(\"#filterType_min .multiple-column-selector option:selected\").map(function () { return this.value }).get();\n",
    "                parameters.push(cols_array);\n",
    "                filterObj[\"parameters\"] = parameters\n",
    "                filterObj[\"descr\"] = \"Min (\" + parameters[1].toString() + \") - Group by \" + parameters[0][0]\n",
    "                applied_filters[filter_count.toString()] = filterObj;\n",
    "                break;\n",
    "            }\n",
    "            case \"max\": {\n",
    "                filterObj[\"type\"] = enabled_filter\n",
    "                filterObj[\"executed\"] = false\n",
    "                parameters = []\n",
    "                parameters.push([$(\"#filterType_max .single-column-selector\").find(\":selected\").val()])\n",
    "                cols_array = $(\"#filterType_max .multiple-column-selector option:selected\").map(function () { return this.value }).get();\n",
    "                parameters.push(cols_array);\n",
    "                filterObj[\"parameters\"] = parameters\n",
    "                filterObj[\"descr\"] = \"Max (\" + parameters[1].toString() + \") - Group by \" + parameters[0][0]\n",
    "                applied_filters[filter_count.toString()] = filterObj;\n",
    "                break;\n",
    "            }\n",
    "            case \"sum\": {\n",
    "                filterObj[\"type\"] = enabled_filter\n",
    "                filterObj[\"executed\"] = false\n",
    "                parameters = []\n",
    "                parameters.push([$(\"#filterType_sum .single-column-selector\").find(\":selected\").val()])\n",
    "                cols_array = $(\"#filterType_sum .multiple-column-selector option:selected\").map(function () { return this.value }).get();\n",
    "                parameters.push(cols_array);\n",
    "                filterObj[\"parameters\"] = parameters\n",
    "                filterObj[\"descr\"] = \"Sum (\" + parameters[1].toString() + \") - Group by \" + parameters[0][0]\n",
    "                applied_filters[filter_count.toString()] = filterObj;\n",
    "                break;\n",
    "            }\n",
    "            case \"avg\": {\n",
    "                filterObj[\"type\"] = enabled_filter\n",
    "                filterObj[\"executed\"] = false\n",
    "                parameters = []\n",
    "                parameters.push([$(\"#filterType_avg .single-column-selector\").find(\":selected\").val()])\n",
    "                cols_array = $(\"#filterType_avg .multiple-column-selector option:selected\").map(function () { return this.value }).get();\n",
    "                parameters.push(cols_array);\n",
    "                filterObj[\"parameters\"] = parameters\n",
    "                filterObj[\"descr\"] = \"Average (\" + parameters[1].toString() + \") - Group by \" + parameters[0][0]\n",
    "                applied_filters[filter_count.toString()] = filterObj;\n",
    "                break;\n",
    "            }\n",
    "            case \"year\": {\n",
    "                filterObj[\"type\"] = enabled_filter\n",
    "                filterObj[\"executed\"] = false\n",
    "                parameters = []\n",
    "                parameters.push($(\"#filterType_year .single-column-selector\").find(\":selected\").val())\n",
    "                parameters.push($(\"#filterType_year .user-defined-value\").val())\n",
    "                filterObj[\"parameters\"] = parameters\n",
    "                filterObj[\"descr\"] = \"New Year Col:\" + parameters.toString();\n",
    "                applied_filters[filter_count.toString()] = filterObj;\n",
    "                break;\n",
    "            }\n",
    "            case \"quarter\": {\n",
    "                filterObj[\"type\"] = enabled_filter\n",
    "                filterObj[\"executed\"] = false\n",
    "                parameters = []\n",
    "                parameters.push($(\"#filterType_quarter .single-column-selector\").find(\":selected\").val())\n",
    "                parameters.push($(\"#filterType_quarter .user-defined-value\").val())\n",
    "                filterObj[\"parameters\"] = parameters\n",
    "                filterObj[\"descr\"] = \"New Quarter Col:\" + parameters.toString();\n",
    "                applied_filters[filter_count.toString()] = filterObj;\n",
    "                break;\n",
    "            }\n",
    "            case \"month\": {\n",
    "                filterObj[\"type\"] = enabled_filter\n",
    "                filterObj[\"executed\"] = false\n",
    "                parameters = []\n",
    "                parameters.push($(\"#filterType_month .single-column-selector\").find(\":selected\").val())\n",
    "                parameters.push($(\"#filterType_month .user-defined-value\").val())\n",
    "                filterObj[\"parameters\"] = parameters\n",
    "                filterObj[\"descr\"] = \"New Month Col:\" + parameters.toString();\n",
    "                applied_filters[filter_count.toString()] = filterObj;\n",
    "                break;\n",
    "            }\n",
    "            case \"hour\": {\n",
    "                filterObj[\"type\"] = enabled_filter\n",
    "                filterObj[\"executed\"] = false\n",
    "                parameters = []\n",
    "                parameters.push($(\"#filterType_hour .single-column-selector\").find(\":selected\").val())\n",
    "                parameters.push($(\"#filterType_hour .user-defined-value\").val())\n",
    "                filterObj[\"parameters\"] = parameters\n",
    "                filterObj[\"descr\"] = \"New Hour Col:\" + parameters.toString();\n",
    "                applied_filters[filter_count.toString()] = filterObj;\n",
    "                break;\n",
    "            }\n",
    "            case \"minute\": {\n",
    "                filterObj[\"type\"] = enabled_filter\n",
    "                filterObj[\"executed\"] = false\n",
    "                parameters = []\n",
    "                parameters.push($(\"#filterType_minute .single-column-selector\").find(\":selected\").val())\n",
    "                parameters.push($(\"#filterType_minute .user-defined-value\").val())\n",
    "                filterObj[\"parameters\"] = parameters\n",
    "                filterObj[\"descr\"] = \"New Minute Col:\" + parameters.toString();\n",
    "                applied_filters[filter_count.toString()] = filterObj;\n",
    "                break;\n",
    "            }\n",
    "            case \"dayofmonth\": {\n",
    "                filterObj[\"type\"] = enabled_filter\n",
    "                filterObj[\"executed\"] = false\n",
    "                parameters = []\n",
    "                parameters.push($(\"#filterType_dayofmonth .single-column-selector\").find(\":selected\").val())\n",
    "                parameters.push($(\"#filterType_dayofmonth .user-defined-value\").val())\n",
    "                filterObj[\"parameters\"] = parameters\n",
    "                filterObj[\"descr\"] = \"New MonthDay Col:\" + parameters.toString();\n",
    "                applied_filters[filter_count.toString()] = filterObj;\n",
    "                break;\n",
    "            }\n",
    "            case \"dayofyear\": {\n",
    "                filterObj[\"type\"] = enabled_filter\n",
    "                filterObj[\"executed\"] = false\n",
    "                parameters = []\n",
    "                parameters.push($(\"#filterType_dayofyear .single-column-selector\").find(\":selected\").val())\n",
    "                parameters.push($(\"#filterType_dayofyear .user-defined-value\").val())\n",
    "                filterObj[\"parameters\"] = parameters\n",
    "                filterObj[\"descr\"] = \"New YearDay Col:\" + parameters.toString();\n",
    "                applied_filters[filter_count.toString()] = filterObj;\n",
    "                break;\n",
    "            }\n",
    "            case \"to_date\": {\n",
    "                filterObj[\"type\"] = enabled_filter\n",
    "                filterObj[\"executed\"] = false\n",
    "                parameters = []\n",
    "                parameters.push($(\"#filterType_to_date .single-column-selector\").find(\":selected\").val())\n",
    "                parameters.push($(\"#filterType_to_date .user-defined-value\").val())\n",
    "                filterObj[\"parameters\"] = parameters\n",
    "                filterObj[\"descr\"] = \"New Date Col:\" + parameters.toString();\n",
    "                applied_filters[filter_count.toString()] = filterObj;\n",
    "                break;\n",
    "            }\n",
    "            case \"unix_timestamp\": {\n",
    "                filterObj[\"type\"] = enabled_filter\n",
    "                filterObj[\"executed\"] = false\n",
    "                parameters = []\n",
    "                parameters.push($(\"#filterType_unix_timestamp .single-column-selector\").find(\":selected\").val())\n",
    "                parameters.push($(\"#filterType_unix_timestamp .user-defined-value\").val())\n",
    "                filterObj[\"parameters\"] = parameters\n",
    "                filterObj[\"descr\"] = \"New UNIX timestamp Col:\" + parameters.toString();\n",
    "                applied_filters[filter_count.toString()] = filterObj;\n",
    "                break;\n",
    "            }\n",
    "            case \"sqrt\": {\n",
    "                filterObj[\"type\"] = enabled_filter\n",
    "                filterObj[\"executed\"] = false\n",
    "                parameters = []\n",
    "                parameters.push($(\"#filterType_sqrt .single-column-selector\").find(\":selected\").val())\n",
    "                parameters.push($(\"#filterType_sqrt .user-defined-value\").val())\n",
    "                filterObj[\"parameters\"] = parameters\n",
    "                filterObj[\"descr\"] = \"New SQRT Col:\" + parameters.toString();\n",
    "                applied_filters[filter_count.toString()] = filterObj;\n",
    "                break;\n",
    "            }\n",
    "            case \"round\": {\n",
    "                filterObj[\"type\"] = enabled_filter\n",
    "                filterObj[\"executed\"] = false\n",
    "                parameters = []\n",
    "                parameters.push($(\"#filterType_round .single-column-selector\").find(\":selected\").val())\n",
    "                parameters.push($(\"#filterType_round .int-input\").val().toString())\n",
    "                parameters.push($(\"#filterType_round .col-name\").val())\n",
    "                filterObj[\"parameters\"] = parameters\n",
    "                filterObj[\"descr\"] = \"New round Col:\" + parameters.toString();\n",
    "                applied_filters[filter_count.toString()] = filterObj;\n",
    "                break;\n",
    "            }\n",
    "            case \"ytrunc\": {\n",
    "                filterObj[\"type\"] = enabled_filter\n",
    "                filterObj[\"executed\"] = false\n",
    "                parameters = []\n",
    "                parameters.push($(\"#filterType_ytrunc .single-column-selector\").find(\":selected\").val())\n",
    "                parameters.push($(\"#filterType_ytrunc .user-defined-value\").val())\n",
    "                filterObj[\"parameters\"] = parameters\n",
    "                filterObj[\"descr\"] = \"Year Trunc:\" + parameters.toString();\n",
    "                applied_filters[filter_count.toString()] = filterObj;\n",
    "                break;\n",
    "            }\n",
    "            case \"mtrunc\": {\n",
    "                filterObj[\"type\"] = enabled_filter\n",
    "                filterObj[\"executed\"] = false\n",
    "                parameters = []\n",
    "                parameters.push($(\"#filterType_mtrunc .single-column-selector\").find(\":selected\").val())\n",
    "                parameters.push($(\"#filterType_mtrunc .user-defined-value\").val())\n",
    "                filterObj[\"parameters\"] = parameters\n",
    "                filterObj[\"descr\"] = \"Month Trunc:\" + parameters.toString();\n",
    "                applied_filters[filter_count.toString()] = filterObj;\n",
    "                break;\n",
    "            }\n",
    "            case \"datediff\": {\n",
    "                filterObj[\"type\"] = enabled_filter\n",
    "                filterObj[\"executed\"] = false\n",
    "                parameters = []\n",
    "                parameters.push($(\"#filterType_datediff .temp-column-A\").find(\":selected\").val())\n",
    "                parameters.push($(\"#filterType_datediff .temp-column-B\").find(\":selected\").val())\n",
    "                parameters.push($(\"#filterType_datediff .user-defined-value\").val())\n",
    "                filterObj[\"parameters\"] = parameters\n",
    "                filterObj[\"descr\"] = \"DateDiff:\" + parameters.toString();\n",
    "                applied_filters[filter_count.toString()] = filterObj;\n",
    "                break;\n",
    "            }\n",
    "            case \"months_between\": {\n",
    "                filterObj[\"type\"] = enabled_filter\n",
    "                filterObj[\"executed\"] = false\n",
    "                parameters = []\n",
    "                parameters.push($(\"#filterType_months_between .temp-column-A\").find(\":selected\").val())\n",
    "                parameters.push($(\"#filterType_months_between .temp-column-B\").find(\":selected\").val())\n",
    "                parameters.push($(\"#filterType_months_between .user-defined-value\").val())\n",
    "                filterObj[\"parameters\"] = parameters\n",
    "                filterObj[\"descr\"] = \"MonthDiff:\" + parameters.toString();\n",
    "                applied_filters[filter_count.toString()] = filterObj;\n",
    "                break;\n",
    "            }\n",
    "            case \"col_operation_1\": {\n",
    "                filterObj[\"type\"] = enabled_filter\n",
    "                filterObj[\"executed\"] = false\n",
    "                parameters = []\n",
    "                parameters.push($(\"#filterType_col_operation_1 .temp-column\").find(\":selected\").val())\n",
    "                parameters.push($(\"#filterType_col_operation_1 .operator-selector\").find(\":selected\").val())\n",
    "                parameters.push($(\"#filterType_col_operation_1 .user-defined-value.operation-value\").val())\n",
    "                parameters.push($(\"#filterType_col_operation_1 .user-defined-value.new-col-name\").val())\n",
    "                filterObj[\"parameters\"] = parameters\n",
    "                filterObj[\"descr\"] = \"Add column:\" + parameters.slice(0, 3).toString() + \" (\" + parameters[3].toString() + \")\";\n",
    "                applied_filters[filter_count.toString()] = filterObj;\n",
    "                break;\n",
    "            }\n",
    "            case \"col_operation_2\": {\n",
    "                filterObj[\"type\"] = enabled_filter\n",
    "                filterObj[\"executed\"] = false\n",
    "                parameters = []\n",
    "                parameters.push($(\"#filterType_col_operation_2 .temp-column-A\").find(\":selected\").val())\n",
    "                parameters.push($(\"#filterType_col_operation_2 .operator-selector\").find(\":selected\").val())\n",
    "                parameters.push($(\"#filterType_col_operation_2 .temp-column-B\").find(\":selected\").val())\n",
    "                parameters.push($(\"#filterType_col_operation_2 .user-defined-value.new-col-name\").val())\n",
    "                filterObj[\"parameters\"] = parameters\n",
    "                filterObj[\"descr\"] = \"Add column:\" + parameters.slice(0, 3).toString() + \" (\" + parameters[3].toString() + \")\";\n",
    "                applied_filters[filter_count.toString()] = filterObj;\n",
    "                break;\n",
    "            }\n",
    "            case \"when_1\": {\n",
    "                filterObj[\"type\"] = enabled_filter\n",
    "                filterObj[\"executed\"] = false\n",
    "                parameters = []\n",
    "                parameters.push($(\"#filterType_when_1 .temp-column.when-div\").find(\":selected\").val())\n",
    "                parameters.push($(\"#filterType_when_1 .operator-selector.when-div\").find(\":selected\").val())\n",
    "                parameters.push($(\"#filterType_when_1 .user-defined-value.when-div\").val())\n",
    "                parameters.push($(\"#filterType_when_1 .user-defined-value.then-div\").val())\n",
    "                parameters.push($(\"#filterType_when_1 .user-defined-value.else-div\").val())\n",
    "                parameters.push($(\"#filterType_when_1 .user-defined-value.new-col-name\").val())\n",
    "                filterObj[\"parameters\"] = parameters\n",
    "                filterObj[\"descr\"] = \"When:\" + parameters.slice(0, 3).toString() + \" Then:\" + parameters[3].toString() + \" Else:\" + parameters[4].toString() + \"(\" + parameters[5].toString() + \")\";\n",
    "                applied_filters[filter_count.toString()] = filterObj;\n",
    "                break;\n",
    "            }\n",
    "            case \"when_2\": {\n",
    "                filterObj[\"type\"] = enabled_filter\n",
    "                filterObj[\"executed\"] = false\n",
    "                parameters = []\n",
    "                parameters.push($(\"#filterType_when_2 .temp-column.when-div\").find(\":selected\").val())\n",
    "                parameters.push($(\"#filterType_when_2 .operator-selector.when-div\").find(\":selected\").val())\n",
    "                parameters.push($(\"#filterType_when_2 .user-defined-value.when-div\").val())\n",
    "                parameters.push($(\"#filterType_when_2 .temp-column.then-div\").find(\":selected\").val())\n",
    "                parameters.push($(\"#filterType_when_2 .operator-selector.then-div\").find(\":selected\").val())\n",
    "                parameters.push($(\"#filterType_when_2 .user-defined-value.then-div\").val())\n",
    "                parameters.push($(\"#filterType_when_2 .temp-column.else-div\").find(\":selected\").val())\n",
    "                parameters.push($(\"#filterType_when_2 .operator-selector.else-div\").find(\":selected\").val())\n",
    "                parameters.push($(\"#filterType_when_2 .user-defined-value.else-div\").val())\n",
    "                parameters.push($(\"#filterType_when_2 .user-defined-value.new-col-name\").val())\n",
    "                filterObj[\"parameters\"] = parameters\n",
    "                filterObj[\"descr\"] = \"When:\" + parameters.slice(0, 3).toString() + \" Then:\" + parameters.slice(3, 6).toString() + \" Else:\" + parameters.slice(6, 9).toString() + \"(\" + parameters[9].toString() + \")\";\n",
    "                applied_filters[filter_count.toString()] = filterObj;\n",
    "                break;\n",
    "            }\n",
    "            case \"when_3\": {\n",
    "                filterObj[\"type\"] = enabled_filter\n",
    "                filterObj[\"executed\"] = false\n",
    "                parameters = []\n",
    "                parameters.push($(\"#filterType_when_3 .temp-column.when-div\").find(\":selected\").val())\n",
    "                parameters.push($(\"#filterType_when_3 .operator-selector.when-div\").find(\":selected\").val())\n",
    "                parameters.push($(\"#filterType_when_3 .user-defined-value.when-div\").val())\n",
    "                parameters.push($(\"#filterType_when_3 .temp-column-A.then-div\").find(\":selected\").val())\n",
    "                parameters.push($(\"#filterType_when_3 .operator-selector.then-div\").find(\":selected\").val())\n",
    "                parameters.push($(\"#filterType_when_3 .temp-column-B.then-div\").find(\":selected\").val())\n",
    "                parameters.push($(\"#filterType_when_3 .temp-column-A.else-div\").find(\":selected\").val())\n",
    "                parameters.push($(\"#filterType_when_3 .operator-selector.else-div\").find(\":selected\").val())\n",
    "                parameters.push($(\"#filterType_when_3 .temp-column-B.else-div\").find(\":selected\").val())\n",
    "                parameters.push($(\"#filterType_when_3 .user-defined-value.new-col-name\").val())\n",
    "                filterObj[\"parameters\"] = parameters\n",
    "                filterObj[\"descr\"] = \"When:\" + parameters.slice(0, 3).toString() + \" Then:\" + parameters.slice(3, 6).toString() + \" Else:\" + parameters.slice(6, 9).toString() + \"(\" + parameters[9].toString() + \")\";\n",
    "                applied_filters[filter_count.toString()] = filterObj;\n",
    "                break;\n",
    "            }\n",
    "            case \"fillna\": {\n",
    "                filterObj[\"type\"] = enabled_filter\n",
    "                filterObj[\"executed\"] = false\n",
    "                parameters = []\n",
    "                pdict = {}\n",
    "                col = $(\"#filterType_fillna .single-column-selector\").find(\":selected\").val();\n",
    "                newv = $(\"#filterType_fillna .user-defined-value\").val();\n",
    "                pdict[col] = newv\n",
    "                parameters.push(pdict)\n",
    "                filterObj[\"parameters\"] = parameters\n",
    "                filterObj[\"descr\"] = col + \":Fill n/a with \" + newv;\n",
    "                applied_filters[filter_count.toString()] = filterObj;\n",
    "                break;\n",
    "            }\n",
    "            case \"replace\": {\n",
    "                filterObj[\"type\"] = enabled_filter\n",
    "                filterObj[\"executed\"] = false\n",
    "                parameters = []\n",
    "                parameters.push([$(\"#filterType_replace .user-defined-value.existing-value\").val()])\n",
    "                parameters.push([$(\"#filterType_replace .user-defined-value.to-be-value\").val()])\n",
    "                parameters.push($(\"#filterType_replace .single-column-selector\").find(\":selected\").val())\n",
    "                filterObj[\"parameters\"] = parameters\n",
    "                filterObj[\"descr\"] = parameters[2] + \" [\" + parameters[0][0] + \" --> \" + parameters[1][0] + \"]\"\n",
    "                applied_filters[filter_count.toString()] = filterObj;\n",
    "                break;\n",
    "            }\n",
    "            case \"withColumnRenamed\": {\n",
    "                filterObj[\"type\"] = enabled_filter\n",
    "                filterObj[\"executed\"] = false\n",
    "                parameters = []\n",
    "                parameters.push($(\"#filterType_withColumnRenamed .single-column-selector\").find(\":selected\").val())\n",
    "                parameters.push($(\"#filterType_withColumnRenamed .user-defined-value\").val())\n",
    "                filterObj[\"parameters\"] = parameters\n",
    "                filterObj[\"descr\"] = \"Rename \" + parameters[0] + \" to \" + parameters[1]\n",
    "                applied_filters[filter_count.toString()] = filterObj;\n",
    "                break;\n",
    "            }\n",
    "            case \"join\": {\n",
    "                filterObj[\"type\"] = enabled_filter\n",
    "                filterObj[\"executed\"] = false\n",
    "                parameters = []\n",
    "                operator = $(\"#filterType_join .operator-selector\").find(\":selected\").val()\n",
    "                tempCol1 = $(\"#filterType_join .single-column-selector.temp-col-1\").find(\":selected\").val()\n",
    "                tempCol2 = $(\"#filterType_join .single-column-selector.temp-col-2\").find(\":selected\").val() || null\n",
    "                tempCol3 = $(\"#filterType_join .single-column-selector.temp-col-3\").find(\":selected\").val() || null\n",
    "                masterCol1 = $(\"#filterType_join .single-column-selector.master-col-1\").find(\":selected\").val()\n",
    "                masterCol2 = $(\"#filterType_join .single-column-selector.master-col-2\").find(\":selected\").val() || null\n",
    "                masterCol3 = $(\"#filterType_join .single-column-selector.master-col-3\").find(\":selected\").val() || null\n",
    "                parameters.push(operator)\n",
    "                parameters.push([masterCol1, tempCol1])\n",
    "                parameters.push([masterCol2, tempCol2])\n",
    "                parameters.push([masterCol3, tempCol3])\n",
    "                filterObj[\"parameters\"] = parameters\n",
    "                filterObj[\"descr\"] = operator + \"join: (\" + masterCol1 + \",\" + tempCol1 + \")\" + \"(\" + masterCol2 + \",\" + tempCol2 + \")\" + \"(\" + masterCol3 + \",\" + tempCol3 + \")\"\n",
    "                applied_filters[filter_count.toString()] = filterObj;\n",
    "                break;\n",
    "            }\n",
    "            case \"joinApproxTime\": {\n",
    "                filterObj[\"type\"] = enabled_filter\n",
    "                filterObj[\"executed\"] = false\n",
    "                parameters = []\n",
    "                operator = $(\"#filterType_joinApproxTime .operator-selector\").find(\":selected\").val()\n",
    "                tempTimeCol = $(\"#filterType_joinApproxTime .single-column-selector.temp-time-col\").find(\":selected\").val()\n",
    "                masterTimeCol = $(\"#filterType_joinApproxTime .single-column-selector.master-time-col\").find(\":selected\").val()\n",
    "                tempCol1 = $(\"#filterType_joinApproxTime .single-column-selector.temp-col-1\").find(\":selected\").val()\n",
    "                tempCol2 = $(\"#filterType_joinApproxTime .single-column-selector.temp-col-2\").find(\":selected\").val() || null\n",
    "                tempCol3 = $(\"#filterType_joinApproxTime .single-column-selector.temp-col-3\").find(\":selected\").val() || null\n",
    "                masterCol1 = $(\"#filterType_joinApproxTime .single-column-selector.master-col-1\").find(\":selected\").val()\n",
    "                masterCol2 = $(\"#filterType_joinApproxTime .single-column-selector.master-col-2\").find(\":selected\").val() || null\n",
    "                masterCol3 = $(\"#filterType_joinApproxTime .single-column-selector.master-col-3\").find(\":selected\").val() || null\n",
    "                parameters.push(operator)\n",
    "                parameters.push([masterCol1, tempCol1])\n",
    "                parameters.push([masterCol2, tempCol2])\n",
    "                parameters.push([masterCol3, tempCol3])\n",
    "                parameters.push([masterTimeCol, tempTimeCol])\n",
    "                filterObj[\"parameters\"] = parameters\n",
    "                filterObj[\"descr\"] = operator + \"approx time join on [\" + masterTimeCol + \",\" + tempTimeCol + \"]: (\" + masterCol1 + \",\" + tempCol1 + \")\" + \"(\" + masterCol2 + \",\" + tempCol2 + \")\" + \"(\" + masterCol3 + \",\" + tempCol3 + \")\"\n",
    "                applied_filters[filter_count.toString()] = filterObj;\n",
    "                break;\n",
    "            }\n",
    "        }\n",
    "        show_filters();\n",
    "        divVisibility('null-filterType');\n",
    "        $(\"#filterType\")[0].selectedIndex = 0;\n",
    "\n",
    "    }\n",
    "\n",
    "    function resetAllFilters() {\n",
    "        applied_filters = []\n",
    "        filter_count = 0\n",
    "    }\n",
    "\n",
    "    //helper: adds options to help populate column dropdowns\n",
    "    function optionGenerator(instance, paramObj) {\n",
    "        instance\n",
    "            .append($(\"<option></option>\")\n",
    "                .attr(\"value\", paramObj)\n",
    "                .text(paramObj));\n",
    "    };\n",
    "</script>\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hideCode": true,
    "hideOutput": false,
    "hidePrompt": false
   },
   "source": [
    "### Temp Dataset Preview (first 40 rows)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hideCode": true,
    "hideOutput": false,
    "hidePrompt": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#P4\n",
    "\n",
    "if tempDF is not None:\n",
    "    tempDF.show(40, False)\n",
    "else:\n",
    "    print(\"------------- empty ------------------\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hideCode": true,
    "hideOutput": false,
    "hidePrompt": false
   },
   "source": [
    "### Master Dataset Preview (first  20 rows)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hideCode": true,
    "hideOutput": false,
    "hidePrompt": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#P6\n",
    "\n",
    "if masterDF is not None:\n",
    "    masterDF.show(20, False)\n",
    "else:\n",
    "    print(\"------------- empty ------------------\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hideCode": true,
    "hideOutput": false,
    "hidePrompt": false
   },
   "source": [
    "### Query Output\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hideCode": true,
    "hideOutput": false,
    "hidePrompt": false
   },
   "outputs": [],
   "source": [
    "#P8\n",
    "print(query)\n",
    "print(\"\\n***The code shown here corresponds to the processes that have been applied on the current temporary Dataframe (tempDF)\")\n"
   ]
  }
 ],
 "metadata": {
  "celltoolbar": "Hide code",
  "hide_code_all_hidden": false,
  "kernelspec": {
   "display_name": "PySpark",
   "language": "",
   "name": "pysparkkernel"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "python",
    "version": 2
   },
   "mimetype": "text/x-python",
   "name": "pyspark",
   "pygments_lexer": "python2"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {
     "167424505fd34435b6b4c97cb1b3e0a1": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.0.0",
      "model_name": "ButtonModel",
      "state": {
       "button_style": "primary",
       "description": "Click to initialise Query Builder",
       "layout": "IPY_MODEL_5e2143cd99ee4cd58e622c9d362a34ce",
       "style": "IPY_MODEL_a6407adcb36e4a10a3331a79fca72281"
      }
     },
     "227bb92092c748d4b1d826425ec45e45": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.0.0",
      "model_name": "ButtonStyleModel",
      "state": {}
     },
     "5e2143cd99ee4cd58e622c9d362a34ce": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.0.0",
      "model_name": "LayoutModel",
      "state": {
       "flex": "1 1 auto",
       "width": "20%"
      }
     },
     "66e5ae58e88d4d659ee5bde6f9de6efe": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.0.0",
      "model_name": "ButtonModel",
      "state": {
       "button_style": "primary",
       "description": "Click to initialise Query Builder",
       "layout": "IPY_MODEL_da5a0058efaf48ba9d15143bbcd4c32e",
       "style": "IPY_MODEL_227bb92092c748d4b1d826425ec45e45"
      }
     },
     "a6407adcb36e4a10a3331a79fca72281": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.0.0",
      "model_name": "ButtonStyleModel",
      "state": {}
     },
     "da5a0058efaf48ba9d15143bbcd4c32e": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.0.0",
      "model_name": "LayoutModel",
      "state": {
       "flex": "1 1 auto",
       "width": "20%"
      }
     }
    },
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}