Interactive_plots.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from collections import defaultdict\n",
    "import os\n",
    "\n",
    "import plotly.plotly as py\n",
    "import plotly.graph_objs as go\n",
    "\n",
    "import plotly\n",
    "from plotly.graph_objs import Scatter, Layout\n",
    "\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Load in colours and labels for plotting\n",
    "%store -r continents\n",
    "%store -r pop_by_continent\n",
    "%store -r indices_of_population_members\n",
    "%store -r name_by_code\n",
    "%store -r continent_by_population\n",
    "%store -r individuals\n",
    "%store -r population_by_individual\n",
    "%store -r individuals_by_population\n",
    "%store -r populations\n",
    "%store -r color_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#in_file = '/Volumes/Stockage/alex/1000G/projections/1000G_UMAP_PC20_NC3_NN50_MD0.5_201851182419'\n",
    "in_file = '/Volumes/Stockage/alex/1000G/projections/pca_1000g_100'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "in_data = np.loadtxt(in_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x_1000g = in_data[:,0]\n",
    "y_1000g = in_data[:,1]\n",
    "z_1000g = in_data[:,2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "color_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "traces = []\n",
    "\n",
    "for cont in continents:\n",
    "    for pop in pop_by_continent[cont]:\n",
    "        proj_subset = in_data[indices_of_population_members[pop]]\n",
    "        # Need to format the colour string correctly.\n",
    "        color_string = str(int('0x' + color_dict[pop][1:3].upper(),16)) + ',' + \\\n",
    "        str(int('0x' + color_dict[pop][3:5].upper(),16)) + ',' + \\\n",
    "        str(int('0x' + color_dict[pop][5:7].upper(),16))\n",
    "        temp_trace = go.Scatter3d(\n",
    "            x=proj_subset[:,0],\n",
    "            y=proj_subset[:,1],\n",
    "            z=proj_subset[:,2],\n",
    "            name=name_by_code[pop],\n",
    "            mode='markers',\n",
    "            marker=dict(\n",
    "                size=2,\n",
    "                color='rgba('+color_string+',0.9)',\n",
    "                line=dict(\n",
    "                    color='rgb('+color_string+')',\n",
    "                    width=2\n",
    "                )\n",
    "            )\n",
    "        )\n",
    "        traces.append(temp_trace)\n",
    "        \n",
    "layout = go.Layout(\n",
    "    margin=dict(\n",
    "        l=0,\n",
    "        r=0,\n",
    "        b=0,\n",
    "        t=0\n",
    "    )\n",
    ")\n",
    "\n",
    "fig = go.Figure(data=traces, layout=layout)\n",
    "plotly.offline.plot(fig, filename='test.html')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "in_data = np.loadtxt('/Volumes/Stockage/alex/1000G/projections/1000G_UMAP_PC20_NC3_NN15_MD0.5_20185118104')\n",
    "traces = []\n",
    "\n",
    "for cont in continents:\n",
    "    for pop in pop_by_continent[cont]:\n",
    "        proj_subset = in_data[indices_of_population_members[pop]]\n",
    "        # Need to format the colour string correctly.\n",
    "        color_string = str(int('0x' + color_dict[pop][1:3].upper(),16)) + ',' + \\\n",
    "        str(int('0x' + color_dict[pop][3:5].upper(),16)) + ',' + \\\n",
    "        str(int('0x' + color_dict[pop][5:7].upper(),16))\n",
    "        temp_trace = go.Scattergl(\n",
    "            x=proj_subset[:,0],\n",
    "            y=proj_subset[:,1],\n",
    "            name=name_by_code[pop],\n",
    "            mode='markers',\n",
    "            marker=dict(\n",
    "                size=2,\n",
    "                color='rgba('+color_string+',0.9)',\n",
    "                line=dict(\n",
    "                    color='rgb('+color_string+')',\n",
    "                    width=2\n",
    "                )\n",
    "            )\n",
    "        )\n",
    "        traces.append(temp_trace)\n",
    "        \n",
    "layout = go.Layout(\n",
    "    margin=dict(\n",
    "        l=0,\n",
    "        r=0,\n",
    "        b=0,\n",
    "        t=0\n",
    "    )\n",
    ")\n",
    "\n",
    "fig = go.Figure(data=traces, layout=layout)\n",
    "plotly.offline.plot(fig, filename='test_1000g_2d_gl.html')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3D plotting of UKBB data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%store -r ukbb_eth_dict_parent\n",
    "%store -r indices_of_population_members_ukbb\n",
    "%store -r color_dict_ukbb\n",
    "%store -r markers_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "{'Asian or Asian British': 'P',\n",
    " 'Black or Black British': 'P',\n",
    " 'Chinese': '^',\n",
    " 'Mixed': '*',\n",
    " 'NA': 'X',\n",
    " 'Other ethnic group': 's',\n",
    " 'White': 'o'}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "markers_dict_ukbb_3d = {'Asian or Asian British': 'cross',\n",
    " 'Black or Black British': 'cross',\n",
    " 'Chinese': 'cross',\n",
    " 'Mixed': 'diamond',\n",
    " 'NA': 'x',\n",
    " 'Other ethnic group': 'square-open',\n",
    " 'White': 'circle'}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#ukbb_umap_proj = np.loadtxt('/Volumes/Stockage/alex/ukbb_projections/3d/UKBB_UMAP_PC10_NC3_NN10_MD0.5_2018331162028')\n",
    "ukbb_umap_proj = np.loadtxt('/Volumes/Stockage/alex/ukbb_projections/ukbb_tsne_pc10_plex30_iter_1000_3d')\n",
    "\n",
    "ukbb_traces=[]\n",
    "temp_size=2\n",
    "\n",
    "for pop in ukbb_eth_dict_parent:\n",
    "    if pop in ['White','Mixed','Asian or Asian British','Black or Black British']:\n",
    "        temp_proj = ukbb_umap_proj[indices_of_population_members_ukbb[pop],:]\n",
    "        color_string = str(int('0x' + color_dict_ukbb[pop][1:3].upper(),16)) + ',' + \\\n",
    "        str(int('0x' + color_dict_ukbb[pop][3:5].upper(),16)) + ',' + \\\n",
    "        str(int('0x' + color_dict_ukbb[pop][5:7].upper(),16))\n",
    "        temp_trace = go.Scatter3d(\n",
    "            x = temp_proj[:,0],\n",
    "            y = temp_proj[:,1],\n",
    "            z = temp_proj[:,2],\n",
    "            name=pop,\n",
    "            mode='markers',\n",
    "            marker=dict(\n",
    "                symbol=markers_dict_ukbb_3d[pop],\n",
    "                size=temp_size,\n",
    "                color='rgba('+color_string+',0.3)',\n",
    "                line=dict(\n",
    "                    color='rgb('+color_string+')',\n",
    "                    width=2\n",
    "                )\n",
    "            )\n",
    "        )\n",
    "        ukbb_traces.append(temp_trace)\n",
    "\n",
    "    for subpop in ukbb_eth_dict_parent[pop]:\n",
    "        temp_proj = ukbb_umap_proj[indices_of_population_members_ukbb[subpop],:]\n",
    "        color_string = str(int('0x' + color_dict_ukbb[subpop][1:3].upper(),16)) + ',' + \\\n",
    "        str(int('0x' + color_dict_ukbb[subpop][3:5].upper(),16)) + ',' + \\\n",
    "        str(int('0x' + color_dict_ukbb[subpop][5:7].upper(),16))\n",
    "        temp_trace = go.Scatter3d(\n",
    "            x = temp_proj[:,0],\n",
    "            y = temp_proj[:,1],\n",
    "            z = temp_proj[:,2],\n",
    "            name=subpop,\n",
    "            mode='markers',\n",
    "            marker=dict(\n",
    "                symbol=markers_dict_ukbb_3d[pop],\n",
    "                size=temp_size,\n",
    "                color='rgba('+color_string+',0.3)',\n",
    "                line=dict(\n",
    "                    color='rgb('+color_string+')',\n",
    "                    width=2\n",
    "                )\n",
    "            )\n",
    "        )\n",
    "        ukbb_traces.append(temp_trace)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "layout = go.Layout(\n",
    "    margin=dict(\n",
    "        l=0,\n",
    "        r=0,\n",
    "        b=0,\n",
    "        t=0\n",
    "    )\n",
    ")\n",
    "\n",
    "fig = go.Figure(data=ukbb_traces, layout=layout)\n",
    "plotly.offline.plot(fig, filename='/Volumes/Stockage/alex/sandbox/test_3d_tsne.html')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Generalized 3d plotting for UKBB:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#ukbb_umap_proj = np.loadtxt('/Volumes/Stockage/alex/ukbb_projections/3d/UKBB_UMAP_PC10_NC3_NN10_MD0.5_2018331162028')\n",
    "#ukbb_umap_proj = np.loadtxt('/Volumes/Stockage/alex/ukbb_projections/ukbb_tsne_pc10_plex30_iter_1000_3d')\n",
    "\n",
    "proj_dir = '/Volumes/Stockage/alex/ukbb_projections'\n",
    "out_dir = '/Volumes/Stockage/alex/ukbb_3d_interactive'\n",
    "\n",
    "for fname in os.listdir(proj_dir):\n",
    "    if not 'NC3' in fname or os.path.exists(os.path.join(out_dir,fname+'.html')):\n",
    "        continue\n",
    "    else:\n",
    "        ukbb_proj = np.loadtxt(os.path.join(proj_dir,fname))\n",
    "        ukbb_traces=[]\n",
    "        temp_size=2\n",
    "\n",
    "        for pop in ukbb_eth_dict_parent:\n",
    "            if pop in ['White','Mixed','Asian or Asian British','Black or Black British']:\n",
    "                temp_proj = ukbb_proj[indices_of_population_members_ukbb[pop],:]\n",
    "                color_string = str(int('0x' + color_dict_ukbb[pop][1:3].upper(),16)) + ',' + \\\n",
    "                str(int('0x' + color_dict_ukbb[pop][3:5].upper(),16)) + ',' + \\\n",
    "                str(int('0x' + color_dict_ukbb[pop][5:7].upper(),16))\n",
    "                temp_trace = go.Scatter3d(\n",
    "                    x = temp_proj[:,0],\n",
    "                    y = temp_proj[:,1],\n",
    "                    z = temp_proj[:,2],\n",
    "                    name=pop,\n",
    "                    mode='markers',\n",
    "                    marker=dict(\n",
    "                        symbol=markers_dict_ukbb_3d[pop],\n",
    "                        size=temp_size,\n",
    "                        color='rgba('+color_string+',0.3)',\n",
    "                        line=dict(\n",
    "                            color='rgb('+color_string+')',\n",
    "                            width=2\n",
    "                        )\n",
    "                    )\n",
    "                )\n",
    "                ukbb_traces.append(temp_trace)\n",
    "\n",
    "            for subpop in ukbb_eth_dict_parent[pop]:\n",
    "                temp_proj = ukbb_proj[indices_of_population_members_ukbb[subpop],:]\n",
    "                color_string = str(int('0x' + color_dict_ukbb[subpop][1:3].upper(),16)) + ',' + \\\n",
    "                str(int('0x' + color_dict_ukbb[subpop][3:5].upper(),16)) + ',' + \\\n",
    "                str(int('0x' + color_dict_ukbb[subpop][5:7].upper(),16))\n",
    "                temp_trace = go.Scatter3d(\n",
    "                    x = temp_proj[:,0],\n",
    "                    y = temp_proj[:,1],\n",
    "                    z = temp_proj[:,2],\n",
    "                    name=subpop,\n",
    "                    mode='markers',\n",
    "                    marker=dict(\n",
    "                        symbol=markers_dict_ukbb_3d[pop],\n",
    "                        size=temp_size,\n",
    "                        color='rgba('+color_string+',0.3)',\n",
    "                        line=dict(\n",
    "                            color='rgb('+color_string+')',\n",
    "                            width=2\n",
    "                        )\n",
    "                    )\n",
    "                )\n",
    "                ukbb_traces.append(temp_trace)\n",
    "            \n",
    "            layout = go.Layout(\n",
    "            margin=dict(\n",
    "                l=0,\n",
    "                r=0,\n",
    "                b=0,\n",
    "                t=0\n",
    "            )\n",
    "        )\n",
    "\n",
    "        fig = go.Figure(data=ukbb_traces, layout=layout)\n",
    "        plotly.offline.plot(fig, filename=os.path.join(out_dir, fname + '.html'), auto_open=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3D plotting of HRS data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "hrs_data_dir = '/Volumes/Stockage/alex/hrs/projections'\n",
    "\n",
    "# Define the files we'll be using\n",
    "aux_file = 'allIndivs_filtered.txt'\n",
    "pc_files = ['plink.eigenvec_200','HRS_PCA_black.eigenvec','HRS_PCA_hispanic.eigenvec','HRS_PCA_white.eigenvec']\n",
    "pc_file = 'plink.eigenvec_200'\n",
    "\n",
    "aux_path = os.path.join(hrs_data_dir, aux_file)\n",
    "pc_path = os.path.join(hrs_data_dir, pc_file)\n",
    "\n",
    "# Import auxiliary data. Contains IDs and demographic information.\n",
    "# NOTE: The auxiliary data is sorted in an order different from the PC data.\n",
    "aux_data = []\n",
    "with open(aux_path) as input_file:\n",
    "    for line in input_file:\n",
    "        aux_data.append(line.strip().split(','))\n",
    "\n",
    "# Import PC data. This data must be converted to an array.\n",
    "with open(pc_path) as h:\n",
    "    hrs_contents = h.readlines()\n",
    "\n",
    "hrs_data = []\n",
    "\n",
    "for h in hrs_contents:\n",
    "    hrs_data.append(h.split()[2:len(h)])\n",
    "\n",
    "hrs_data_array = np.array(hrs_data).astype(np.float)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2D UKBB plotting (flawed but maybe useful)\n",
    "\n",
    "Two options:\n",
    "* go.Scatter - very slow, but correctly selects/deselects points\n",
    "* go.Scattergl - very fast, but point selection doesn't work as expected. Seems like a bug in JS when the dataset gets too large. Results are inconsistent but basically groups that are listed lower in the legend get removed by groups above them. This doesn't happen with other implementations and I can't figure out what the problem is. Doesn't seem to be related to which datasets are being used or whether they're introduced via loop or separate chunks. The colours also change occasionally, which leads me to believe that somewhere in there the indices are being rearranged."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ukbb_umap_proj = np.loadtxt('/Volumes/Stockage/alex/ukbb_projections/UKBB_UMAP_PC10_NN15_MD0.5_2018328174511')\n",
    "\n",
    "ukbb_traces=[]\n",
    "temp_alpha = 0.6\n",
    "temp_size = 3\n",
    "\n",
    "for pop in ukbb_eth_dict_parent:\n",
    "    if pop in ['White','Mixed','Asian or Asian British','Black or Black British']:\n",
    "        temp_proj = ukbb_umap_proj[indices_of_population_members_ukbb[pop],:]\n",
    "        color_string = str(int('0x' + color_dict_ukbb[pop][1:3].upper(),16)) + ',' + \\\n",
    "        str(int('0x' + color_dict_ukbb[pop][3:5].upper(),16)) + ',' + \\\n",
    "        str(int('0x' + color_dict_ukbb[pop][5:7].upper(),16))\n",
    "        temp_trace = go.Scatter(\n",
    "            x = temp_proj[:,0],\n",
    "            y = temp_proj[:,1],\n",
    "            name=pop,\n",
    "            mode='markers',\n",
    "            marker=dict(\n",
    "                symbol=markers_dict_ukbb_3d[pop],\n",
    "                size=temp_size,\n",
    "                color='rgba('+color_string+','+str(temp_alpha)+')',\n",
    "                #line=dict(\n",
    "                #    color='rgb('+color_string+')',\n",
    "                #    width=2\n",
    "                #)\n",
    "            )\n",
    "        )\n",
    "        ukbb_traces.append(temp_trace)\n",
    "\n",
    "    for subpop in ukbb_eth_dict_parent[pop]:\n",
    "        temp_proj = ukbb_umap_proj[indices_of_population_members_ukbb[subpop],:]\n",
    "        color_string = str(int('0x' + color_dict_ukbb[subpop][1:3].upper(),16)) + ',' + \\\n",
    "        str(int('0x' + color_dict_ukbb[subpop][3:5].upper(),16)) + ',' + \\\n",
    "        str(int('0x' + color_dict_ukbb[subpop][5:7].upper(),16))\n",
    "        temp_trace = go.Scatter(\n",
    "            x = temp_proj[:,0],\n",
    "            y = temp_proj[:,1],\n",
    "            name=subpop,\n",
    "            mode='markers',\n",
    "            marker=dict(\n",
    "                symbol=markers_dict_ukbb_3d[pop],\n",
    "                size=temp_size,\n",
    "                color='rgba('+color_string+','+str(temp_alpha)+')',\n",
    "                #line=dict(\n",
    "                #    color='rgb('+color_string+')',\n",
    "                #    width=2\n",
    "                #)\n",
    "            )\n",
    "        )\n",
    "        ukbb_traces.append(temp_trace)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "layout = go.Layout(\n",
    "    margin=dict(\n",
    "        l=0,\n",
    "        r=0,\n",
    "        b=0,\n",
    "        t=0\n",
    "    )\n",
    ")\n",
    "\n",
    "fig = go.Figure(data=ukbb_traces, layout=layout)\n",
    "plotly.offline.plot(fig, filename='/Volumes/Stockage/alex/sandbox/test_2d.html',auto_open=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Scatter GL\n",
    "ukbb_umap_proj = np.loadtxt('/Volumes/Stockage/alex/ukbb_projections/UKBB_UMAP_PC10_NN15_MD0.5_2018328174511')\n",
    "\n",
    "ukbb_traces=[]\n",
    "temp_alpha = 0.6\n",
    "temp_size = 3\n",
    "\n",
    "for pop in ukbb_eth_dict_parent:\n",
    "    if pop in ['White','Mixed','Asian or Asian British','Black or Black British']:\n",
    "        temp_proj = ukbb_umap_proj[indices_of_population_members_ukbb[pop],:]\n",
    "        color_string = str(int('0x' + color_dict_ukbb[pop][1:3].upper(),16)) + ',' + \\\n",
    "        str(int('0x' + color_dict_ukbb[pop][3:5].upper(),16)) + ',' + \\\n",
    "        str(int('0x' + color_dict_ukbb[pop][5:7].upper(),16))\n",
    "        temp_trace = go.Scattergl(\n",
    "            x = temp_proj[:,0],\n",
    "            y = temp_proj[:,1],\n",
    "            name=pop,\n",
    "            mode='markers',\n",
    "            marker=dict(\n",
    "                symbol=markers_dict_ukbb_3d[pop],\n",
    "                size=temp_size,\n",
    "                color='rgba('+color_string+','+str(temp_alpha)+')',\n",
    "                #line=dict(\n",
    "                #    color='rgb('+color_string+')',\n",
    "                #    width=2\n",
    "                #)\n",
    "            )\n",
    "        )\n",
    "        ukbb_traces.append(temp_trace)\n",
    "\n",
    "    for subpop in ukbb_eth_dict_parent[pop]:\n",
    "        temp_proj = ukbb_umap_proj[indices_of_population_members_ukbb[subpop],:]\n",
    "        color_string = str(int('0x' + color_dict_ukbb[subpop][1:3].upper(),16)) + ',' + \\\n",
    "        str(int('0x' + color_dict_ukbb[subpop][3:5].upper(),16)) + ',' + \\\n",
    "        str(int('0x' + color_dict_ukbb[subpop][5:7].upper(),16))\n",
    "        temp_trace = go.Scattergl(\n",
    "            x = temp_proj[:,0],\n",
    "            y = temp_proj[:,1],\n",
    "            name=subpop,\n",
    "            mode='markers',\n",
    "            marker=dict(\n",
    "                symbol=markers_dict_ukbb_3d[pop],\n",
    "                size=temp_size,\n",
    "                color='rgba('+color_string+','+str(temp_alpha)+')',\n",
    "                #line=dict(\n",
    "                #    color='rgb('+color_string+')',\n",
    "                #    width=2\n",
    "                #)\n",
    "            )\n",
    "        )\n",
    "        ukbb_traces.append(temp_trace)\n",
    "\n",
    "layout = go.Layout(\n",
    "    margin=dict(\n",
    "        l=0,\n",
    "        r=0,\n",
    "        b=0,\n",
    "        t=0\n",
    "    )\n",
    ")\n",
    "\n",
    "fig = go.Figure(data=ukbb_traces, layout=layout)\n",
    "plotly.offline.plot(fig, filename='/Volumes/Stockage/alex/sandbox/test_2d_gl.html',auto_open=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}