pre-activation representation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.keras import Input, Model\n",
    "from tensorflow.keras.layers import Dense\n",
    "from tensorflow.keras.layers import BatchNormalization\n",
    "from tensorflow.keras.layers import Dropout\n",
    "from tensorflow.keras.layers import Flatten\n",
    "from tensorflow.keras.models import Model, load_model\n",
    "from tensorflow.keras.layers import Input\n",
    "from tensorflow.keras.layers import Activation\n",
    "from tensorflow.keras import optimizers\n",
    "from tensorflow.keras.callbacks import EarlyStopping, Callback, ModelCheckpoint\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from collections import Counter\n",
    "import re, os, sys, csv, math, operator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "architecture = '1024x4D-512x3D-256x3D-128x3D-64x2-32x1-1'\n",
    "activation = 'relu'\n",
    "dropouts = [0.8, 0.9, 0.7, 0.8]\n",
    "SEED = 1234567\n",
    "num_input = 86"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def define_model(data, architecture, num_labels=1, activation='relu', dropouts=[]):\n",
    "\n",
    "        assert '-' in architecture\n",
    "        archs = architecture.strip().split('-')\n",
    "        net = data\n",
    "        pen_layer = net\n",
    "        prev_layer = net\n",
    "        prev_num_outputs = None\n",
    "        prev_block_num_outputs = None\n",
    "        prev_stub_output = net\n",
    "        for i in range(len(archs)):\n",
    "            arch = archs[i]\n",
    "            if 'x' in arch:\n",
    "                arch = arch.split('x')\n",
    "                num_outputs = int(re.findall(r'\\d+',arch[0])[0])\n",
    "                layers = int(re.findall(r'\\d+',arch[1])[0])\n",
    "                j = 0\n",
    "                aux_layers = re.findall(r'[A-Z]',arch[0])\n",
    "                for l in range(layers):\n",
    "                    if aux_layers and aux_layers[0] == 'B':\n",
    "                        if len(aux_layers)>1 and aux_layers[1]=='A':\n",
    "                            print('adding fully connected layers with %d outputs followed by batch_norm and act' % num_outputs)\n",
    "\n",
    "                            net = Dense(num_outputs, \n",
    "                                        name='fc' + str(i) + '_' + str(j),\n",
    "                                        activation=None)(net)\n",
    "                            net = BatchNormalization(center=True, scale=True, name='fc_bn'+str(i)+'_'+str(j))(net)\n",
    "                            if activation =='relu': net = Activation('relu')(net)\n",
    "                        else:\n",
    "                            print('adding fully connected layers with %d outputs followed by batch_norm' % num_outputs)\n",
    "                            net = Dense(num_outputs,\n",
    "                                        name='fc' + str(i) + '_' + str(j),\n",
    "                                        activation=activation)(net)\n",
    "                            net = BatchNormalization(center=True, scale=True,\n",
    "                                             name='fc_bn' + str(i) + '_' + str(j))(net)\n",
    "\n",
    "                    else:\n",
    "                        print('adding fully connected layers with %d outputs' % num_outputs)\n",
    "\n",
    "                        net = Dense(num_outputs,\n",
    "                                    name='fc' + str(i) + '_' + str(j), \n",
    "                                    activation=None)(net)\n",
    "                        if activation =='relu': net = Activation('relu')(net)\n",
    "\n",
    "                    if 'R' in aux_layers:\n",
    "                        if prev_num_outputs and prev_num_outputs==num_outputs:\n",
    "                            print('adding residual, both sizes are same')\n",
    "\n",
    "                            net = net+prev_layer\n",
    "                        else:\n",
    "                            print('adding residual with fc as the size are different')\n",
    "                            net = net + Dense(num_outputs,\n",
    "                                                name='fc' + str(i) + '_' +'dim_'+ str(j),\n",
    "                                                activation=None)(prev_layer)\n",
    "                    prev_num_outputs = num_outputs\n",
    "                    j += 1\n",
    "                    prev_layer = net\n",
    "                aux_layers_sub = re.findall(r'[A-Z]', arch[1])\n",
    "                if 'R' in aux_layers_sub:\n",
    "                    if prev_block_num_outputs and prev_block_num_outputs == num_outputs:\n",
    "                        print('adding residual to stub, both sizes are same')\n",
    "                        net = net + prev_stub_output\n",
    "                    else:\n",
    "                        print('adding residual to stub with fc as the size are different')\n",
    "                        net = net + Dense(num_outputs,\n",
    "                                         name='fc' + str(i) + '_' + 'stub_dim_' + str(j),\n",
    "                                         activation=None)(prev_stub_output)\n",
    "\n",
    "                if 'D' in aux_layers_sub and (num_labels == 1) and len(dropouts) > i:\n",
    "                    print('adding dropout', dropouts[i])\n",
    "                    net = Dropout(1.-dropouts[i], seed=SEED)(net, training=False)\n",
    "                prev_stub_output = net\n",
    "                prev_block_num_outputs = num_outputs\n",
    "                prev_layer = net\n",
    "\n",
    "            else:\n",
    "                if 'R' in arch:\n",
    "                    act_fun = 'relu'\n",
    "                    print('using ReLU at last layer')  \n",
    "                else:\n",
    "                    act_fun = None\n",
    "                pen_layer = net\n",
    "                print('adding final layer with ' + str(num_labels) + ' output')\n",
    "                net = Dense(num_labels, name='fc' + str(i),\n",
    "                            activation=act_fun)(net)\n",
    "\n",
    "        return net"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "adding fully connected layers with 1024 outputs\n",
      "adding fully connected layers with 1024 outputs\n",
      "adding fully connected layers with 1024 outputs\n",
      "adding fully connected layers with 1024 outputs\n",
      "adding dropout 0.8\n",
      "adding fully connected layers with 512 outputs\n",
      "adding fully connected layers with 512 outputs\n",
      "adding fully connected layers with 512 outputs\n",
      "adding dropout 0.9\n",
      "adding fully connected layers with 256 outputs\n",
      "adding fully connected layers with 256 outputs\n",
      "adding fully connected layers with 256 outputs\n",
      "adding dropout 0.7\n",
      "adding fully connected layers with 128 outputs\n",
      "adding fully connected layers with 128 outputs\n",
      "adding fully connected layers with 128 outputs\n",
      "adding dropout 0.8\n",
      "adding fully connected layers with 64 outputs\n",
      "adding fully connected layers with 64 outputs\n",
      "adding fully connected layers with 32 outputs\n",
      "adding final layer with 1 output\n",
      "Model: \"ElemNet\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "elemental_fractions (InputLa [(None, 86)]              0         \n",
      "_________________________________________________________________\n",
      "fc0_0 (Dense)                (None, 1024)              89088     \n",
      "_________________________________________________________________\n",
      "fc0_1 (Dense)                (None, 1024)              1049600   \n",
      "_________________________________________________________________\n",
      "fc0_2 (Dense)                (None, 1024)              1049600   \n",
      "_________________________________________________________________\n",
      "fc0_3 (Dense)                (None, 1024)              1049600   \n",
      "_________________________________________________________________\n",
      "dropout (Dropout)            (None, 1024)              0         \n",
      "_________________________________________________________________\n",
      "fc1_0 (Dense)                (None, 512)               524800    \n",
      "_________________________________________________________________\n",
      "fc1_1 (Dense)                (None, 512)               262656    \n",
      "_________________________________________________________________\n",
      "fc1_2 (Dense)                (None, 512)               262656    \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "fc2_0 (Dense)                (None, 256)               131328    \n",
      "_________________________________________________________________\n",
      "fc2_1 (Dense)                (None, 256)               65792     \n",
      "_________________________________________________________________\n",
      "fc2_2 (Dense)                (None, 256)               65792     \n",
      "_________________________________________________________________\n",
      "dropout_2 (Dropout)          (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "fc3_0 (Dense)                (None, 128)               32896     \n",
      "_________________________________________________________________\n",
      "fc3_1 (Dense)                (None, 128)               16512     \n",
      "_________________________________________________________________\n",
      "fc3_2 (Dense)                (None, 128)               16512     \n",
      "_________________________________________________________________\n",
      "dropout_3 (Dropout)          (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "fc4_0 (Dense)                (None, 64)                8256      \n",
      "_________________________________________________________________\n",
      "fc4_1 (Dense)                (None, 64)                4160      \n",
      "_________________________________________________________________\n",
      "fc5_0 (Dense)                (None, 32)                2080      \n",
      "_________________________________________________________________\n",
      "fc6 (Dense)                  (None, 1)                 33        \n",
      "=================================================================\n",
      "Total params: 4,631,361\n",
      "Trainable params: 4,631,361\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "Loaded model from disk\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/vgf3011/.local/lib/python3.6/site-packages/tensorflow/python/keras/optimizer_v2/optimizer_v2.py:375: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
      "  \"The `lr` argument is deprecated, use `learning_rate` instead.\")\n"
     ]
    }
   ],
   "source": [
    "inputs = Input(shape=(num_input,), name='elemental_fractions')\n",
    "outputs = define_model(inputs, architecture, dropouts=dropouts)\n",
    "model = Model(inputs=inputs, outputs=outputs, name= 'ElemNet')\n",
    "model.summary(print_fn=lambda x: print(x))\n",
    "\n",
    "model.load_weights(\"../elemnet/model/model_elemnet_deltae_tf2.h5\")\n",
    "\n",
    "print(\"Loaded model from disk\")\n",
    "\n",
    "adam = optimizers.Adam(lr=0.0001)\n",
    "model.compile(loss=tf.keras.losses.mean_absolute_error, optimizer=adam, metrics=['mean_absolute_error'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Contains 86 elements (Without Noble elements as it does not forms compounds in normal condition)\n",
    "elements = ['H','Li','Be', 'B', 'C', 'N', 'O', 'F', 'Na', 'Mg', 'Al', 'Si', 'P', 'S', 'Cl',\n",
    "            'K', 'Ca', 'Sc', 'Ti', 'V', 'Cr', 'Mn', 'Fe','Co', 'Ni', 'Cu', 'Zn', 'Ga', 'Ge',\n",
    "            'As', 'Se', 'Br', 'Kr', 'Rb', 'Sr', 'Y', 'Zr', 'Nb', 'Mo', 'Tc', 'Ru', 'Rh', 'Pd',\n",
    "            'Ag', 'Cd', 'In', 'Sn', 'Sb', 'Te', 'I', 'Xe', 'Cs', 'Ba', 'La', 'Ce', 'Pr', 'Nd',\n",
    "            'Pm', 'Sm', 'Eu', 'Gd', 'Tb', 'Dy', 'Ho', 'Er','Tm', 'Yb', 'Lu', 'Hf', 'Ta', 'W', \n",
    "            'Re', 'Os', 'Ir', 'Pt', 'Au', 'Hg', 'Tl', 'Pb', 'Bi', 'Ac', 'Th', 'Pa', 'U', 'Np', 'Pu' ]\n",
    "\n",
    "elements_all = ['H', 'He', 'Li', 'Be', 'B', 'C', 'N', 'O', 'F', 'Ne', 'Na', 'Mg', 'Al', 'Si', \n",
    "                'P', 'S', 'Cl', 'Ar', 'K', 'Ca', 'Sc', 'Ti', 'V', 'Cr', 'Mn', 'Fe', 'Co', 'Ni',\n",
    "                'Cu', 'Zn', 'Ga', 'Ge', 'As', 'Se', 'Br', 'Kr', 'Rb', 'Sr', 'Y', 'Zr', 'Nb', \n",
    "                'Mo', 'Tc', 'Ru', 'Rh', 'Pd', 'Ag', 'Cd', 'In', 'Sn', 'Sb', 'Te', 'I', 'Xe',\n",
    "                'Cs', 'Ba', 'La', 'Ce', 'Pr', 'Nd', 'Pm', 'Sm', 'Eu', 'Gd', 'Tb', 'Dy', 'Ho',\n",
    "                'Er', 'Tm', 'Yb', 'Lu', 'Hf', 'Ta', 'W', 'Re', 'Os', 'Ir', 'Pt', 'Au', 'Hg',\n",
    "                'Tl', 'Pb', 'Bi', 'Po', 'At', 'Rn', 'Fr', 'Ra', 'Ac', 'Th', 'Pa', 'U', 'Np',\n",
    "                'Pu', 'Am', 'Cm', 'Bk', 'Cf', 'Es', 'Fm', 'Md', 'No', 'Lr', 'Rf', 'Db', 'Sg',\n",
    "                'Bh', 'Hs', 'Mt', 'Ds', 'Rg', 'Cn']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_path = '../data/sample'\n",
    "file_name = 'sample_train_set.csv'\n",
    "features_filename = 'sample'\n",
    "\n",
    "pred = pd.read_csv('{}/{}'.format(data_path, file_name)) \n",
    "pred_prop = pred[elements]\n",
    "\n",
    "new_x_pred = pred_prop.values\n",
    "new_x_pred = np.asarray(new_x_pred, dtype=np.float)\n",
    "\n",
    "y_pred = pred.pop('formation_energy_peratom').to_frame()\n",
    "new_y_pred = np.array(y_pred)\n",
    "new_y_pred.shape = (len(new_y_pred),)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "extractor = tf.keras.models.Model(inputs=model.inputs, outputs=[layer.output for layer in model.layers])\n",
    "features = extractor(new_x_pred)\n",
    "len(features)\n",
    "#composition = pred.pop('composition').to_frame()\n",
    "composition = pred.pop('pretty_comp').to_frame()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# The following are the values for num_layer if you want to extract features from a specific layer\n",
    "# layer 1 = 1\n",
    "# layer 2 = 2\n",
    "# layer 3 = 3\n",
    "# layer 4 = 4\n",
    "# layer 5 = 6\n",
    "# layer 6 = 7\n",
    "# layer 7 = 8\n",
    "# layer 8 = 10\n",
    "# layer 9 = 11\n",
    "# layer 10 = 12\n",
    "# layer 11 = 14\n",
    "# layer 12 = 15\n",
    "# layer 13 = 16\n",
    "# layer 14 = 18 \n",
    "# layer 15 = 19\n",
    "# layer 16 = 20\n",
    "\n",
    "layer_num = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "f_n = features[layer_num].numpy()\n",
    "df_n = pd.DataFrame(data=f_n)\n",
    "ndf_n = pd.concat([composition,df_n,y_pred], axis=1)\n",
    "path = '{}/{}_feat.csv'.format(data_path, features_filename)\n",
    "ndf_n.to_csv(path, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# In order to extract features from all the layers including after dropout use the following command"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(len(features)):\n",
    "    globals()[\"f\" + str(i)] = features[i].numpy()\n",
    "    globals()[\"df\" + str(i)] = pd.DataFrame(data=globals()[\"f\" + str(i)])\n",
    "    globals()[\"ndf\" + str(i)] = pd.concat([composition,globals()[\"df\" + str(i)],y_pred], axis=1)\n",
    "    path = '{}/{}_f{}.csv'.format(data_path, features_filename, i)\n",
    "    globals()[\"ndf\" + str(i)].to_csv(path, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}