PandasExtras.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 104,
      "metadata": {
        "id": "z7tDEz9uTSOT"
      },
      "outputs": [],
      "source": [
        "#imports\n",
        "\n",
        "import numpy as np\n",
        "import pandas as pd\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 105,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0xBGhEpjrcSk",
        "outputId": "baadc4a7-df3b-4fb5-989a-fa14e4daf892"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "['one', 'two', 'three']\n",
            "one two three\n"
          ]
        }
      ],
      "source": [
        "# Refreshing on what [*'anything'] means\n",
        "arr = ['one','two','three']\n",
        "\n",
        "print(arr)\n",
        "\n",
        "print(*arr)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "M-sayhLNsoVR"
      },
      "source": [
        "For understading more about `(*)` please go through this link : [LINK](https://treyhunner.com/2018/10/asterisks-in-python-what-they-are-and-how-to-use-them/), it's very important that you understand what it does before continuing the notebook!!"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 106,
      "metadata": {
        "id": "OqxE3gOaTl2G"
      },
      "outputs": [],
      "source": [
        "# numpy random integer generator\n",
        "\n",
        "rng = np.random.default_rng(0) #Return random integers\n",
        "\n",
        "s = pd.Series(rng.integers(-10, +10, size=5))\n",
        "df1 = pd.DataFrame(rng.integers(-10, +10, size =(5,3)), columns=[*'abc'])\n",
        "df2 = pd.DataFrame(rng.integers(-10, +10, size =(5,3)), columns=[*'def'])\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "P3iprPEZTsn8"
      },
      "source": [
        "[`rng.integers()`](https://numpy.org/doc/stable/reference/random/generated/numpy.random.Generator.integers.html)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 107,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_FrX9AgfV45P",
        "outputId": "9a45095d-7776-4aae-97ce-6d841a4a09a1"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "0    7\n",
              "1    2\n",
              "2    0\n",
              "3   -5\n",
              "4   -4\n",
              "dtype: int64"
            ]
          },
          "execution_count": 107,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "s"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 108,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "VM5Mbo_FVsf_",
        "outputId": "e8091487-afa6-488f-8a1f-c3218632102c"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>a</th>\n",
              "      <th>b</th>\n",
              "      <th>c</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>-10</td>\n",
              "      <td>-9</td>\n",
              "      <td>-10</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>-7</td>\n",
              "      <td>6</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>8</td>\n",
              "      <td>0</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>9</td>\n",
              "      <td>4</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "    a  b   c\n",
              "0 -10 -9 -10\n",
              "1  -7  6   2\n",
              "2   8  0   2\n",
              "3   9  4   2\n",
              "4   0  1   8"
            ]
          },
          "execution_count": 108,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df1"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 109,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "ftD39uqhVp3C",
        "outputId": "8e555501-d75a-4c5e-8cae-ef7575564925"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>d</th>\n",
              "      <th>e</th>\n",
              "      <th>f</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>-5</td>\n",
              "      <td>6</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>-10</td>\n",
              "      <td>-3</td>\n",
              "      <td>7</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1</td>\n",
              "      <td>-10</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>4</td>\n",
              "      <td>6</td>\n",
              "      <td>-7</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>-9</td>\n",
              "      <td>7</td>\n",
              "      <td>-10</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "    d   e   f\n",
              "0  -5   6   3\n",
              "1 -10  -3   7\n",
              "2   1 -10   5\n",
              "3   4   6  -7\n",
              "4  -9   7 -10"
            ]
          },
          "execution_count": 109,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df2"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QOgqY7ZBV_qU"
      },
      "source": [
        "### Fun Part"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 110,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EKTZ1ybrV7kW",
        "outputId": "dd164358-0a68-4ba9-d163-2c5c615b1bbf"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "    a   b   c   0   1   2   3   4\n",
            "0 NaN NaN NaN NaN NaN NaN NaN NaN\n",
            "1 NaN NaN NaN NaN NaN NaN NaN NaN\n",
            "2 NaN NaN NaN NaN NaN NaN NaN NaN\n",
            "3 NaN NaN NaN NaN NaN NaN NaN NaN\n",
            "4 NaN NaN NaN NaN NaN NaN NaN NaN\n"
          ]
        }
      ],
      "source": [
        "print(s*df1)  # What in the Lord's Name is Going in Here"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 111,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IR2ijc8uWEZM",
        "outputId": "f6dc4e5f-98f4-4903-90a6-8aaaf5932763"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "    a   b   c   d   e   f\n",
            "0 NaN NaN NaN NaN NaN NaN\n",
            "1 NaN NaN NaN NaN NaN NaN\n",
            "2 NaN NaN NaN NaN NaN NaN\n",
            "3 NaN NaN NaN NaN NaN NaN\n",
            "4 NaN NaN NaN NaN NaN NaN\n"
          ]
        }
      ],
      "source": [
        "print(df1+df2)  # Why "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 112,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YSQMvecLWMGf",
        "outputId": "101fc45b-c970-4cc3-f95b-eb2dc61adf68"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "array([[-15,  -3,  -7],\n",
              "       [-17,   3,   9],\n",
              "       [  9, -10,   7],\n",
              "       [ 13,  10,  -5],\n",
              "       [ -9,   8,  -2]])"
            ]
          },
          "execution_count": 112,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "#Let's throw away the Pandas part !!\n",
        "\n",
        "df1.values + df2.values  # it works!!!!"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 113,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 167
        },
        "id": "X10JR2rTWZUq",
        "outputId": "afa062c2-dd63-4c41-c16e-4fa3f94b8f66"
      },
      "outputs": [
        {
          "ename": "ValueError",
          "evalue": "operands could not be broadcast together with shapes (5,) (5,3) ",
          "output_type": "error",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
            "\u001b[1;32m/home/vishu/JupyterNotebooks/PandasExtras.ipynb Cell 13\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> <a href='vscode-notebook-cell:/home/vishu/JupyterNotebooks/PandasExtras.ipynb#X15sZmlsZQ%3D%3D?line=0'>1</a>\u001b[0m s\u001b[39m.\u001b[39;49mvalues \u001b[39m+\u001b[39;49m df1\u001b[39m.\u001b[39;49mvalues\n",
            "\u001b[0;31mValueError\u001b[0m: operands could not be broadcast together with shapes (5,) (5,3) "
          ]
        }
      ],
      "source": [
        "s.values + df1.values # Returns an ERROR : Oh no no..."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Kwxvbf-GXd__",
        "outputId": "dd1a3b60-0642-4bb8-bd36-6008aa05fac7"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "array([[ 7],\n",
              "       [ 2],\n",
              "       [ 0],\n",
              "       [-5],\n",
              "       [-4]])"
            ]
          },
          "execution_count": 13,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# let's add a new axis to satify broadcasting rules!\n",
        "\n",
        "s.values[:, None]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TECvDaAaXwqP",
        "outputId": "d4ce878f-071b-4b59-c5d5-38c4ab6cbe46"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "(5,)\n"
          ]
        }
      ],
      "source": [
        "#Check if using `None` really add a new axis!!!\n",
        "\n",
        "print(s.values.shape)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nZMsqPHaYd0q",
        "outputId": "662d445c-e41a-4d7f-c8b0-aa6c112a297d"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "(5, 1)"
            ]
          },
          "execution_count": 15,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "s.values[:, None].shape  # we can see a new axis is added!!"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1CBj56XtYgOW",
        "outputId": "df02dc08-1157-4846-e09d-092405f0a2e3"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "array([[-3, -2, -3],\n",
              "       [-5,  8,  4],\n",
              "       [ 8,  0,  2],\n",
              "       [ 4, -1, -3],\n",
              "       [-4, -3,  4]])"
            ]
          },
          "execution_count": 16,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "s.values[:, None] + df1.values"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "aC7eGLEKYluq"
      },
      "outputs": [],
      "source": [
        "rng = np.random.default_rng(0)\n",
        "\n",
        "s = pd.Series(rng.integers(-10, +10, size=5))\n",
        "df1 = pd.DataFrame(rng.integers(-10, +10, size =(5,3)), columns=[*'abc'])\n",
        "df2 = pd.DataFrame(rng.integers(-10, +10, size =(5,3)), columns=[*'abc'])\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "s68xRS7bbdek",
        "outputId": "39ae60d9-0af7-4a4a-ac07-741bc38b9d0e"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-c99dda32-8bc4-445c-8e98-bcc8a92f2656\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>a</th>\n",
              "      <th>b</th>\n",
              "      <th>c</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>-10</td>\n",
              "      <td>-9</td>\n",
              "      <td>-10</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>-7</td>\n",
              "      <td>6</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>8</td>\n",
              "      <td>0</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>9</td>\n",
              "      <td>4</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-c99dda32-8bc4-445c-8e98-bcc8a92f2656')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-c99dda32-8bc4-445c-8e98-bcc8a92f2656 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-c99dda32-8bc4-445c-8e98-bcc8a92f2656');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "    a  b   c\n",
              "0 -10 -9 -10\n",
              "1  -7  6   2\n",
              "2   8  0   2\n",
              "3   9  4   2\n",
              "4   0  1   8"
            ]
          },
          "execution_count": 18,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df1"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 238
        },
        "id": "2eHUpgkvbkq8",
        "outputId": "90a3601e-f3b9-427a-b7e8-3362d6410d57"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-d96f5381-be62-4201-bed8-af8a77a3bdf2\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>b</th>\n",
              "      <th>c</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>a</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>-10</th>\n",
              "      <td>-9</td>\n",
              "      <td>-10</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>-7</th>\n",
              "      <td>6</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>0</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>4</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-d96f5381-be62-4201-bed8-af8a77a3bdf2')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-d96f5381-be62-4201-bed8-af8a77a3bdf2 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-d96f5381-be62-4201-bed8-af8a77a3bdf2');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "     b   c\n",
              "a         \n",
              "-10 -9 -10\n",
              "-7   6   2\n",
              " 0   1   8\n",
              " 8   0   2\n",
              " 9   4   2"
            ]
          },
          "execution_count": 19,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df1.groupby('a').sum()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "937tmFzSb_GT"
      },
      "source": [
        "[`.groupby()`](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.groupby.html) : A `groupby` operation involves some combination of splitting the object, applying a function, and combining the results"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 332
        },
        "id": "MxI8H5lvbpIK",
        "outputId": "6b24797e-9ab9-4044-c60f-50f2adc2b7b8"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-4f13457a-2d4e-453d-9380-228769292de8\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>a</th>\n",
              "      <th>b</th>\n",
              "      <th>c</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>-10</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>-7</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>NaN</td>\n",
              "      <td>7.0</td>\n",
              "      <td>11.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-4f13457a-2d4e-453d-9380-228769292de8')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-4f13457a-2d4e-453d-9380-228769292de8 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-4f13457a-2d4e-453d-9380-228769292de8');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "      a    b     c\n",
              "-10 NaN  NaN   NaN\n",
              "-7  NaN  NaN   NaN\n",
              " 0  NaN  7.0  11.0\n",
              " 1  NaN  NaN   NaN\n",
              " 2  NaN  NaN   NaN\n",
              " 3  NaN  NaN   NaN\n",
              " 4  NaN  NaN   NaN\n",
              " 8  NaN  NaN   NaN\n",
              " 9  NaN  NaN   NaN"
            ]
          },
          "execution_count": 20,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df1.groupby('a').sum() +df2 # Will this work?\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "az4pJXFGegcN"
      },
      "source": [
        "Cells that don't have any matching values with the another DataFrame are filled with `NaN`"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "c6iUsvrdcmXh",
        "outputId": "d050a365-fc05-4aa3-bff1-8954418776cd"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-18738096-74cd-4127-ab19-8fa91f757e9b\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>a</th>\n",
              "      <th>b</th>\n",
              "      <th>c</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>-15</td>\n",
              "      <td>-3</td>\n",
              "      <td>-7</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>-17</td>\n",
              "      <td>3</td>\n",
              "      <td>9</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1</td>\n",
              "      <td>-9</td>\n",
              "      <td>13</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>12</td>\n",
              "      <td>6</td>\n",
              "      <td>-5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0</td>\n",
              "      <td>11</td>\n",
              "      <td>-8</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-18738096-74cd-4127-ab19-8fa91f757e9b')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-18738096-74cd-4127-ab19-8fa91f757e9b button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-18738096-74cd-4127-ab19-8fa91f757e9b');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "    a   b   c\n",
              "0 -15  -3  -7\n",
              "1 -17   3   9\n",
              "2   1  -9  13\n",
              "3  12   6  -5\n",
              "4   0  11  -8"
            ]
          },
          "execution_count": 21,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df1.groupby('a').sum().reset_index()+df2 # This works!"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OAm4hMpNeAWL"
      },
      "source": [
        "[`reset_index()`](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.reset_index.html?highlight=reset_index#pandas.DataFrame.reset_index): Reset the index of the DataFrame, and use the default one instead. "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "GxcB0Tezcqh1"
      },
      "outputs": [],
      "source": [
        "rng = np.random.default_rng(0)\n",
        "\n",
        "s1 = pd.Series(rng.integers(-10, +10, size=4), index=[*'aabb'])\n",
        "s2 = pd.Series(rng.integers(-10, +10, size=4), index=[*'aabb'])\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "N8IR9zNegInH",
        "outputId": "d1f3aee9-6fa5-4cff-9e40-cc48104a6d00"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "a    7\n",
              "a    2\n",
              "b    0\n",
              "b   -5\n",
              "dtype: int64"
            ]
          },
          "execution_count": 23,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "s1 # series with 4 elements"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "e3hk9HrSgJCu",
        "outputId": "43892dea-740f-41cd-d243-7c1572d443ab"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "a    -4\n",
              "a   -10\n",
              "b    -9\n",
              "b   -10\n",
              "dtype: int64"
            ]
          },
          "execution_count": 24,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "s2"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XGiMo3bngJWy",
        "outputId": "3e1876e9-82d2-4849-8bae-cbaa2ed75772"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "a     3\n",
              "a    -8\n",
              "b    -9\n",
              "b   -15\n",
              "dtype: int64"
            ]
          },
          "execution_count": 25,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "s1+s2 # when we add ; we return 4 elements too!! "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "wQ570WRegNWO"
      },
      "outputs": [],
      "source": [
        "#Let's Change the index!!\n",
        "\n",
        "rng = np.random.default_rng(0)\n",
        "\n",
        "s1 = pd.Series(rng.integers(-10, +10, size=4), index=[*'aaab'])  \n",
        "s2 = pd.Series(rng.integers(-10, +10, size=4), index=[*'abbb'])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LAczIum3gXuZ",
        "outputId": "a8e15ea3-7c25-4bfb-dce2-0436e528cc7c"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "a    7\n",
              "a    2\n",
              "a    0\n",
              "b   -5\n",
              "dtype: int64"
            ]
          },
          "execution_count": 27,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "s1"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LKosRR9pgYoh",
        "outputId": "eef7181d-05ac-4b5c-8c8e-c05249893e0a"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "a    -4\n",
              "b   -10\n",
              "b    -9\n",
              "b   -10\n",
              "dtype: int64"
            ]
          },
          "execution_count": 28,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "s2"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SfktmW-vgZDY",
        "outputId": "29301fc7-5efa-4095-c690-7676c6bec1a6"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "a     3\n",
              "a    -2\n",
              "a    -4\n",
              "b   -15\n",
              "b   -14\n",
              "b   -15\n",
              "dtype: int64"
            ]
          },
          "execution_count": 29,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "s1+s2 # by default we had 4 elements each in s1 and s2  ; got 6 elements in return! How?"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "3930d5iigbNY"
      },
      "outputs": [],
      "source": [
        "#Let's Change the index again to see if we can find some more strange behaviour!!\n",
        "\n",
        "rng = np.random.default_rng(0)\n",
        "\n",
        "s1 = pd.Series(rng.integers(-10, +10, size=4), index=[*'aabb'])  \n",
        "s2 = pd.Series(rng.integers(-10, +10, size=4), index=[*'abbb'])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dXqEaxJWg8Sb",
        "outputId": "5271759c-d8dd-4cff-8cb9-baf6381f335d"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "a     3\n",
              "a    -2\n",
              "b   -10\n",
              "b    -9\n",
              "b   -10\n",
              "b   -15\n",
              "b   -14\n",
              "b   -15\n",
              "dtype: int64"
            ]
          },
          "execution_count": 31,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "s1+s2 # returns 8 elements when have 4 elements in each of the two series(s1,s2)!"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 175
        },
        "id": "6ZatSqmPg92_",
        "outputId": "c21101fa-4e32-4470-b786-480576043ab6"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-247e861e-c106-4c0d-b614-ee85930773df\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>index</th>\n",
              "      <th>0</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>aa</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>ab</td>\n",
              "      <td>-8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>bb</td>\n",
              "      <td>-9</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>bb</td>\n",
              "      <td>-15</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-247e861e-c106-4c0d-b614-ee85930773df')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-247e861e-c106-4c0d-b614-ee85930773df button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-247e861e-c106-4c0d-b614-ee85930773df');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "  index   0\n",
              "0    aa   3\n",
              "1    ab  -8\n",
              "2    bb  -9\n",
              "3    bb -15"
            ]
          },
          "execution_count": 32,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# let's try a good ol' reset_index()!\n",
        "\n",
        "s1.reset_index()+s2.reset_index() # this returns the same elements as we have defined for our two series but the indexing dosen't make any sense!"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0IctRcIthf1E",
        "outputId": "e1860f40-4020-401c-96f1-4f80fc61bc60"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "0     3\n",
              "1    -8\n",
              "2    -9\n",
              "3   -15\n",
              "dtype: int64"
            ]
          },
          "execution_count": 33,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "s1.reset_index(drop=True)+s2.reset_index(drop=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "BP_vVjJIht26"
      },
      "outputs": [],
      "source": [
        "# imports \n",
        "\n",
        "from numpy import repeat \n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "g999kt9UZiDi"
      },
      "source": [
        "`numpy.repeat`:https://numpy.org/doc/stable/reference/generated/numpy.repeat.html"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Az0FIj5Ei2i1"
      },
      "outputs": [],
      "source": [
        "rng = np.random.default_rng(0)\n",
        "size = 8\n",
        "d = {\n",
        "    'a': repeat([True,False],(size)//2),\n",
        "     'b': rng.integers(-10,+10, size=size)\n",
        "}"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "KwtfHzyjjybY"
      },
      "outputs": [],
      "source": [
        "df = pd.DataFrame(data=d)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 300
        },
        "id": "fC9ZqjXcjzL6",
        "outputId": "3f80390f-4c26-4a58-cdeb-f8c9743529e6"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-64d22eec-ba64-497f-a81e-3aff40a5aed5\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>a</th>\n",
              "      <th>b</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>True</td>\n",
              "      <td>7</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>True</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>True</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>True</td>\n",
              "      <td>-5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>False</td>\n",
              "      <td>-4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>False</td>\n",
              "      <td>-10</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>False</td>\n",
              "      <td>-9</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>False</td>\n",
              "      <td>-10</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-64d22eec-ba64-497f-a81e-3aff40a5aed5')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-64d22eec-ba64-497f-a81e-3aff40a5aed5 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-64d22eec-ba64-497f-a81e-3aff40a5aed5');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "       a   b\n",
              "0   True   7\n",
              "1   True   2\n",
              "2   True   0\n",
              "3   True  -5\n",
              "4  False  -4\n",
              "5  False -10\n",
              "6  False  -9\n",
              "7  False -10"
            ]
          },
          "execution_count": 37,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 143
        },
        "id": "ArMeij9oj2hk",
        "outputId": "e0aef155-90ed-4922-82cb-7cd9b3273f2b"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-a35d8a30-794c-4c1a-a1c9-cc2ce9ff2b21\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>b</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>a</th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>False</th>\n",
              "      <td>-33</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>True</th>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-a35d8a30-794c-4c1a-a1c9-cc2ce9ff2b21')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-a35d8a30-794c-4c1a-a1c9-cc2ce9ff2b21 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-a35d8a30-794c-4c1a-a1c9-cc2ce9ff2b21');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "        b\n",
              "a        \n",
              "False -33\n",
              "True    4"
            ]
          },
          "execution_count": 38,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Let's find out all the place where `a` is True and where `a` is False\n",
        "\n",
        "df.groupby('a').sum()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 300
        },
        "id": "rIuoR8q9mDu0",
        "outputId": "eeb967bf-0803-4272-9cc8-8230b2515363"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-280295d4-e408-4245-80c9-fbbc396d6808\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>b</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0.711103</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0.711103</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0.711103</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0.711103</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>3.412305</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>3.412305</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>3.412305</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>3.412305</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-280295d4-e408-4245-80c9-fbbc396d6808')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-280295d4-e408-4245-80c9-fbbc396d6808 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-280295d4-e408-4245-80c9-fbbc396d6808');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "          b\n",
              "0  0.711103\n",
              "1  0.711103\n",
              "2  0.711103\n",
              "3  0.711103\n",
              "4  3.412305\n",
              "5  3.412305\n",
              "6  3.412305\n",
              "7  3.412305"
            ]
          },
          "execution_count": 39,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Let's say we want to find  kurtosis value in the above group-by!\n",
        "\n",
        "\n",
        "# Method 1 : Using `transform`\n",
        "\n",
        "\n",
        "df.groupby('a').transform(lambda x: x.kurt()) # This return 8 values , why?"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "InrgzXypoDLK"
      },
      "source": [
        "[`.transform()`](https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.core.groupby.DataFrameGroupBy.transform.html?highlight=transform#pandas.core.groupby.DataFrameGroupBy.transform) : Call function producing a like-indexed DataFrame on each group and return a DataFrame having the same indexes as the original object filled with the transformed values."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 143
        },
        "id": "bXr2YPE4nOCd",
        "outputId": "0950f244-a454-4090-e497-2c3229a41d09"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-4ba4ecde-6af3-4f9a-9882-97bd7d4597f9\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>a</th>\n",
              "      <th>b</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>a</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>False</th>\n",
              "      <td>0.0</td>\n",
              "      <td>3.412305</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>True</th>\n",
              "      <td>0.0</td>\n",
              "      <td>0.711103</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-4ba4ecde-6af3-4f9a-9882-97bd7d4597f9')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-4ba4ecde-6af3-4f9a-9882-97bd7d4597f9 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-4ba4ecde-6af3-4f9a-9882-97bd7d4597f9');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "         a         b\n",
              "a                   \n",
              "False  0.0  3.412305\n",
              "True   0.0  0.711103"
            ]
          },
          "execution_count": 40,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Method 2 : Using `apply`\n",
        "\n",
        "df.groupby('a').apply(lambda x: x.kurt()) # now, its getting really bizzare and confusing!"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "toC2s06doeXF"
      },
      "source": [
        "[`.apply()`](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.apply.html): Apply a function along an axis of the DataFrame."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 143
        },
        "id": "okkMTAI6nfAF",
        "outputId": "29cf8789-1be1-45a5-9f2e-e9d6b6a707f7"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-246e510a-7637-48f5-8a8b-a37b05634583\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>b</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>a</th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>False</th>\n",
              "      <td>3.412305</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>True</th>\n",
              "      <td>0.711103</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-246e510a-7637-48f5-8a8b-a37b05634583')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-246e510a-7637-48f5-8a8b-a37b05634583 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-246e510a-7637-48f5-8a8b-a37b05634583');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "              b\n",
              "a              \n",
              "False  3.412305\n",
              "True   0.711103"
            ]
          },
          "execution_count": 41,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Method 3 : Using `agg`\n",
        "\n",
        "df.groupby('a').agg(lambda x: x.kurt())  # Finally, something that works!!!!"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yiNVzuGxovTf"
      },
      "source": [
        "[`.agg()`](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.agg.html):Aggregate using one or more operations over the specified axis."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3CroSGB2qWEw"
      },
      "source": [
        "But wait! Do we even know what kurtosis values are ? And, why are we doing this ? "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Nv8pO256peJu"
      },
      "source": [
        "### The Big Picture:\n",
        "\n",
        "Maybe its worth to go through all this trouble because the `DataFrame` gives us the ability to store multiple one-dimensional data sets, at the cost of code-complexity!"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hrNe1uLRtlFz"
      },
      "source": [
        "### Tradeoff : NumPy v/s Python\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ARVe9M--nwbl",
        "outputId": "0c620cdb-9da8-419f-dac2-39a68089b25e"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "sleep                   elasped 𝚫t: 1.001149\n"
          ]
        }
      ],
      "source": [
        "# Reason to use Numpy\n",
        "\n",
        "from time import perf_counter, sleep\n",
        "from contextlib import contextmanager\n",
        "from random import randint as py_randint\n",
        "from numpy.random import randint as np_randint\n",
        "from numpy import dot as np_dot\n",
        "\n",
        "### This is just a timer function : N0 need to understand this !!!\n",
        "\n",
        "\n",
        "@contextmanager\n",
        "def  timed(msg):\n",
        "  start = perf_counter()\n",
        "  try:\n",
        "      yield\n",
        "  finally:\n",
        "      stop = perf_counter()\n",
        "      print(f'{msg:<24}elasped \\N{mathematical bold capital delta}t: {stop-start:.6f}')\n",
        "\n",
        "\n",
        "#Checking if the timer function works correctly!\n",
        "with timed('sleep'):\n",
        "  sleep(1)\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YDtFEhFMumNf",
        "outputId": "fb9ad2a6-1979-4b09-8733-4bded16e5324"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "py: create              elasped 𝚫t: 0.285596\n",
            "np: create              elasped 𝚫t: 0.004485\n"
          ]
        }
      ],
      "source": [
        "# Global variable are written in all-capital\n",
        "\n",
        "SIZE = 100_000 # Using underscore for readability purpose\n",
        "\n",
        "with timed('py: create'):\n",
        "    py_xs = [py_randint(-10,+10) for _ in range(SIZE)]\n",
        "    py_ys = [py_randint(-10, +10)for _  in range(SIZE)]\n",
        "\n",
        "\n",
        "with timed('np: create'):\n",
        "    np_xs = np_randint(-10, +10, size=SIZE)\n",
        "    np_ys = np_randint(-10, +10, size=SIZE)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4Z-1ZhAo4zHo",
        "outputId": "66e499cb-5ae2-4299-fa34-9079a20dd7fb"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "py: dot                 elasped 𝚫t: 0.017085\n",
            "np: dot                 elasped 𝚫t: 0.000793\n",
            "np: dot(py data)        elasped 𝚫t: 0.024950\n"
          ]
        }
      ],
      "source": [
        "## Dot product in python\n",
        "\n",
        "py_dot = lambda xs,ys: sum(x * y for x,y in zip(xs,ys)) # take each of the value of x's and y's -> pair them up -> multiply them -> sum the result\n",
        "\n",
        "with timed('py: dot'):\n",
        "    py_dot(py_xs, py_ys)\n",
        "\n",
        "with timed('np: dot'):\n",
        "    np_dot(np_xs, np_ys) # we are almost 70x faster than python\n",
        "\n",
        "# Mixing numpy + python\n",
        "\n",
        "with timed('np: dot(py data)'):\n",
        "    py_dot(np_xs,np_ys) # we can see that we're are losing that speed when we mix Python with Numpy!"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MzFuaH7f7Uvv"
      },
      "source": [
        "### Diggin Deep into NumPy"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kpGgnGC_7UH7",
        "outputId": "0d8e2bda-b00b-4206-a5d9-ad284651c7c9"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "xs                                = [0 1 2 3]\n",
            "xs.__array_interface__[\"data\"][0] = 41000960\n",
            "xs.dtype                          = int64\n",
            "xs.shape                          = (4,)\n",
            "xs.strides                        = (8,)\n"
          ]
        }
      ],
      "source": [
        "from numpy import array\n",
        "\n",
        "xs = array([0,1,2,3])\n",
        "# A np.ndarray is an interpreted view of raw memory\n",
        "print(\n",
        "    f'xs                                = {xs                                }',\n",
        "    f'xs.__array_interface__[\"data\"][0] = {xs.__array_interface__[\"data\"][0] }',\n",
        "    f'xs.dtype                          = {xs.dtype                           }',\n",
        "    f'xs.shape                          = {xs.shape                           }',\n",
        "    f'xs.strides                        = {xs.strides                         }',\n",
        "    sep='\\n',\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wq3eTk6q5m48",
        "outputId": "c3f0b50e-37ed-4220-f9a3-5d570f66f032"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "xs+ys   = [1, 2, 3, 4, 5, 6]\n",
            "xs * 3  = [1, 2, 3, 1, 2, 3, 1, 2, 3]\n"
          ]
        }
      ],
      "source": [
        "#Python\n",
        "\n",
        "# Python list is some sort of opaque collection of items!\n",
        "xs = [1,2,3]\n",
        "ys = [4,5,6]\n",
        "\n",
        "print(f'xs+ys   = {xs+ys}')\n",
        "print(f'xs * 3  = {xs * 3}')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6tHCK1yb9utd",
        "outputId": "c2bdfda6-83ee-4fc4-dbf5-3f0cc25ae547"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "xs+ys   = [5 7 9]\n",
            "xs * 3  = [3 6 9]\n"
          ]
        }
      ],
      "source": [
        "# Same operations is Numpy\n",
        "\n",
        "from numpy import array\n",
        "# `numpy` provides us with a \"mathematicial type- what we called vector or matrix or tensor -with corresponding operations \"\n",
        "# np array provide us with : fixed size , dynamic shape , higher-dimensional structure\n",
        "\n",
        "xs = array([1,2,3])\n",
        "ys = array([4,5,6])\n",
        "\n",
        "print(f'xs+ys   = {xs+ys}')\n",
        "print(f'xs * 3  = {xs * 3}')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 117,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Kyu7gvQ4AANh",
        "outputId": "b3e0e6a1-5efe-4eb2-f391-480565f317c0"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "xs[0][1]  = 1\n",
            "xs[0]     = [0, 1, 2]\n",
            "[1, 4, 7]\n"
          ]
        }
      ],
      "source": [
        "xs = [\n",
        "    [0,1,2],\n",
        "    [3,4,5],\n",
        "    [6,7,8],\n",
        "]\n",
        "\n",
        "print(\n",
        "    f'xs[0][1]  = {xs[0][1]}',\n",
        "    f'xs[0]     = {xs[0]}',\n",
        "    f'{[row[1]for row in xs]}', # Accessing column; although it may look like we're providing two coordinates, we're not! we just provide 1 coordinate - its not a 2d structure but a nested list; we're just doing 2 lookups on two list!\n",
        "    sep='\\n',\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 118,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rtcjuqumAkGe",
        "outputId": "8e08aab8-500d-429f-bfe9-56db307ea246"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "xs[0][1]       =  1\n",
            "xs[2][2][2][0] = 10\n"
          ]
        }
      ],
      "source": [
        "# Python list is never 2D data but it can be nested , has no mathematical meaning\n",
        "\n",
        "xs = [\n",
        "    [0,1,2],\n",
        "    [3,4,5],\n",
        "    [6,7,[8,9,[10]]],\n",
        "]\n",
        "\n",
        "print(\n",
        "    f'xs[0][1]       =  {xs[0][1]}',\n",
        "    f'xs[2][2][2][0] = {xs[2][2][2][0]}',\n",
        "    sep='\\n',\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 119,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GhCUQht6EZg-",
        "outputId": "9591c524-6175-412d-fbec-cb04ed3124d9"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Memory location = 139928223879360\n",
            "Memory location = 139928223879360\n",
            "xs              = [1, 2, 3, 4, 5, 6]\n",
            "ys              = [1, 2, 3, 4, 5, 6]\n"
          ]
        }
      ],
      "source": [
        "# One major thing that a `Python list` provides which a Numpy `ndarray` dosen't provide us is the ability to update the size!\n",
        "\n",
        "\n",
        "xs = ys = [1,2,3,4]\n",
        "xs.append(5)\n",
        "ys.append(6)\n",
        "\n",
        "\n",
        "print(\n",
        "    f'Memory location = {id(xs)}',  # Same memory location for xs and ys\n",
        "    f'Memory location = {id(ys)}',\n",
        "    f'xs              = {xs}',\n",
        "    f'ys              = {ys}',\n",
        "    sep='\\n',\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 120,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rLcTKGT8FXjk",
        "outputId": "1a4530f3-fbd0-4cc5-a9c2-4b5eb4277ab2"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "a               = [1 2]\n",
            "b               = [3 4]\n",
            "np.hstack((a,b))= [1 2 3 4]\n"
          ]
        }
      ],
      "source": [
        "# Numpy\n",
        "\n",
        "from numpy import array, hstack\n",
        "\n",
        "\n",
        "# quick showing of what hstack does\n",
        "\n",
        "a = np.array((1,2))\n",
        "b = np.array((3,4))\n",
        "print(\n",
        "    f'a               = {a}',\n",
        "    f'b               = {b}',\n",
        "    f'np.hstack((a,b))= {np.hstack((a,b))}',\n",
        "    sep='\\n',\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8c7Vi7tEGWJ4"
      },
      "source": [
        "For more info about `hstack` : [LINK](https://numpy.org/doc/stable/reference/generated/numpy.hstack.html)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 121,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Y2L6dpaLGUDe",
        "outputId": "e62aeb82-5d74-4448-a620-89bee4a42b62"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "xs                                                   = [0 1 2 3]\n",
            "ys                                                   = [0 1 2]\n",
            "xs.__array_interface__[\"data\"][0] or Memory location = 94394930057472\n",
            "ys.__array_interface__[\"data\"][0] or Memory location = 94394931086640\n"
          ]
        }
      ],
      "source": [
        "xs = ys = array([0,1,2]) \n",
        "xs = hstack([xs,[3]])\n",
        "\n",
        "print(\n",
        "    f'xs                                                   = {xs}',\n",
        "    f'ys                                                   = {ys}',\n",
        "    f'xs.__array_interface__[\"data\"][0] or Memory location = {xs.__array_interface__[\"data\"][0]}', # xs and ys are at two different blocks of memory unlike python\n",
        "    f'ys.__array_interface__[\"data\"][0] or Memory location = {ys.__array_interface__[\"data\"][0]}',\n",
        "    sep='\\n',\n",
        "    \n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IpxhjC_zJeyx"
      },
      "source": [
        "### Concluding python v/s numpy\n",
        "\n",
        "Python `list`:\n",
        "- fixed \"shape\" (linear) # Always 1-dimenisonal\n",
        "- dynamic size\n",
        "\n",
        "`numpy.ndarry`:\n",
        "- fixed size\n",
        "- dynamic shape"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 122,
      "metadata": {
        "id": "uIEEpyWSHfEZ"
      },
      "outputs": [],
      "source": [
        "from xarray import DataArray\n",
        "from numpy.random import default_rng\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 123,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_y8WoJPEPiCI",
        "outputId": "cb34ba51-e62e-46ad-e218-4d65d7ceacd7"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "<xarray.DataArray (x: 3, y: 3)>\n",
            "array([[240, 213,  37],\n",
            "       [122, 206,  57],\n",
            "       [ 52, 107,  11]], dtype=uint8)\n",
            "Dimensions without coordinates: x, y\n",
            "────────────────────────────────────────────────────────────\n",
            "img.sel(x=0)             = [240 213  37]\n",
            "────────────────────────────────────────────────────────────\n",
            "img.sel(x=0,y=1)         = 213\n",
            "────────────────────────────────────────────────────────────\n",
            "img.sel(x=[0,1],y=[0,1]) = [[240 213]\n",
            " [122 206]]\n"
          ]
        }
      ],
      "source": [
        "img = DataArray(\n",
        "    data=rng.integers(0,255, size=(3,3), dtype='uint8'),\n",
        "    dims=[*'xy'],\n",
        ")\n",
        "\n",
        "print(\n",
        "    img,\n",
        "    f'img.sel(x=0)             = {img.sel(x=0)}',\n",
        "    f'img.sel(x=0,y=1)         = {img.sel(x=0,y=1)}',\n",
        "    f'img.sel(x=[0,1],y=[0,1]) = {img.sel(x=[0,1],y=[0,1])}',\n",
        "    sep='\\n{}\\n'.format('\\N{box drawings light horizontal}' * 60),\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cVUTQ-YySfH4"
      },
      "source": [
        "[`xarray.DataArray`](https://docs.xarray.dev/en/stable/generated/xarray.DataArray.html): DataArray provides a wrapper around numpy ndarrays that uses labeled dimensions and coordinates to support metadata aware operations.\n",
        "\n",
        "- Go to official docs to learn more about xarray : https://docs.xarray.dev/en/stable/getting-started-guide/why-xarray.html"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 125,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WsoUbFEwQabf",
        "outputId": "b020892b-6157-41be-f845-58b20f0c9d61"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "<xarray.DataArray (x: 3, y: 3)>\n",
            "array([[ 94, 129, 193],\n",
            "       [216, 206, 234],\n",
            "       [ 14, 162,  32]], dtype=uint8)\n",
            "Coordinates:\n",
            "  * x        (x) float64 10.0 15.0 20.0\n",
            "  * y        (y) float64 10.0 15.0 20.0\n",
            "────────────────────────────────────────────────────────────\n",
            "<xarray.DataArray ()>\n",
            "array(206, dtype=uint8)\n",
            "Coordinates:\n",
            "    x        float64 15.0\n",
            "    y        float64 15.0\n",
            "────────────────────────────────────────────────────────────\n",
            "<xarray.DataArray ()>\n",
            "array(206, dtype=uint8)\n",
            "Coordinates:\n",
            "    x        float64 15.0\n",
            "    y        float64 15.0\n",
            "────────────────────────────────────────────────────────────\n",
            "<xarray.DataArray (y: 6)>\n",
            "array([212. , 210. , 208. , 206. , 211.6, 217.2])\n",
            "Coordinates:\n",
            "    x        int64 15\n",
            "  * y        (y) int64 12 13 14 15 16 17\n"
          ]
        }
      ],
      "source": [
        "# Some functionalitites of xarray ! \n",
        "\n",
        "from xarray import DataArray\n",
        "from numpy import linspace\n",
        "from numpy.random import default_rng\n",
        "\n",
        "\n",
        "\n",
        "rng = default_rng(0)\n",
        "data= rng.integers(0,255,size=(3,3), dtype='uint8')\n",
        "img = DataArray(\n",
        "    data= data,\n",
        "    dims=[*'xy'],\n",
        "    coords={ # we can define our own coordinate system\n",
        "        'x': linspace(10, 20, data.shape[0]),\n",
        "        'y': linspace(10, 20, data.shape[1]),\n",
        "    },\n",
        ")\n",
        "\n",
        "print(\n",
        "    img,\n",
        "    img.sel(x=15, y=15), #instead of picking values by default instead we're here asking to return a value that is at a specific location in our defined coordinate system\n",
        "    img.sel(x=15, y=13, method='nearest'), # if we can't find any value at the specified coordinate then return the nearest value\n",
        "    img.interp(x=15, y=range(12,18), method='linear'), # find data at give coordinate and if you're missing data then do a linear interpolation\n",
        "    sep='\\n{}\\n'.format('\\N{box drawings light horizontal}' * 60),\n",
        "    \n",
        ")\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QRP1w6k9X0Iu"
      },
      "source": [
        "[`xarray.DataArray.interp`](https://docs.xarray.dev/en/stable/generated/xarray.DataArray.interp.html#xarray.DataArray.interp):Interpolate a DataArray onto new coordinates"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CA_gPejYZ1t4"
      },
      "source": [
        "### Why Pandas?"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 126,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kwL_Nb4FX8nQ",
        "outputId": "0ecda166-9aa9-4c25-eebb-98f049a39f43"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "0    7\n",
            "1    2\n",
            "2    0\n",
            "3   -5\n",
            "4   -4\n",
            "dtype: int64\n",
            "\n",
            "<PandasArray>\n",
            "[7, 2, 0, -5, -4]\n",
            "Length: 5, dtype: int64\n",
            "\n",
            "[ 7  2  0 -5 -4]\n"
          ]
        }
      ],
      "source": [
        "from pandas import Series\n",
        "from numpy.random import default_rng\n",
        "\n",
        "rng = default_rng(0)\n",
        "\n",
        "s = Series(rng.integers(-10, +10, size=5))\n",
        "\n",
        "print(\n",
        "    s,\n",
        "    s.array,\n",
        "    s.array._ndarray,\n",
        "    sep='\\n\\n'\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cRt78SlDa2Lt"
      },
      "source": [
        "[`pandas.array`](https://pandas.pydata.org/docs/reference/api/pandas.array.html):Create an array."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 127,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ctxXFzIhaJTQ",
        "outputId": "fcacf17c-5c6b-45ee-a78b-15ac92936255"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "date\n",
            "2022-01-01    7\n",
            "2022-01-02    2\n",
            "2022-01-03    0\n",
            "2022-01-04   -5\n",
            "2022-01-05   -4\n",
            "Freq: D, dtype: int64\n",
            "────────────────────────────────────────────────────────────\n",
            "<PandasArray>\n",
            "[7, 2, 0, -5, -4]\n",
            "Length: 5, dtype: int64\n",
            "────────────────────────────────────────────────────────────\n",
            "DatetimeIndex(['2022-01-01', '2022-01-02', '2022-01-03', '2022-01-04',\n",
            "               '2022-01-05'],\n",
            "              dtype='datetime64[ns]', name='date', freq='D')\n",
            "────────────────────────────────────────────────────────────\n",
            "7\n",
            "────────────────────────────────────────────────────────────\n",
            "date\n",
            "2022-01-01    7\n",
            "2022-01-02    2\n",
            "2022-01-03    0\n",
            "2022-01-04   -5\n",
            "2022-01-05   -4\n",
            "Freq: D, dtype: int64\n"
          ]
        }
      ],
      "source": [
        "from pandas import Series,date_range\n",
        "from numpy.random import default_rng\n",
        "\n",
        "rng = default_rng(0)\n",
        "\n",
        "idx =  date_range('2022-01-01', periods=5, name='date')\n",
        "s = Series(rng.integers(-10, +10, size=len(idx)), index=idx)\n",
        "\n",
        "print(\n",
        "    s,\n",
        "    s.array,\n",
        "    s.index,\n",
        "    s['2022-01-01'],\n",
        "    s['2022-01-01':],# Value from 1st of Jan till whatever has been captured!\n",
        "    sep='\\n{}\\n'.format('\\N{box drawings light horizontal}' * 60),\n",
        "\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 128,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kGvMIWnFb1Le",
        "outputId": "7a6b64c4-6db2-451c-b4a1-37785058a290"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "-1.0     7\n",
            " 0.0     2\n",
            " 1.0     0\n",
            " 2.0    -5\n",
            " 3.0    -4\n",
            " 4.0   -10\n",
            "dtype: int64\n",
            "────────────────────────────────────────────────────────────\n",
            "Float64Index([-1.0, 0.0, 1.0, 2.0, 3.0, 4.0], dtype='float64')\n",
            "────────────────────────────────────────────────────────────\n",
            "2\n",
            "────────────────────────────────────────────────────────────\n",
            "0.0    2\n",
            "1.0    0\n",
            "2.0   -5\n",
            "3.0   -4\n",
            "dtype: int64\n",
            "────────────────────────────────────────────────────────────\n",
            "3\n",
            "4\n",
            "────────────────────────────────────────────────────────────\n",
            "2\n",
            "────────────────────────────────────────────────────────────\n",
            "0.0    2\n",
            "1.0    0\n",
            "2.0   -5\n",
            "3.0   -4\n",
            "dtype: int64\n",
            "────────────────────────────────────────────────────────────\n",
            "7\n",
            "────────────────────────────────────────────────────────────\n",
            "-1.0    7\n",
            " 0.0    2\n",
            " 1.0    0\n",
            "dtype: int64\n",
            "────────────────────────────────────────────────────────────\n",
            "-1.0    7\n",
            " 0.0    2\n",
            " 1.0    0\n",
            "dtype: int64\n",
            "────────────────────────────────────────────────────────────\n",
            "<PandasArray>\n",
            "[7, 2, 0]\n",
            "Length: 3, dtype: int64\n",
            "────────────────────────────────────────────────────────────\n",
            "-1.0    7\n",
            " 0.0    2\n",
            " 1.0    0\n",
            " 2.0   -5\n",
            "dtype: int64\n",
            "────────────────────────────────────────────────────────────\n",
            "<PandasArray>\n",
            "[7, 2, 0, -5]\n",
            "Length: 4, dtype: int64\n"
          ]
        }
      ],
      "source": [
        "# The index is how we access values\n",
        "\n",
        "from pandas.core import series\n",
        "from pandas import Series,date_range\n",
        "from numpy.random import default_rng\n",
        "from numpy import linspace, array\n",
        "\n",
        "rng = default_rng(0)\n",
        "\n",
        "# Uncomment the labels:'idx' one by one to see the how the miss-match occurs!\n",
        "idx = linspace(-20, +20,5)\n",
        "idx = linspace(-20, +20,5, dtype=int)\n",
        "idx = array([-1,0,1,2,3,4], dtype=float)\n",
        "#idx = [*'abcde']\n",
        "#idx = [*'.a...c.']\n",
        "#idx = [*'.aa.cc.']\n",
        "#idx = [*'.ac.ac.']\n",
        "\n",
        "s = Series(rng.integers(-10, +10, size=len(idx)), index=idx)\n",
        "\n",
        "print(\n",
        "    s,\n",
        "    s.index,\n",
        "    s[0], # value at label 0\n",
        "    s[0:3], # Give values of label 0 till label 3, which is just that value'0'\n",
        "    f'{len(s.array[0:3]) }\\n{len(s[0:3])}', # Asking for values from 0-3 from pandas array v/s the values that are in label location 0-3\n",
        "    s.loc[0], # if we want to use \"index\" : labels\n",
        "    s.loc[0:3],\n",
        "    s.iloc[0], # use position instead of \"index\"; we don't use the index!\n",
        "    s.iloc[0:3],\n",
        "    # Comparing s.array v/s s.iloc\n",
        "    s.iloc[0:3], # one thing to noice is that `interval notation \":\"` includes the end-point which is not the case with python!\n",
        "    s.array[0:3],\n",
        "    s.iloc[0:3+1],\n",
        "    s.array[0:3+1],\n",
        "    sep='\\n{}\\n'.format('\\N{box drawings light horizontal}' * 60),\n",
        "\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 129,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EhkuODpem6eQ",
        "outputId": "69d8e375-a913-473f-a0ef-0f8fbcf160ec"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "a    7\n",
            "b    2\n",
            "c    0\n",
            "d   -5\n",
            "e   -4\n",
            "dtype: int64\n",
            "────────────────────────────────────────────────────────────\n",
            "Index(['a', 'b', 'c', 'd', 'e'], dtype='object')\n",
            "────────────────────────────────────────────────────────────\n",
            "a    7\n",
            "b    2\n",
            "c    0\n",
            "dtype: int64\n"
          ]
        }
      ],
      "source": [
        "# The index is how we access values\n",
        "\n",
        "from pandas.core import series\n",
        "from pandas import Series,date_range\n",
        "from numpy.random import default_rng\n",
        "from numpy import linspace, array\n",
        "\n",
        "rng = default_rng(0)\n",
        "\n",
        "# Uncomment the labels:'idx' one by one to see the how the miss-match occurs!\n",
        "\n",
        "idx = [*'abcde']\n",
        "#idx = [*'.a...c.']\n",
        "#idx = [*'.aa.cc.']\n",
        "#idx = [*'.ac.ac.']\n",
        "\n",
        "s = Series(rng.integers(-10, +10, size=len(idx)), index=idx)\n",
        "\n",
        "print(\n",
        "    s,\n",
        "    s.index,\n",
        "    #s.loc['a':'c'+1], # This will return an error because the index does not have some sort of order associated with it \n",
        "                       # and one of the main reason we had to include \"end-point\" in Pandas\n",
        "    s.loc['a':'c'],\n",
        "    sep='\\n{}\\n'.format('\\N{box drawings light horizontal}' * 60),\n",
        "\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aqLnWdrZv3ID"
      },
      "source": [
        "### Let's try different index to understand the bhehaviour of `.loc`"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 130,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nZh5q00OnXGC",
        "outputId": "43f12285-ec5d-488a-e61c-4f3d73b1a959"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            ".   -10\n",
            "a    -9\n",
            ".   -10\n",
            ".    -7\n",
            ".     6\n",
            "c     2\n",
            ".     8\n",
            "dtype: int64\n",
            "────────────────────────────────────────────────────────────\n",
            "Index(['.', 'a', '.', '.', '.', 'c', '.'], dtype='object')\n",
            "────────────────────────────────────────────────────────────\n",
            "a    -9\n",
            ".   -10\n",
            ".    -7\n",
            ".     6\n",
            "c     2\n",
            "dtype: int64\n"
          ]
        }
      ],
      "source": [
        "#idx = [*'abcde']\n",
        "idx = [*'.a...c.']\n",
        "#idx = [*'.aa.cc.']\n",
        "#idx = [*'.ac.ac.']\n",
        "\n",
        "s = Series(rng.integers(-10, +10, size=len(idx)), index=idx)\n",
        "\n",
        "print(\n",
        "    s,\n",
        "    s.index,\n",
        "    #s.loc['a':'c'+1], # This will return an error because the index does not have some sort of order associated with it \n",
        "                       # and one of the main reason we had to include \"end-point\" in Pandas\n",
        "    s.loc['a':'c'],\n",
        "    sep='\\n{}\\n'.format('\\N{box drawings light horizontal}' * 60),\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 131,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NYGmRMgnvrv3",
        "outputId": "4207e58b-d081-4ffc-ba6b-c93c9139104f"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            ".    0\n",
            "a    2\n",
            "a    9\n",
            ".    4\n",
            "c    2\n",
            "c    0\n",
            ".    1\n",
            "dtype: int64\n",
            "────────────────────────────────────────────────────────────\n",
            "Index(['.', 'a', 'a', '.', 'c', 'c', '.'], dtype='object')\n",
            "────────────────────────────────────────────────────────────\n",
            "a    2\n",
            "a    9\n",
            ".    4\n",
            "c    2\n",
            "c    0\n",
            "dtype: int64\n"
          ]
        }
      ],
      "source": [
        "#idx = [*'abcde']\n",
        "#idx = [*'.a...c.']\n",
        "idx = [*'.aa.cc.']\n",
        "#idx = [*'.ac.ac.']\n",
        "\n",
        "s = Series(rng.integers(-10, +10, size=len(idx)), index=idx)\n",
        "\n",
        "print(\n",
        "    s,\n",
        "    s.index,\n",
        "    #s.loc['a':'c'+1], # This will return an error because the index does not have some sort of order associated with it \n",
        "                       # and one of the main reason we had to include \"end-point\" in Pandas\n",
        "    s.loc['a':'c'],\n",
        "    sep='\\n{}\\n'.format('\\N{box drawings light horizontal}' * 60),\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 132,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 382
        },
        "id": "ie-LpyJcvum9",
        "outputId": "b635fc5e-6bdd-4281-f3cd-2d34dff26ca0"
      },
      "outputs": [
        {
          "ename": "KeyError",
          "evalue": "\"Cannot get left slice bound for non-unique label: 'a'\"",
          "output_type": "error",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
            "\u001b[1;32m/home/vishu/JupyterNotebooks/PandasExtras.ipynb Cell 81\u001b[0m in \u001b[0;36m<cell line: 10>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      <a href='vscode-notebook-cell:/home/vishu/JupyterNotebooks/PandasExtras.ipynb#Y143sZmlsZQ%3D%3D?line=5'>6</a>\u001b[0m idx \u001b[39m=\u001b[39m [\u001b[39m*\u001b[39m\u001b[39m'\u001b[39m\u001b[39m.ac.ac.\u001b[39m\u001b[39m'\u001b[39m]  \u001b[39m# Can't differentiate b/w which 'a' and 'c' we're asking for!\u001b[39;00m\n\u001b[1;32m      <a href='vscode-notebook-cell:/home/vishu/JupyterNotebooks/PandasExtras.ipynb#Y143sZmlsZQ%3D%3D?line=7'>8</a>\u001b[0m s \u001b[39m=\u001b[39m Series(rng\u001b[39m.\u001b[39mintegers(\u001b[39m-\u001b[39m\u001b[39m10\u001b[39m, \u001b[39m+\u001b[39m\u001b[39m10\u001b[39m, size\u001b[39m=\u001b[39m\u001b[39mlen\u001b[39m(idx)), index\u001b[39m=\u001b[39midx)\n\u001b[1;32m     <a href='vscode-notebook-cell:/home/vishu/JupyterNotebooks/PandasExtras.ipynb#Y143sZmlsZQ%3D%3D?line=9'>10</a>\u001b[0m \u001b[39mprint\u001b[39m(\n\u001b[1;32m     <a href='vscode-notebook-cell:/home/vishu/JupyterNotebooks/PandasExtras.ipynb#Y143sZmlsZQ%3D%3D?line=10'>11</a>\u001b[0m     s,\n\u001b[1;32m     <a href='vscode-notebook-cell:/home/vishu/JupyterNotebooks/PandasExtras.ipynb#Y143sZmlsZQ%3D%3D?line=11'>12</a>\u001b[0m     s\u001b[39m.\u001b[39mindex,\n\u001b[1;32m     <a href='vscode-notebook-cell:/home/vishu/JupyterNotebooks/PandasExtras.ipynb#Y143sZmlsZQ%3D%3D?line=12'>13</a>\u001b[0m     \u001b[39m#s.loc['a':'c'+1], # This will return an error because the index does not have some sort of order associated with it \u001b[39;00m\n\u001b[1;32m     <a href='vscode-notebook-cell:/home/vishu/JupyterNotebooks/PandasExtras.ipynb#Y143sZmlsZQ%3D%3D?line=13'>14</a>\u001b[0m                        \u001b[39m# and one of the main reason we had to include \"end-point\" in Pandas\u001b[39;00m\n\u001b[0;32m---> <a href='vscode-notebook-cell:/home/vishu/JupyterNotebooks/PandasExtras.ipynb#Y143sZmlsZQ%3D%3D?line=14'>15</a>\u001b[0m     s\u001b[39m.\u001b[39;49mloc[\u001b[39m'\u001b[39;49m\u001b[39ma\u001b[39;49m\u001b[39m'\u001b[39;49m:\u001b[39m'\u001b[39;49m\u001b[39mc\u001b[39;49m\u001b[39m'\u001b[39;49m],\n\u001b[1;32m     <a href='vscode-notebook-cell:/home/vishu/JupyterNotebooks/PandasExtras.ipynb#Y143sZmlsZQ%3D%3D?line=15'>16</a>\u001b[0m     sep\u001b[39m=\u001b[39m\u001b[39m'\u001b[39m\u001b[39m\\n\u001b[39;00m\u001b[39m{}\u001b[39;00m\u001b[39m\\n\u001b[39;00m\u001b[39m'\u001b[39m\u001b[39m.\u001b[39mformat(\u001b[39m'\u001b[39m\u001b[39m\\N{box drawings light horizontal}\u001b[39;00m\u001b[39m'\u001b[39m \u001b[39m*\u001b[39m \u001b[39m60\u001b[39m),\n\u001b[1;32m     <a href='vscode-notebook-cell:/home/vishu/JupyterNotebooks/PandasExtras.ipynb#Y143sZmlsZQ%3D%3D?line=16'>17</a>\u001b[0m )\n",
            "File \u001b[0;32m~/mambaforge/envs/jlab/lib/python3.10/site-packages/pandas/core/indexing.py:967\u001b[0m, in \u001b[0;36m_LocationIndexer.__getitem__\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m    964\u001b[0m axis \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39maxis \u001b[39mor\u001b[39;00m \u001b[39m0\u001b[39m\n\u001b[1;32m    966\u001b[0m maybe_callable \u001b[39m=\u001b[39m com\u001b[39m.\u001b[39mapply_if_callable(key, \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mobj)\n\u001b[0;32m--> 967\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_getitem_axis(maybe_callable, axis\u001b[39m=\u001b[39;49maxis)\n",
            "File \u001b[0;32m~/mambaforge/envs/jlab/lib/python3.10/site-packages/pandas/core/indexing.py:1180\u001b[0m, in \u001b[0;36m_LocIndexer._getitem_axis\u001b[0;34m(self, key, axis)\u001b[0m\n\u001b[1;32m   1178\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39misinstance\u001b[39m(key, \u001b[39mslice\u001b[39m):\n\u001b[1;32m   1179\u001b[0m     \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_validate_key(key, axis)\n\u001b[0;32m-> 1180\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_get_slice_axis(key, axis\u001b[39m=\u001b[39;49maxis)\n\u001b[1;32m   1181\u001b[0m \u001b[39melif\u001b[39;00m com\u001b[39m.\u001b[39mis_bool_indexer(key):\n\u001b[1;32m   1182\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_getbool_axis(key, axis\u001b[39m=\u001b[39maxis)\n",
            "File \u001b[0;32m~/mambaforge/envs/jlab/lib/python3.10/site-packages/pandas/core/indexing.py:1214\u001b[0m, in \u001b[0;36m_LocIndexer._get_slice_axis\u001b[0;34m(self, slice_obj, axis)\u001b[0m\n\u001b[1;32m   1211\u001b[0m     \u001b[39mreturn\u001b[39;00m obj\u001b[39m.\u001b[39mcopy(deep\u001b[39m=\u001b[39m\u001b[39mFalse\u001b[39;00m)\n\u001b[1;32m   1213\u001b[0m labels \u001b[39m=\u001b[39m obj\u001b[39m.\u001b[39m_get_axis(axis)\n\u001b[0;32m-> 1214\u001b[0m indexer \u001b[39m=\u001b[39m labels\u001b[39m.\u001b[39;49mslice_indexer(slice_obj\u001b[39m.\u001b[39;49mstart, slice_obj\u001b[39m.\u001b[39;49mstop, slice_obj\u001b[39m.\u001b[39;49mstep)\n\u001b[1;32m   1216\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39misinstance\u001b[39m(indexer, \u001b[39mslice\u001b[39m):\n\u001b[1;32m   1217\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mobj\u001b[39m.\u001b[39m_slice(indexer, axis\u001b[39m=\u001b[39maxis)\n",
            "File \u001b[0;32m~/mambaforge/envs/jlab/lib/python3.10/site-packages/pandas/core/indexes/base.py:6274\u001b[0m, in \u001b[0;36mIndex.slice_indexer\u001b[0;34m(self, start, end, step, kind)\u001b[0m\n\u001b[1;32m   6231\u001b[0m \u001b[39m\"\"\"\u001b[39;00m\n\u001b[1;32m   6232\u001b[0m \u001b[39mCompute the slice indexer for input labels and step.\u001b[39;00m\n\u001b[1;32m   6233\u001b[0m \n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m   6270\u001b[0m \u001b[39mslice(1, 3, None)\u001b[39;00m\n\u001b[1;32m   6271\u001b[0m \u001b[39m\"\"\"\u001b[39;00m\n\u001b[1;32m   6272\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_deprecated_arg(kind, \u001b[39m\"\u001b[39m\u001b[39mkind\u001b[39m\u001b[39m\"\u001b[39m, \u001b[39m\"\u001b[39m\u001b[39mslice_indexer\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[0;32m-> 6274\u001b[0m start_slice, end_slice \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mslice_locs(start, end, step\u001b[39m=\u001b[39;49mstep)\n\u001b[1;32m   6276\u001b[0m \u001b[39m# return a slice\u001b[39;00m\n\u001b[1;32m   6277\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m is_scalar(start_slice):\n",
            "File \u001b[0;32m~/mambaforge/envs/jlab/lib/python3.10/site-packages/pandas/core/indexes/base.py:6484\u001b[0m, in \u001b[0;36mIndex.slice_locs\u001b[0;34m(self, start, end, step, kind)\u001b[0m\n\u001b[1;32m   6482\u001b[0m start_slice \u001b[39m=\u001b[39m \u001b[39mNone\u001b[39;00m\n\u001b[1;32m   6483\u001b[0m \u001b[39mif\u001b[39;00m start \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[0;32m-> 6484\u001b[0m     start_slice \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mget_slice_bound(start, \u001b[39m\"\u001b[39;49m\u001b[39mleft\u001b[39;49m\u001b[39m\"\u001b[39;49m)\n\u001b[1;32m   6485\u001b[0m \u001b[39mif\u001b[39;00m start_slice \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[1;32m   6486\u001b[0m     start_slice \u001b[39m=\u001b[39m \u001b[39m0\u001b[39m\n",
            "File \u001b[0;32m~/mambaforge/envs/jlab/lib/python3.10/site-packages/pandas/core/indexes/base.py:6411\u001b[0m, in \u001b[0;36mIndex.get_slice_bound\u001b[0;34m(self, label, side, kind)\u001b[0m\n\u001b[1;32m   6409\u001b[0m     slc \u001b[39m=\u001b[39m lib\u001b[39m.\u001b[39mmaybe_booleans_to_slice(slc\u001b[39m.\u001b[39mview(\u001b[39m\"\u001b[39m\u001b[39mu1\u001b[39m\u001b[39m\"\u001b[39m))\n\u001b[1;32m   6410\u001b[0m     \u001b[39mif\u001b[39;00m \u001b[39misinstance\u001b[39m(slc, np\u001b[39m.\u001b[39mndarray):\n\u001b[0;32m-> 6411\u001b[0m         \u001b[39mraise\u001b[39;00m \u001b[39mKeyError\u001b[39;00m(\n\u001b[1;32m   6412\u001b[0m             \u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39mCannot get \u001b[39m\u001b[39m{\u001b[39;00mside\u001b[39m}\u001b[39;00m\u001b[39m slice bound for non-unique \u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m   6413\u001b[0m             \u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39mlabel: \u001b[39m\u001b[39m{\u001b[39;00m\u001b[39mrepr\u001b[39m(original_label)\u001b[39m}\u001b[39;00m\u001b[39m\"\u001b[39m\n\u001b[1;32m   6414\u001b[0m         )\n\u001b[1;32m   6416\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39misinstance\u001b[39m(slc, \u001b[39mslice\u001b[39m):\n\u001b[1;32m   6417\u001b[0m     \u001b[39mif\u001b[39;00m side \u001b[39m==\u001b[39m \u001b[39m\"\u001b[39m\u001b[39mleft\u001b[39m\u001b[39m\"\u001b[39m:\n",
            "\u001b[0;31mKeyError\u001b[0m: \"Cannot get left slice bound for non-unique label: 'a'\""
          ]
        }
      ],
      "source": [
        "# This will return an error!\n",
        "\n",
        "#idx = [*'abcde']\n",
        "#idx = [*'.a...c.']\n",
        "#idx = [*'.aa.cc.']\n",
        "idx = [*'.ac.ac.']  # Can't differentiate b/w which 'a' and 'c' we're asking for!\n",
        "\n",
        "s = Series(rng.integers(-10, +10, size=len(idx)), index=idx)\n",
        "\n",
        "print(\n",
        "    s,\n",
        "    s.index,\n",
        "    #s.loc['a':'c'+1], # This will return an error because the index does not have some sort of order associated with it \n",
        "                       # and one of the main reason we had to include \"end-point\" in Pandas\n",
        "    s.loc['a':'c'],\n",
        "    sep='\\n{}\\n'.format('\\N{box drawings light horizontal}' * 60),\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-f6Q2gfBwV1X"
      },
      "source": [
        "### Different Types of Index "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 133,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "O9L4HWtqv1sU",
        "outputId": "8100a110-bd4e-4e52-9a1f-e063ddb4258d"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Int64Index([0, 1, 2, 3, 4], dtype='int64')\n",
            "────────────────────────────────────────────────────────────\n",
            "<PandasArray>\n",
            "[0, 1, 2, 3, 4]\n",
            "Length: 5, dtype: int64\n"
          ]
        }
      ],
      "source": [
        "from pandas import Series, date_range, interval_range, period_range\n",
        "from numpy import arange\n",
        "from numpy.random import default_rng\n",
        "\n",
        "rng = default_rng(0)\n",
        "\n",
        "idx = arange(0,5)\n",
        "\n",
        "\n",
        "s = Series(rng.integers(-10, +10, size=len(idx)), index=idx)\n",
        "\n",
        "print(\n",
        "    s.index,\n",
        "    s.index.array ,\n",
        "    sep='\\n{}\\n'.format('\\N{box drawings light horizontal}' * 60),\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 134,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vkhNGxPPxzbv",
        "outputId": "0b5e8962-b8f7-4ccf-bae4-81c54d166b12"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "RangeIndex(start=0, stop=10, step=2)\n",
            "────────────────────────────────────────────────────────────\n",
            "<PandasArray>\n",
            "[0, 2, 4, 6, 8]\n",
            "Length: 5, dtype: int64\n"
          ]
        }
      ],
      "source": [
        "\n",
        "rng = default_rng(0)\n",
        "\n",
        "idx = range(0, 10, 2 )\n",
        "\n",
        "s = Series(rng.integers(-10, +10, size=len(idx)), index=idx)\n",
        "\n",
        "\n",
        "print(\n",
        "    s.index,\n",
        "    s.index.array ,\n",
        "    sep='\\n{}\\n'.format('\\N{box drawings light horizontal}' * 60),\n",
        ")\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 135,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "l8gCzfgXyV2v",
        "outputId": "fa183505-4cd3-4d89-cb5f-f6ce8765cc53"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Int64Index([0, 1, 2, 3, 4], dtype='int64')\n",
            "────────────────────────────────────────────────────────────\n",
            "<PandasArray>\n",
            "[0, 1, 2, 3, 4]\n",
            "Length: 5, dtype: int64\n"
          ]
        }
      ],
      "source": [
        "rng = default_rng(0)\n",
        "\n",
        "idx = arange(0, 5 )\n",
        "\n",
        "s = Series(rng.integers(-10, +10, size=len(idx)), index=idx)\n",
        "\n",
        "\n",
        "print(\n",
        "    s.index,\n",
        "    s.index.array ,\n",
        "    sep='\\n{}\\n'.format('\\N{box drawings light horizontal}' * 60),\n",
        ")\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oDht1NSW1cLD"
      },
      "source": [
        "`date_range`: https://pandas.pydata.org/docs/reference/api/pandas.date_range.html"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 136,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "skxZeHKgydtS",
        "outputId": "4ba00d13-7eeb-4a93-c89b-0352fccb1351"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "DatetimeIndex(['2022-01-01 00:00:00', '2022-01-01 00:02:00',\n",
            "               '2022-01-01 00:04:00', '2022-01-01 00:06:00',\n",
            "               '2022-01-01 00:08:00'],\n",
            "              dtype='datetime64[ns]', name='date', freq='2T')\n",
            "────────────────────────────────────────────────────────────\n",
            "<DatetimeArray>\n",
            "['2022-01-01 00:00:00', '2022-01-01 00:02:00', '2022-01-01 00:04:00',\n",
            " '2022-01-01 00:06:00', '2022-01-01 00:08:00']\n",
            "Length: 5, dtype: datetime64[ns]\n"
          ]
        }
      ],
      "source": [
        "rng = default_rng(0)\n",
        "\n",
        "# Uncomment each 'idx' one by one!\n",
        "\n",
        "#idx = date_range('2022-01-01', periods=5, name='date')\n",
        "idx = date_range('2022-01-01', periods=5, freq='2T', name='date')\n",
        "\n",
        "s = Series(rng.integers(-10, +10, size=len(idx)), index=idx)\n",
        "\n",
        "\n",
        "print(\n",
        "    s.index,\n",
        "    s.index.array ,\n",
        "    sep='\\n{}\\n'.format('\\N{box drawings light horizontal}' * 60),\n",
        ")\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "exzjOvQ61lQS"
      },
      "source": [
        "`inerval_range`: https://pandas.pydata.org/docs/reference/api/pandas.interval_range.html"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 137,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Z8V2ltljy43H",
        "outputId": "43799faa-05b4-46fb-d931-eba9ad798cd1"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "IntervalIndex([[0, 2), [2, 4), [4, 6), [6, 8), [8, 10)], dtype='interval[int64, left]', name='value')\n",
            "────────────────────────────────────────────────────────────\n",
            "<IntervalArray>\n",
            "[[0, 2), [2, 4), [4, 6), [6, 8), [8, 10)]\n",
            "Length: 5, dtype: interval[int64, left]\n"
          ]
        }
      ],
      "source": [
        "rng = default_rng(0)\n",
        "\n",
        "# Uncomment each 'idx' one by one!\n",
        "\n",
        "#idx = interval_range(0,5, name='value')\n",
        "idx = interval_range(0,10, freq=2, closed='left', name='value')\n",
        "s = Series(rng.integers(-10, +10, size=len(idx)), index=idx)\n",
        "\n",
        "\n",
        "print(\n",
        "    s.index,\n",
        "    s.index.array ,\n",
        "    sep='\\n{}\\n'.format('\\N{box drawings light horizontal}' * 60),\n",
        ")\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SewmdjIc1oot"
      },
      "source": [
        "`period_range` : https://pandas.pydata.org/docs/reference/api/pandas.interval_range.html"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 138,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "39Qqtt5Bz_ZM",
        "outputId": "c57b7aaa-a63f-4c7e-c8b3-a65c7c5a1b4c"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "PeriodIndex(['2022Q1', '2022Q2', '2022Q3'], dtype='period[Q-DEC]', name='quater')\n",
            "────────────────────────────────────────────────────────────\n",
            "<PeriodArray>\n",
            "['2022Q1', '2022Q2', '2022Q3']\n",
            "Length: 3, dtype: period[Q-DEC]\n"
          ]
        }
      ],
      "source": [
        "rng = default_rng(0)\n",
        "\n",
        "# Uncomment each 'idx' one by one!\n",
        "idx = period_range('2022Q1', periods=3, freq='Q', name='quater')\n",
        "s = Series(rng.integers(-10, +10, size=len(idx)), index=idx)\n",
        "\n",
        "\n",
        "print(\n",
        "    s.index,\n",
        "    s.index.array ,\n",
        "    sep='\\n{}\\n'.format('\\N{box drawings light horizontal}' * 60),\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-4cxZ-u50X3q"
      },
      "source": [
        " All these 'idx' may let you believe that Pandas  `index` is data but actually it isn't  data. It is something that we can convert to data. Pandas `index` is actually a mechanism "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8w6OxMnJ7LXR"
      },
      "source": [
        "### Pandas `index` is not actually data!"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 139,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0XiQHTXt2Kr5",
        "outputId": "b6c44619-a16d-4e16-efcb-1c7bb4cc8e5d"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "RangeIndex(start=0, stop=100, step=1)\n",
            "────────────────────────────────────────────────────────────\n",
            "False\n",
            "────────────────────────────────────────────────────────────\n",
            "True\n",
            "────────────────────────────────────────────────────────────\n",
            "True\n"
          ]
        }
      ],
      "source": [
        "from pandas import RangeIndex\n",
        "\n",
        "idx = RangeIndex(0,100)\n",
        "print(\n",
        "    idx,\n",
        "    idx.has_duplicates,\n",
        "    idx.is_monotonic,\n",
        "    idx.is_unique,\n",
        "    sep='\\n{}\\n'.format('\\N{box drawings light horizontal}' * 60),\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 140,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jAJN8yMq4Tet",
        "outputId": "b935e3eb-4e6a-452f-e65f-d19c479fa27d"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "RangeIndex(start=0, stop=100000000000000, step=1)\n",
            "────────────────────────────────────────────────────────────\n",
            "False\n",
            "────────────────────────────────────────────────────────────\n",
            "True\n",
            "────────────────────────────────────────────────────────────\n",
            "True\n"
          ]
        }
      ],
      "source": [
        "idx = RangeIndex(0,100_000_000_000_000) # If it was data then there is no way we could've stored values upto this big of a number!\n",
        "print(\n",
        "    idx,\n",
        "    idx.has_duplicates,\n",
        "    idx.is_monotonic,\n",
        "    idx.is_unique,\n",
        "    sep='\\n{}\\n'.format('\\N{box drawings light horizontal}' * 60),\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 141,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pYS6qkSY4scV",
        "outputId": "e955a6d7-e00a-4408-bf83-523f93767e89"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "RangeIndex(start=10, stop=20, step=5)\n",
            "────────────────────────────────────────────────────────────\n",
            "False\n",
            "────────────────────────────────────────────────────────────\n",
            "True\n",
            "────────────────────────────────────────────────────────────\n",
            "True\n",
            "────────────────────────────────────────────────────────────\n",
            "<bound method RangeIndex.get_loc of RangeIndex(start=10, stop=20, step=5)>\n",
            "────────────────────────────────────────────────────────────\n",
            "0\n",
            "────────────────────────────────────────────────────────────\n",
            "(0, 1)\n",
            "────────────────────────────────────────────────────────────\n",
            "[ 0  1 -1 -1]\n"
          ]
        }
      ],
      "source": [
        "idx = RangeIndex(10, 20, 5)\n",
        "print(\n",
        "    idx,\n",
        "    idx.has_duplicates,\n",
        "    idx.is_monotonic,\n",
        "    idx.is_unique,\n",
        "    idx.get_loc, # Return what one-key transistion into when we want to find a physical position\n",
        "    idx.get_loc(10), # This is interpreted as : the value 10 on the range-index from 10 upto 20 with step size of 5 will get the value of position at the start which is 0!\n",
        "    idx.slice_locs(10,13),\n",
        "    idx.get_indexer(range(10, 30, 5)), # -1 means data is not located and we might need to do some NaN filling for these positon!\n",
        "\n",
        "    sep='\\n{}\\n'.format('\\N{box drawings light horizontal}' * 60),\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "amvxe_FK2J4x"
      },
      "source": [
        "If you wanna know more about `index methods` : https://pandas.pydata.org/docs/reference/indexing.html"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 142,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OAIIcHjs7JGN",
        "outputId": "aea614ad-ff55-491e-e0a5-a001950ad588"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "RangeIndex(start=0, stop=5, step=1)\n",
            "s.index.get_loc(s.index[0])0\n",
            "s.index.is_monotonic       True\n",
            "s.index.is_unique          True\n",
            "s.index.has_duplicates     False\n",
            "\n",
            "RangeIndex(start=5, stop=10, step=1)\n",
            "s.index.get_loc(s.index[0])0\n",
            "s.index.is_monotonic       True\n",
            "s.index.is_unique          True\n",
            "s.index.has_duplicates     False\n",
            "\n",
            "RangeIndex(start=5, stop=15, step=2)\n",
            "s.index.get_loc(s.index[0])0\n",
            "s.index.is_monotonic       True\n",
            "s.index.is_unique          True\n",
            "s.index.has_duplicates     False\n",
            "\n",
            "Int64Index([0, 1, 2, 3, 5], dtype='int64')\n",
            "s.index.get_loc(s.index[0])0\n",
            "s.index.is_monotonic       True\n",
            "s.index.is_unique          True\n",
            "s.index.has_duplicates     False\n",
            "\n",
            "Float64Index([0.0, 1.0, 2.0, 3.0, 5.0], dtype='float64')\n",
            "s.index.get_loc(s.index[0])0\n",
            "s.index.is_monotonic       True\n",
            "s.index.is_unique          True\n",
            "s.index.has_duplicates     False\n",
            "\n",
            "Index(['a', 'b', 'c', 'd', 'e'], dtype='object')\n",
            "s.index.get_loc(s.index[0])0\n",
            "s.index.is_monotonic       True\n",
            "s.index.is_unique          True\n",
            "s.index.has_duplicates     False\n",
            "\n",
            "IntervalIndex([[0, 2), [2, 4), [4, 6), [6, 8), [8, 10)], dtype='interval[int64, left]')\n",
            "s.index.get_loc(s.index[0])0\n",
            "s.index.is_monotonic       True\n",
            "s.index.is_unique          True\n",
            "s.index.has_duplicates     False\n",
            "\n",
            "DatetimeIndex(['2022-01-01', '2022-01-02', '2022-01-03', '2022-01-04',\n",
            "               '2022-01-05'],\n",
            "              dtype='datetime64[ns]', freq='D')\n",
            "s.index.get_loc(s.index[0])0\n",
            "s.index.is_monotonic       True\n",
            "s.index.is_unique          True\n",
            "s.index.has_duplicates     False\n",
            "\n",
            "DatetimeIndex(['2022-01-01', '2022-01-01', '2022-01-02', '2022-01-02',\n",
            "               '2022-01-03'],\n",
            "              dtype='datetime64[ns]', freq=None)\n",
            "s.index.get_loc(s.index[0])slice(0, 2, None)\n",
            "s.index.is_monotonic       True\n",
            "s.index.is_unique          False\n",
            "s.index.has_duplicates     True\n",
            "\n",
            "PeriodIndex(['2022Q1', '2022Q2', '2022Q3', '2022Q4', '2023Q1'], dtype='period[Q-DEC]')\n",
            "s.index.get_loc(s.index[0])0\n",
            "s.index.is_monotonic       True\n",
            "s.index.is_unique          True\n",
            "s.index.has_duplicates     False\n",
            "\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "/tmp/ipykernel_276611/691046892.py:4: FutureWarning: pandas.Int64Index is deprecated and will be removed from pandas in a future version. Use pandas.Index with the appropriate dtype instead.\n",
            "  from pandas import(\n",
            "/tmp/ipykernel_276611/691046892.py:4: FutureWarning: pandas.Float64Index is deprecated and will be removed from pandas in a future version. Use pandas.Index with the appropriate dtype instead.\n",
            "  from pandas import(\n"
          ]
        }
      ],
      "source": [
        "# Diving more in-depth into indices in Pandas\n",
        "\n",
        "\n",
        "from pandas import(\n",
        "    RangeIndex,\n",
        "    Int64Index,Float64Index, Index,\n",
        "    IntervalIndex, DatetimeIndex, PeriodIndex,\n",
        ")\n",
        "\n",
        "from pandas import Series\n",
        "from pandas import(\n",
        "    interval_range, date_range, period_range,\n",
        "    to_datetime, to_timedelta,\n",
        ")\n",
        "from numpy.random import default_rng\n",
        "from functools import reduce\n",
        "from operator import and_ \n",
        "\n",
        "\n",
        "rng = default_rng(0)\n",
        "\n",
        "indices = [\n",
        "    (range(5),                                    RangeIndex),\n",
        "    (range(5, 10),                                RangeIndex),\n",
        "    (range(5, 15, 2),                             RangeIndex),\n",
        "    ([0, 1, 2, 3, 5],                             Int64Index),\n",
        "    ([0, 1, 2, 3, 5.0],                           Float64Index),\n",
        "    ([*'abcde'],                                  Index),\n",
        "    (interval_range(0, 10 , 5, closed='left'),    IntervalIndex),\n",
        "    (date_range('2022-01-01', periods=5),         DatetimeIndex),\n",
        "    (to_datetime('2022-01-01')\n",
        "    + to_timedelta([0, 0, 1, 1, 2], unit='D'),    DatetimeIndex),\n",
        "    (period_range('2022Q1', periods=5, freq='Q'), PeriodIndex),\n",
        "]\n",
        "\n",
        "#iterating through each one of these indices\n",
        "\n",
        "for idx, typ in indices:\n",
        "    s = Series(rng.integers(-10, +10, size=len(idx)), index=idx)\n",
        "    assert isinstance(s.index, typ)\n",
        "\n",
        "for idx, _ in indices:\n",
        "    s = Series(rng.integers(-10, +10, size=len(idx)), index=idx)\n",
        "    print(\n",
        "        s.index,\n",
        "        f's.index.get_loc(s.index[0]){s.index.get_loc(s.index[0])}',\n",
        "        f's.index.is_monotonic       {s.index.is_monotonic}',\n",
        "        f's.index.is_unique          {s.index.is_unique}',\n",
        "        f's.index.has_duplicates     {s.index.has_duplicates}',\n",
        "        sep='\\n', end='\\n\\n',\n",
        "        \n",
        "    )\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9M94XPWuC4oq",
        "outputId": "cfea24af-108a-4c05-bb12-17fe67ad4cef"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "methods={frozenset({'year', 'to_flat_index', '_get_reconciled_name_object', 'ndim', 'is_numeric', '_typ', 'isin', '_convert_tolerance', '_summary', '_find_common_type_compat', 'array', '_reduce', 'isnull', 'factorize', '_dir_deletions', '_format_attrs', 'sort_values', '_map_values', '_format_space', '_maybe_convert_timedelta', 'union', '_get_attributes_dict', '_field_ops', '_get_indexer', '_na_value', 'get_indexer_non_unique', 'day_of_year', 'insert', '_get_delete_freq', '_intersection_via_get_indexer', '_values', 'view', '_isnan', 'repeat', 'sortlevel', 'to_numpy', '_engine', 'is_all_dates', '_get_join_target', 'difference', '_hidden_attrs', '_data_cls', '_supports_partial_string_indexing', '_parsed_string_to_bounds', 'asfreq', 'isna', 'day_of_week', 'nunique', 'astype', '_get_values_for_loc', '_name', '_simple_new', '_unary_method', '_join_monotonic', 'fillna', '_convert_slice_indexer', '_is_memory_usage_qualified', '_maybe_disable_logical_methods', '_accessors', 'format', 'notnull', '_deprecated_arg', '_is_numeric_dtype', '_requires_unique_msg', '_validate_partial_date_slice', 'sort', '_get_partial_string_timestamp_match_key', '_assert_can_do_setop', 'any', 'resolution', '_maybe_cast_slice_bound', '_format_native_types', '_maybe_cast_indexer', '_wrap_intersection_result', '_wrap_joined_index', '_logical_method', 'append', 'dayofweek', 'memory_usage', '_validate_names', '_format_duplicate_message', '_mpl_repr', 'where', 'T', '_comparables', 'argmax', 'inferred_freq', '_validate_indexer', 'freqstr', '_maybe_cast_listlike_indexer', '_should_fallback_to_positional', '_transform_index', 'is_monotonic_decreasing', 'argsort', '_can_hold_na', 'is_interval', '_is_strictly_monotonic_increasing', 'get_indexer_for', '_resolution_obj', '_arith_method', '_maybe_check_unique', 'dtype', 'values', 'hasnans', 'names', '_id', '_formatter_func', '_get_nearest_indexer', '_shallow_copy', 'drop_duplicates', '_ensure_array', '_join_multi', '_get_insert_freq', '_is_multi', 'daysinmonth', '_difference', 'freq', 'intersection', '_invalid_indexer', '_get_level_number', '_no_setting_name', 'is_integer', 'shift', 'unique', '_has_complex_internals', '_left_indexer_unique', 'droplevel', 'is_object', '_concat', '_from_join_target', 'inferred_type', '_get_join_freq', '_inner_indexer', 'qyear', '_getitem_slice', 'asof_locs', 'copy', 'is_categorical', '_dtype_to_subclass', '_constructor', '_set_names', 'rename', 'get_level_values', 'to_series', 'weekofyear', '_join_level', '_is_comparable_dtype', 'searchsorted', 'day', 'groupby', '_bool_ops', '_join_precedence', 'dropna', '_intersection', 'week', '_get_indexer_non_comparable', 'max', 'asof', 'reindex', '_validate_sort_keyword', 'strftime', 'slice_locs', 'days_in_month', 'is_type_compatible', '_get_unique_index', '_union', 'equals', '_cleanup', '_get_grouper_for_level', 'hour', 'is_floating', 'to_list', 'is_', '_cmp_method', '_left_indexer', '_engine_type', '_reindex_non_unique', 'delete', 'is_mixed', '_duplicated', 'argmin', '_get_fill_indexer_searchsorted', '_maybe_promote', 'item', '_validate_can_reindex', 'get_indexer', 'is_monotonic', 'is_monotonic_increasing', 'duplicated', '_rename', 'map', '_scalar_data_error', 'slice_indexer', 'weekday', 'join', 'second', '_get_string_slice', 'get_loc', '_can_hold_identifiers_and_holds_name', '_index_as_unique', '_validate_index_level', 'to_native_types', 'minute', 'take', '_join_non_unique', '_format_with_header', '_is_strictly_monotonic_decreasing', '_check_indexing_error', '_maybe_disallow_fill', '_searchsorted_monotonic', '_require_scalar', 'end_time', 'quarter', '_can_hold_strings', 'shape', 'name', '_hasnans', 'empty', 'dayofyear', '_reset_identity', '_format_data', '_validate_fill_value', 'has_duplicates', 'size', '_attributes', 'is_leap_year', '_cache', '_get_fill_indexer', '_dir_additions', '_index_data', 'is_boolean', 'identical', 'to_frame', '_validate_positional_slice', '_convert_can_do_setop', '_view', 'all', 'drop', 'symmetric_difference', '_check_indexing_method', '_filter_indexer_tolerance', 'month', '_wrap_setop_result', '_outer_indexer', 'set_names', '_memory_usage', 'is_full', 'min', 'notna', '_reset_cache', 'is_unique', '_get_names', 'asi8', 'holds_integer', 'value_counts', 'get_value', 'get_slice_bound', 'start_time', '_string_data_error', 'transpose', '_data', '_dir_additions_for_owner', '_get_level_values', '_get_engine_target', '_partial_date_slice', 'putmask', 'ravel', 'nbytes', '_sort_levels_monotonic', 'nlevels', '_drop_level_numbers', 'mean', '_is_all_dates', '_should_compare', 'to_timestamp'}), frozenset({'to_flat_index', '_get_reconciled_name_object', 'ndim', 'is_numeric', '_typ', 'isin', '_convert_tolerance', '_summary', '_intersection_non_unique', '_find_common_type_compat', 'array', '_reduce', 'isnull', 'factorize', '_dir_deletions', '_format_attrs', 'sort_values', '_map_values', '_format_space', 'union', '_get_attributes_dict', 'open_left', '_get_indexer', '_na_value', 'get_indexer_non_unique', 'insert', 'from_arrays', '_intersection_via_get_indexer', '_values', 'view', '_isnan', 'repeat', 'sortlevel', 'to_numpy', '_engine', 'is_all_dates', '_get_join_target', 'difference', '_hidden_attrs', '_data_cls', '_supports_partial_string_indexing', 'isna', 'nunique', 'astype', '_get_values_for_loc', '_name', '_simple_new', '_unary_method', '_join_monotonic', 'fillna', '_convert_slice_indexer', '_is_memory_usage_qualified', '_maybe_disable_logical_methods', '_accessors', 'format', 'open_right', '_deprecated_arg', '_is_numeric_dtype', '_requires_unique_msg', 'is_non_overlapping_monotonic', 'sort', '_get_partial_string_timestamp_match_key', '_assert_can_do_setop', 'any', '_maybe_cast_slice_bound', '_format_native_types', '_maybe_cast_indexer', '_wrap_intersection_result', '_wrap_joined_index', '_logical_method', 'append', 'memory_usage', '_validate_names', '_format_duplicate_message', '_mpl_repr', 'where', 'T', '_comparables', 'argmax', '_validate_indexer', '_maybe_cast_listlike_indexer', '_intersection_unique', '_should_fallback_to_positional', '_transform_index', 'is_monotonic_decreasing', 'closed', 'argsort', '_can_hold_na', 'is_interval', '_is_strictly_monotonic_increasing', 'get_indexer_for', '_arith_method', 'overlaps', '_maybe_check_unique', 'dtype', 'values', 'hasnans', 'names', '_id', '_formatter_func', '_get_nearest_indexer', '_shallow_copy', 'drop_duplicates', '_ensure_array', '_join_multi', 'intersection', '_is_multi', 'notnull', '_difference', 'mid', 'shift', '_invalid_indexer', '_get_level_number', '_no_setting_name', 'is_integer', 'unique', '_has_complex_internals', '_left_indexer_unique', 'droplevel', 'is_object', '_concat', '_from_join_target', 'inferred_type', 'right', '_inner_indexer', 'from_tuples', '_getitem_slice', 'length', '_needs_i8_conversion', 'asof_locs', 'copy', '_dtype_to_subclass', '_constructor', '_set_names', 'is_categorical', 'get_level_values', 'rename', 'to_series', '_join_level', '_is_comparable_dtype', 'searchsorted', 'groupby', '_join_precedence', 'dropna', '_intersection', '_get_indexer_non_comparable', 'max', 'asof', '_get_indexer_pointwise', 'reindex', '_validate_sort_keyword', 'slice_locs', 'to_tuples', 'is_type_compatible', '_get_unique_index', '_union', 'equals', '_cleanup', '_get_grouper_for_level', 'is_floating', 'from_breaks', 'to_list', 'is_', '_cmp_method', '_left_indexer', '_engine_type', '_reindex_non_unique', 'delete', 'is_mixed', '_duplicated', 'argmin', '_get_fill_indexer_searchsorted', '_maybe_promote', 'item', '_validate_can_reindex', 'get_indexer', 'is_monotonic', 'is_monotonic_increasing', 'duplicated', '_rename', 'map', '_scalar_data_error', 'slice_indexer', 'closed_right', 'join', '_get_string_slice', 'get_loc', '_can_hold_identifiers_and_holds_name', '_index_as_unique', '_validate_index_level', 'to_native_types', 'take', '_join_non_unique', 'set_closed', 'is_empty', '_is_strictly_monotonic_decreasing', '_format_with_header', '_check_indexing_error', '_maybe_disallow_fill', '_searchsorted_monotonic', '_require_scalar', 'left', 'shape', '_can_hold_strings', 'name', 'empty', 'has_duplicates', '_reset_identity', '_format_data', '_validate_fill_value', 'size', '_attributes', '_cache', '_get_fill_indexer', '_dir_additions', 'closed_left', 'is_boolean', 'identical', 'to_frame', '_validate_positional_slice', '_convert_can_do_setop', 'is_overlapping', '_view', 'all', 'drop', 'symmetric_difference', '_check_indexing_method', '_filter_indexer_tolerance', '_wrap_setop_result', '_outer_indexer', 'set_names', '_memory_usage', 'min', 'notna', '_reset_cache', 'is_unique', '_get_names', 'asi8', 'holds_integer', '_maybe_convert_i8', 'get_value', 'get_slice_bound', 'value_counts', '_multiindex', '_string_data_error', 'transpose', '_data', '_dir_additions_for_owner', '_get_level_values', '_get_engine_target', 'putmask', 'ravel', 'nbytes', '_sort_levels_monotonic', 'nlevels', '_drop_level_numbers', '_is_all_dates', '_should_compare'}), frozenset({'to_flat_index', '_get_reconciled_name_object', 'ndim', 'is_numeric', '_typ', '_validate_dtype', 'isin', '_convert_tolerance', '_summary', '_find_common_type_compat', 'array', '_reduce', 'isnull', 'factorize', '_dir_deletions', '_format_attrs', 'sort_values', '_map_values', '_format_space', 'union', '_get_attributes_dict', '_get_indexer', '_na_value', 'get_indexer_non_unique', 'insert', '_intersection_via_get_indexer', '_values', 'view', '_isnan', 'repeat', 'sortlevel', 'to_numpy', '_engine', 'is_all_dates', '_get_join_target', 'difference', '_hidden_attrs', '_supports_partial_string_indexing', 'isna', 'nunique', 'astype', '_get_values_for_loc', '_name', '_simple_new', '_unary_method', '_join_monotonic', 'fillna', '_convert_slice_indexer', '_is_memory_usage_qualified', '_maybe_disable_logical_methods', '_accessors', 'format', '_deprecated_arg', '_is_numeric_dtype', '_requires_unique_msg', 'sort', '_get_partial_string_timestamp_match_key', '_assert_can_do_setop', 'any', '_maybe_cast_slice_bound', '_format_native_types', '_maybe_cast_indexer', '_wrap_intersection_result', '_wrap_joined_index', '_logical_method', 'append', 'memory_usage', '_validate_names', '_format_duplicate_message', '_mpl_repr', 'where', 'T', '_comparables', 'argmax', '_validate_indexer', '_maybe_cast_listlike_indexer', '_should_fallback_to_positional', '_transform_index', 'is_monotonic_decreasing', 'argsort', '_can_hold_na', 'is_interval', '_is_strictly_monotonic_increasing', 'get_indexer_for', '_arith_method', '_maybe_check_unique', 'dtype', 'values', '_engine_types', 'hasnans', 'names', '_id', '_formatter_func', '_get_nearest_indexer', '_shallow_copy', 'drop_duplicates', '_ensure_array', '_join_multi', 'intersection', '_is_multi', 'notnull', '_difference', 'shift', '_invalid_indexer', '_get_level_number', '_no_setting_name', 'is_integer', 'unique', '_has_complex_internals', '_left_indexer_unique', 'droplevel', 'is_object', '_concat', '_from_join_target', 'inferred_type', '_inner_indexer', '_getitem_slice', 'asof_locs', 'copy', 'is_categorical', '_dtype_to_subclass', '_constructor', '_set_names', 'rename', 'get_level_values', 'to_series', '_join_level', '_is_comparable_dtype', 'searchsorted', 'groupby', '_join_precedence', 'dropna', '_intersection', '_get_indexer_non_comparable', '_default_dtype', 'asof', 'max', 'reindex', '_validate_sort_keyword', 'slice_locs', 'is_type_compatible', '_get_unique_index', '_union', 'equals', '_ensure_dtype', '_cleanup', '_get_grouper_for_level', 'is_floating', 'to_list', 'is_', '_cmp_method', '_left_indexer', '_engine_type', '_reindex_non_unique', 'delete', 'is_mixed', '_duplicated', 'argmin', '_get_fill_indexer_searchsorted', '_maybe_promote', 'item', '_validate_can_reindex', 'get_indexer', 'is_monotonic', 'is_monotonic_increasing', 'duplicated', '_rename', 'map', '_scalar_data_error', 'slice_indexer', '_index_descr_args', 'join', '_get_string_slice', 'get_loc', '_can_hold_identifiers_and_holds_name', '_index_as_unique', '_validate_index_level', 'to_native_types', 'take', '_join_non_unique', '_format_with_header', '_is_strictly_monotonic_decreasing', '_check_indexing_error', '_maybe_disallow_fill', '_searchsorted_monotonic', '_require_scalar', 'shape', '_can_hold_strings', 'name', 'empty', 'has_duplicates', '_reset_identity', '_format_data', '_validate_fill_value', 'size', '_attributes', '_cache', '_get_fill_indexer', '_dir_additions', '_index_data', 'is_boolean', 'identical', 'to_frame', '_validate_positional_slice', '_convert_can_do_setop', '_view', 'all', 'drop', 'symmetric_difference', '_check_indexing_method', '_filter_indexer_tolerance', '_dtype_validation_metadata', '_wrap_setop_result', '_outer_indexer', 'set_names', '_memory_usage', 'min', 'notna', '_reset_cache', 'is_unique', '_get_names', 'asi8', 'holds_integer', '_assert_safe_casting', 'value_counts', 'get_value', 'get_slice_bound', '_string_data_error', 'transpose', '_data', '_dir_additions_for_owner', '_get_level_values', '_get_engine_target', 'putmask', 'ravel', 'nbytes', '_sort_levels_monotonic', 'nlevels', '_drop_level_numbers', '_is_all_dates', '_should_compare'}), frozenset({'to_flat_index', '_get_reconciled_name_object', 'ndim', 'is_numeric', '_typ', 'isin', '_convert_tolerance', '_summary', '_find_common_type_compat', 'array', '_reduce', 'isnull', 'factorize', '_dir_deletions', '_format_attrs', 'sort_values', '_map_values', '_format_space', 'union', '_get_attributes_dict', '_get_indexer', '_na_value', 'get_indexer_non_unique', 'insert', '_intersection_via_get_indexer', '_values', 'view', '_isnan', 'repeat', 'sortlevel', 'to_numpy', '_engine', 'is_all_dates', '_get_join_target', 'difference', '_hidden_attrs', '_supports_partial_string_indexing', 'isna', 'nunique', 'astype', '_get_values_for_loc', '_name', '_simple_new', '_unary_method', '_join_monotonic', 'fillna', '_convert_slice_indexer', '_is_memory_usage_qualified', '_maybe_disable_logical_methods', '_accessors', 'format', '_deprecated_arg', '_is_numeric_dtype', '_requires_unique_msg', 'sort', '_get_partial_string_timestamp_match_key', '_assert_can_do_setop', 'any', '_maybe_cast_slice_bound', '_format_native_types', '_maybe_cast_indexer', '_wrap_intersection_result', '_wrap_joined_index', '_logical_method', 'append', 'memory_usage', '_validate_names', '_format_duplicate_message', '_mpl_repr', 'where', 'T', '_comparables', 'argmax', '_validate_indexer', '_maybe_cast_listlike_indexer', 'str', '_should_fallback_to_positional', '_transform_index', 'is_monotonic_decreasing', 'argsort', '_can_hold_na', 'is_interval', '_is_strictly_monotonic_increasing', 'get_indexer_for', '_arith_method', '_maybe_check_unique', 'dtype', 'values', 'hasnans', 'names', '_id', '_formatter_func', '_get_nearest_indexer', '_shallow_copy', 'drop_duplicates', '_ensure_array', '_join_multi', 'intersection', '_is_multi', 'notnull', '_difference', 'shift', '_invalid_indexer', '_get_level_number', '_no_setting_name', 'is_integer', 'unique', '_has_complex_internals', '_left_indexer_unique', 'droplevel', 'is_object', '_concat', '_from_join_target', 'inferred_type', '_inner_indexer', '_getitem_slice', 'asof_locs', 'copy', 'is_categorical', '_dtype_to_subclass', '_constructor', '_set_names', 'rename', 'get_level_values', 'to_series', '_join_level', '_is_comparable_dtype', 'searchsorted', 'groupby', '_join_precedence', 'dropna', '_intersection', '_get_indexer_non_comparable', 'max', 'asof', 'reindex', '_validate_sort_keyword', 'slice_locs', 'is_type_compatible', '_get_unique_index', '_union', 'equals', '_cleanup', '_get_grouper_for_level', 'is_floating', 'to_list', 'is_', '_cmp_method', '_left_indexer', '_engine_type', '_reindex_non_unique', 'delete', 'is_mixed', '_duplicated', 'argmin', '_get_fill_indexer_searchsorted', '_maybe_promote', 'item', '_validate_can_reindex', 'get_indexer', 'is_monotonic', 'is_monotonic_increasing', 'duplicated', '_rename', 'map', '_scalar_data_error', 'slice_indexer', 'join', '_get_string_slice', 'get_loc', '_can_hold_identifiers_and_holds_name', '_index_as_unique', '_validate_index_level', 'to_native_types', 'take', '_join_non_unique', '_is_strictly_monotonic_decreasing', '_format_with_header', '_check_indexing_error', '_maybe_disallow_fill', '_searchsorted_monotonic', '_require_scalar', 'shape', '_can_hold_strings', 'name', 'empty', 'has_duplicates', '_reset_identity', '_format_data', '_validate_fill_value', 'size', '_attributes', '_cache', '_get_fill_indexer', '_dir_additions', '_index_data', 'is_boolean', 'identical', 'to_frame', '_validate_positional_slice', '_convert_can_do_setop', '_view', 'all', 'drop', 'symmetric_difference', '_check_indexing_method', '_filter_indexer_tolerance', '_wrap_setop_result', '_outer_indexer', 'set_names', '_memory_usage', 'min', 'notna', '_reset_cache', 'is_unique', '_get_names', 'asi8', 'holds_integer', 'value_counts', 'get_value', 'get_slice_bound', '_string_data_error', 'transpose', '_data', '_dir_additions_for_owner', '_get_level_values', '_get_engine_target', 'putmask', 'ravel', 'nbytes', '_sort_levels_monotonic', 'nlevels', '_drop_level_numbers', '_is_all_dates', '_should_compare'}), frozenset({'year', 'ndim', '_find_common_type_compat', 'factorize', '_dir_deletions', 'sort_values', 'get_indexer_non_unique', '_values', '_hidden_attrs', 'isna', '_parsed_string_to_bounds', 'resolution', '_get_values_for_loc', '_name', '_convert_slice_indexer', '_can_fast_intersect', 'format', '_requires_unique_msg', '_assert_can_do_setop', '_maybe_cast_indexer', 'memory_usage', 'T', '_comparables', 'freqstr', 'argsort', '_can_hold_na', '_is_strictly_monotonic_increasing', '_arith_method', 'dtype', '_id', 'intersection', '_left_indexer_unique', '_concat', 'is_categorical', '_datetimelike_methods', '_join_level', 'normalize', '_get_indexer_non_comparable', 'max', 'asof', '_get_time_micros', 'hour', 'indexer_at_time', 'is_year_end', 'is_', '_cmp_method', '_left_indexer', 'item', 'get_indexer', 'map', 'duplicated', '_scalar_data_error', 'join', '_get_string_slice', '_is_dates_only', '_index_as_unique', '_validate_index_level', 'minute', '_format_with_header', 'name', 'empty', '_format_data', 'tz', 'is_month_start', 'month_name', 'to_frame', '_view', 'all', 'symmetric_difference', '_memory_usage', 'min', 'floor', 'holds_integer', '_maybe_utc_convert', '_get_level_values', '_partial_date_slice', 'nbytes', 'nlevels', 'mean', 'is_numeric', 'date', 'snap', 'timetz', '_format_attrs', '_map_values', '_field_ops', 'insert', 'view', 'repeat', '_supports_partial_string_indexing', '_is_numeric_dtype', '_format_native_types', '_validate_names', 'isocalendar', '_format_duplicate_message', 'nanosecond', 'values', 'hasnans', '_formatter_func', 'day_name', 'tzinfo', '_ensure_array', '_join_multi', '_difference', 'freq', 'shift', '_invalid_indexer', '_get_level_number', '_no_setting_name', 'is_integer', 'unique', '_get_join_freq', '_inner_indexer', '_getitem_slice', 'rename', '_constructor', 'to_series', '_set_names', 'to_pydatetime', '_object_ops', '_is_comparable_dtype', 'searchsorted', '_bool_ops', '_join_precedence', '_intersection', '_validate_sort_keyword', 'strftime', 'slice_locs', 'is_quarter_end', 'is_type_compatible', '_get_unique_index', '_reindex_non_unique', 'is_mixed', '_duplicated', '_can_fast_union', '_get_fill_indexer_searchsorted', '_rename', 'to_period', 'get_loc', '_join_non_unique', '_hasnans', '_reset_cache', 'asi8', '_fast_union', 'get_slice_bound', '_string_data_error', 'transpose', '_deprecate_mismatched_indexing', 'is_normalized', 'putmask', '_sort_levels_monotonic', '_should_compare', '_typ', 'isin', '_summary', 'array', 'union', 'union_many', '_na_value', '_get_delete_freq', '_intersection_via_get_indexer', '_isnan', 'sortlevel', '_engine', '_data_cls', 'astype', '_simple_new', '_accessors', '_deprecated_arg', 'any', '_wrap_intersection_result', '_wrap_joined_index', 'append', '_logical_method', 'dayofweek', '_is_monotonic_increasing', 'where', 'argmax', 'inferred_freq', '_validate_indexer', '_maybe_cast_listlike_indexer', '_transform_index', 'is_monotonic_decreasing', 'is_interval', 'is_year_start', 'get_indexer_for', 'microsecond', 'names', '_get_nearest_indexer', '_shallow_copy', '_is_multi', 'daysinmonth', '_has_complex_internals', '_maybe_cast_for_get_loc', 'is_object', 'inferred_type', 'is_month_end', 'asof_locs', '_dtype_to_subclass', 'get_level_values', 'day', 'dropna', 'week', 'reindex', 'days_in_month', '_union', 'equals', '_cleanup', 'to_list', '_has_same_tz', 'delete', 'is_quarter_start', 'argmin', '_maybe_promote', '_validate_can_reindex', 'is_monotonic', 'to_native_types', '_is_strictly_monotonic_decreasing', 'quarter', '_maybe_disallow_fill', 'shape', '_can_hold_strings', '_datetimelike_ops', 'has_duplicates', '_is_monotonic_decreasing', '_reset_identity', 'dayofyear', '_attributes', 'is_leap_year', '_cache', '_index_data', 'is_boolean', 'time', '_convert_can_do_setop', 'to_julian_date', 'to_perioddelta', 'drop', '_check_indexing_method', 'set_names', '_outer_indexer', 'is_unique', 'value_counts', 'ceil', '_drop_level_numbers', '_is_all_dates', 'to_flat_index', '_get_reconciled_name_object', '_convert_tolerance', '_reduce', 'isnull', 'tz_localize', '_format_space', '_get_attributes_dict', '_get_indexer', 'day_of_year', 'to_numpy', 'is_all_dates', '_get_join_target', 'difference', 'nunique', 'day_of_week', 'fillna', 'sort', '_unary_method', '_join_monotonic', '_is_memory_usage_qualified', '_maybe_disable_logical_methods', '_validate_partial_date_slice', '_get_partial_string_timestamp_match_key', '_maybe_cast_slice_bound', '_is_unique', '_mpl_repr', '_with_freq', '_should_fallback_to_positional', '_resolution_obj', '_maybe_check_unique', 'drop_duplicates', 'notnull', '_get_insert_freq', 'indexer_between_time', 'droplevel', '_from_join_target', 'copy', 'weekofyear', 'tz_convert', 'groupby', 'std', '_get_grouper_for_level', 'is_floating', '_engine_type', 'round', 'is_monotonic_increasing', 'slice_indexer', 'weekday', 'second', '_can_hold_identifiers_and_holds_name', 'take', '_check_indexing_error', '_searchsorted_monotonic', '_require_scalar', '_validate_fill_value', 'size', '_get_fill_indexer', '_dir_additions', 'identical', '_validate_positional_slice', '_filter_indexer_tolerance', 'month', '_wrap_setop_result', 'notna', '_get_names', 'get_value', '_data', '_dir_additions_for_owner', '_get_engine_target', 'ravel'}), frozenset({'to_flat_index', '_get_reconciled_name_object', 'ndim', '_range', 'is_numeric', '_typ', '_validate_dtype', 'isin', '_convert_tolerance', '_summary', '_find_common_type_compat', 'array', '_reduce', 'isnull', 'factorize', '_dir_deletions', '_format_attrs', 'sort_values', '_map_values', '_format_space', 'union', '_get_attributes_dict', '_get_indexer', '_na_value', 'get_indexer_non_unique', 'insert', '_intersection_via_get_indexer', '_values', 'view', '_isnan', 'repeat', 'sortlevel', 'to_numpy', '_engine', 'is_all_dates', '_get_join_target', 'difference', '_hidden_attrs', '_supports_partial_string_indexing', 'isna', '_step', 'nunique', 'astype', '_get_values_for_loc', '_name', '_simple_new', '_unary_method', '_join_monotonic', 'fillna', '_convert_slice_indexer', '_is_memory_usage_qualified', '_maybe_disable_logical_methods', '_accessors', 'format', 'from_range', '_deprecated_arg', '_is_numeric_dtype', '_requires_unique_msg', 'sort', '_get_partial_string_timestamp_match_key', '_assert_can_do_setop', 'any', '_maybe_cast_slice_bound', '_format_native_types', '_maybe_cast_indexer', '_wrap_intersection_result', '_wrap_joined_index', '_logical_method', 'append', 'memory_usage', '_validate_names', '_format_duplicate_message', '_mpl_repr', 'where', 'T', '_comparables', 'argmax', '_validate_indexer', '_maybe_cast_listlike_indexer', '_should_fallback_to_positional', '_transform_index', 'is_monotonic_decreasing', '_start', 'argsort', '_can_hold_na', 'is_interval', '_is_strictly_monotonic_increasing', 'get_indexer_for', '_get_data_as_items', '_arith_method', '_maybe_check_unique', 'dtype', 'values', '_engine_types', '_deprecation_message', 'hasnans', 'names', '_id', '_formatter_func', '_get_nearest_indexer', '_shallow_copy', 'drop_duplicates', '_ensure_array', '_join_multi', 'intersection', '_is_multi', 'notnull', '_difference', 'shift', '_invalid_indexer', '_get_level_number', '_no_setting_name', 'is_integer', 'unique', '_has_complex_internals', '_left_indexer_unique', 'droplevel', 'is_object', '_concat', '_from_join_target', 'inferred_type', '_inner_indexer', '_getitem_slice', 'asof_locs', 'copy', 'is_categorical', '_dtype_to_subclass', '_constructor', '_set_names', 'rename', 'get_level_values', 'to_series', '_join_level', '_is_comparable_dtype', '_int64index', '_min_fitting_element', 'groupby', 'searchsorted', '_join_precedence', 'dropna', '_intersection', '_get_indexer_non_comparable', 'max', 'asof', 'reindex', '_validate_sort_keyword', 'slice_locs', 'stop', 'is_type_compatible', '_get_unique_index', '_union', 'equals', '_ensure_dtype', '_cleanup', '_get_grouper_for_level', 'is_floating', 'to_list', 'is_', '_cmp_method', '_left_indexer', '_engine_type', '_reindex_non_unique', 'delete', 'is_mixed', '_duplicated', 'start', 'argmin', '_get_fill_indexer_searchsorted', '_maybe_promote', 'item', '_validate_can_reindex', 'get_indexer', 'is_monotonic', 'is_monotonic_increasing', 'duplicated', '_rename', 'map', '_scalar_data_error', 'slice_indexer', '_extended_gcd', '_index_descr_args', 'join', '_cached_int64index', '_get_string_slice', 'get_loc', '_can_hold_identifiers_and_holds_name', '_index_as_unique', '_validate_index_level', 'to_native_types', 'take', '_join_non_unique', '_format_with_header', '_is_strictly_monotonic_decreasing', '_check_indexing_error', '_maybe_disallow_fill', '_searchsorted_monotonic', '_require_scalar', 'shape', '_can_hold_strings', 'name', '_stop', 'empty', 'has_duplicates', '_reset_identity', '_format_data', '_validate_fill_value', 'size', '_attributes', '_cache', '_get_fill_indexer', '_dir_additions', 'is_boolean', 'identical', 'step', '_validate_positional_slice', '_convert_can_do_setop', 'to_frame', '_view', 'all', 'drop', 'symmetric_difference', '_check_indexing_method', '_filter_indexer_tolerance', '_dtype_validation_metadata', '_minmax', '_wrap_setop_result', '_outer_indexer', 'set_names', '_memory_usage', 'min', 'notna', '_reset_cache', 'is_unique', '_get_names', 'asi8', 'holds_integer', '_assert_safe_casting', 'value_counts', 'get_value', 'get_slice_bound', '_string_data_error', 'transpose', '_data', '_dir_additions_for_owner', '_get_level_values', '_get_engine_target', 'putmask', 'ravel', 'nbytes', '_sort_levels_monotonic', 'nlevels', '_drop_level_numbers', '_is_all_dates', '_should_compare'})}\n",
            "\n",
            "sorted(reduce(and_, methods))=['T', '_accessors', '_arith_method', '_assert_can_do_setop', '_attributes', '_cache', '_can_hold_identifiers_and_holds_name', '_can_hold_na', '_can_hold_strings', '_check_indexing_error', '_check_indexing_method', '_cleanup', '_cmp_method', '_comparables', '_concat', '_constructor', '_convert_can_do_setop', '_convert_slice_indexer', '_convert_tolerance', '_data', '_deprecated_arg', '_difference', '_dir_additions', '_dir_additions_for_owner', '_dir_deletions', '_drop_level_numbers', '_dtype_to_subclass', '_duplicated', '_engine', '_engine_type', '_ensure_array', '_filter_indexer_tolerance', '_find_common_type_compat', '_format_attrs', '_format_data', '_format_duplicate_message', '_format_native_types', '_format_space', '_format_with_header', '_formatter_func', '_from_join_target', '_get_attributes_dict', '_get_engine_target', '_get_fill_indexer', '_get_fill_indexer_searchsorted', '_get_grouper_for_level', '_get_indexer', '_get_indexer_non_comparable', '_get_join_target', '_get_level_number', '_get_level_values', '_get_names', '_get_nearest_indexer', '_get_partial_string_timestamp_match_key', '_get_reconciled_name_object', '_get_string_slice', '_get_unique_index', '_get_values_for_loc', '_getitem_slice', '_has_complex_internals', '_hidden_attrs', '_id', '_index_as_unique', '_inner_indexer', '_intersection', '_intersection_via_get_indexer', '_invalid_indexer', '_is_all_dates', '_is_comparable_dtype', '_is_memory_usage_qualified', '_is_multi', '_is_numeric_dtype', '_is_strictly_monotonic_decreasing', '_is_strictly_monotonic_increasing', '_isnan', '_join_level', '_join_monotonic', '_join_multi', '_join_non_unique', '_join_precedence', '_left_indexer', '_left_indexer_unique', '_logical_method', '_map_values', '_maybe_cast_indexer', '_maybe_cast_listlike_indexer', '_maybe_cast_slice_bound', '_maybe_check_unique', '_maybe_disable_logical_methods', '_maybe_disallow_fill', '_maybe_promote', '_memory_usage', '_mpl_repr', '_na_value', '_name', '_no_setting_name', '_outer_indexer', '_reduce', '_reindex_non_unique', '_rename', '_require_scalar', '_requires_unique_msg', '_reset_cache', '_reset_identity', '_scalar_data_error', '_searchsorted_monotonic', '_set_names', '_shallow_copy', '_should_compare', '_should_fallback_to_positional', '_simple_new', '_sort_levels_monotonic', '_string_data_error', '_summary', '_supports_partial_string_indexing', '_transform_index', '_typ', '_unary_method', '_union', '_validate_can_reindex', '_validate_fill_value', '_validate_index_level', '_validate_indexer', '_validate_names', '_validate_positional_slice', '_validate_sort_keyword', '_values', '_view', '_wrap_intersection_result', '_wrap_joined_index', '_wrap_setop_result', 'all', 'any', 'append', 'argmax', 'argmin', 'argsort', 'array', 'asi8', 'asof', 'asof_locs', 'astype', 'copy', 'delete', 'difference', 'drop', 'drop_duplicates', 'droplevel', 'dropna', 'dtype', 'duplicated', 'empty', 'equals', 'factorize', 'fillna', 'format', 'get_indexer', 'get_indexer_for', 'get_indexer_non_unique', 'get_level_values', 'get_loc', 'get_slice_bound', 'get_value', 'groupby', 'has_duplicates', 'hasnans', 'holds_integer', 'identical', 'inferred_type', 'insert', 'intersection', 'is_', 'is_all_dates', 'is_boolean', 'is_categorical', 'is_floating', 'is_integer', 'is_interval', 'is_mixed', 'is_monotonic', 'is_monotonic_decreasing', 'is_monotonic_increasing', 'is_numeric', 'is_object', 'is_type_compatible', 'is_unique', 'isin', 'isna', 'isnull', 'item', 'join', 'map', 'max', 'memory_usage', 'min', 'name', 'names', 'nbytes', 'ndim', 'nlevels', 'notna', 'notnull', 'nunique', 'putmask', 'ravel', 'reindex', 'rename', 'repeat', 'searchsorted', 'set_names', 'shape', 'shift', 'size', 'slice_indexer', 'slice_locs', 'sort', 'sort_values', 'sortlevel', 'symmetric_difference', 'take', 'to_flat_index', 'to_frame', 'to_list', 'to_native_types', 'to_numpy', 'to_series', 'transpose', 'union', 'unique', 'value_counts', 'values', 'view', 'where']\n"
          ]
        }
      ],
      "source": [
        "# Let's see all the methods that our indices support!\n",
        "\n",
        "methods = {\n",
        "    frozenset(\n",
        "        x\n",
        "        for x \n",
        "        in dir(Series(rng.integers(-10, +10, size=len(idx)), index=idx).index)\n",
        "        if not x.startswith('__')\n",
        "    )\n",
        "    for idx, _ in indices\n",
        "}\n",
        "print(\n",
        "    f'methods={methods}',\n",
        "    f'sorted(reduce(and_, methods))={sorted(reduce(and_, methods))}',\n",
        "    sep='\\n\\n',\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TdVeY3tJHIT1"
      },
      "source": [
        "### Why is it hard to create our own Index then use Pandas default ones!"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 143,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "s0D6ZVpoD4kj",
        "outputId": "d11b8654-a3bc-4dfc-8ae1-6b880d2411e9"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "SymmetricIndex([0, 2, 4], dtype='int64')\n",
            "────────────────────────────────────────────────────────────\n",
            "0    7\n",
            "2    2\n",
            "4    0\n",
            "dtype: int64\n",
            "────────────────────────────────────────────────────────────\n",
            "SymmetricIndex([0, 2, 4], dtype='int64')\n",
            "────────────────────────────────────────────────────────────\n",
            "s1.index.get_loc(2)=1\n",
            " s1.index.get_loc(-2)=1\n",
            "────────────────────────────────────────────────────────────\n",
            "s1.loc[2]=2\n",
            " s1.loc[-2]=2\n",
            "────────────────────────────────────────────────────────────\n",
            "-4    -5\n",
            "-3    -4\n",
            "-2   -10\n",
            "-1    -9\n",
            " 0   -10\n",
            " 1    -7\n",
            " 2     6\n",
            " 3     2\n",
            " 4     8\n",
            "dtype: int64\n",
            "────────────────────────────────────────────────────────────\n",
            "-4    NaN\n",
            "-3    NaN\n",
            "-2    NaN\n",
            "-1    NaN\n",
            " 0   -3.0\n",
            " 1    NaN\n",
            " 2    8.0\n",
            " 3    NaN\n",
            " 4    8.0\n",
            "dtype: float64\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "/tmp/ipykernel_276611/349071114.py:5: FutureWarning: pandas.Int64Index is deprecated and will be removed from pandas in a future version. Use pandas.Index with the appropriate dtype instead.\n",
            "  from pandas import Series, Int64Index\n"
          ]
        }
      ],
      "source": [
        "# Example of trying to create our own index ; in this case a `symmetrical Index`\n",
        "# A symmetrical index will return the absolute value !\n",
        "\n",
        "\n",
        "from pandas import Series, Int64Index\n",
        "from numpy.random import default_rng\n",
        "from itertools import chain \n",
        "\n",
        "############################################\n",
        "'''\n",
        "chain() function\n",
        "It is a function that takes a series of iterables and returns one iterable. It groups all the iterables together and produces a single iterable as output. \n",
        "\n",
        "This what chain does: \n",
        "\n",
        "def chain(*iterables):\n",
        "     for it in iterables:\n",
        "       for each in it:\n",
        "           yield each\n",
        "'''\n",
        "############################################\n",
        "\n",
        "\n",
        "rng = default_rng(0)\n",
        "\n",
        "class SymmetricIndex(Int64Index):\n",
        "    def get_loc(self,key):\n",
        "        return super().get_loc(abs(key))\n",
        "\n",
        "\n",
        "idx1 = SymmetricIndex([0,2,4])\n",
        "s1 = Series(rng.integers(-10, +10, size=len(idx1)), index=idx1)\n",
        "\n",
        "idx2 = [-4, -3, -2, -1, 0, 1, 2, 3, 4]\n",
        "s2 = Series(rng.integers(-10, +10, size=len(idx2)), index=idx2)\n",
        "\n",
        "print(\n",
        "    idx1,\n",
        "    s1,\n",
        "    s1.index,\n",
        "    f's1.index.get_loc(2)={s1.index.get_loc(2)}\\n s1.index.get_loc(-2)={s1.index.get_loc(-2)}',\n",
        "    f's1.loc[2]={s1.loc[2]}\\n s1.loc[-2]={s1.loc[-2]}',\n",
        "    s2,\n",
        "    s1+s2,\n",
        "    sep='\\n{}\\n'.format('\\N{box drawings light horizontal}' * 60),\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Sk2oiF3zHn2O"
      },
      "source": [
        "### Let's check what's happen in Pandas Library when we try to add two series `s1+s2`\n",
        "\n",
        "- **Don't run this on google colab as the Python version is still 3.7 in it !**\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 144,
      "metadata": {
        "id": "NrjR4tdUHv5U"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "date\n",
            "2022-01-01    7\n",
            "2022-01-02    2\n",
            "2022-01-03    0\n",
            "2022-01-04   -5\n",
            "2022-01-05   -4\n",
            "Freq: D, dtype: int64\n",
            "\n",
            "date\n",
            "2022-01-02   -10\n",
            "2022-01-03    -9\n",
            "2022-01-04   -10\n",
            "2022-01-05    -7\n",
            "2022-01-06     6\n",
            "Freq: D, dtype: int64\n",
            "                     filename  lineno               function  depth\n",
            "0          core/ops/common.py      55             new_method      1\n",
            "1      core/dtypes/generic.py      43                 _check      2\n",
            "2      core/dtypes/generic.py      43                 _check      2\n",
            "3      core/dtypes/generic.py      43                 _check      2\n",
            "4           core/arraylike.py      98                __add__      2\n",
            "..                        ...     ...                    ...    ...\n",
            "533            core/series.py     640                   name      7\n",
            "534     core/dtypes/common.py    1721  validate_all_hashable      8\n",
            "535     core/dtypes/common.py    1740              <genexpr>      9\n",
            "536  core/dtypes/inference.py     321            is_hashable     10\n",
            "537     core/dtypes/common.py    1740              <genexpr>      9\n",
            "\n",
            "[538 rows x 4 columns]\n",
            "\n",
            "                         filename  ...  depth\n",
            "0              core/ops/common.py  ...      1\n",
            "1          core/dtypes/generic.py  ...      2\n",
            "2          core/dtypes/generic.py  ...      2\n",
            "3          core/dtypes/generic.py  ...      2\n",
            "4               core/arraylike.py  ...      2\n",
            "5                  core/series.py  ...      3\n",
            "6            core/ops/__init__.py  ...      4\n",
            "7          core/dtypes/generic.py  ...      5\n",
            "8    core/indexes/datetimelike.py  ...      5\n",
            "23                 core/series.py  ...      5\n",
            "439                  core/base.py  ...      4\n",
            "440            core/ops/common.py  ...      5\n",
            "448                core/series.py  ...      5\n",
            "451          core/construction.py  ...      5\n",
            "458         core/ops/array_ops.py  ...      5\n",
            "459          core/construction.py  ...      5\n",
            "460         core/ops/array_ops.py  ...      5\n",
            "469                core/series.py  ...      5\n",
            "\n",
            "[18 rows x 4 columns]\n",
            "\n",
            "                         filename  lineno            function  depth\n",
            "8    core/indexes/datetimelike.py     109              equals      5\n",
            "9            core/indexes/base.py     803                 is_      6\n",
            "10      core/indexes/extension.py      67                fget      6\n",
            "12      core/indexes/extension.py      67                fget      6\n",
            "14      core/indexes/extension.py      67                fget      6\n",
            "..                            ...     ...                 ...    ...\n",
            "472          core/indexes/base.py    7082  maybe_extract_name      7\n",
            "476          core/indexes/base.py    6987        ensure_index      7\n",
            "478          core/indexes/base.py     884             __len__      8\n",
            "499          core/indexes/base.py     884             __len__      8\n",
            "518  core/indexes/datetimelike.py     103       _is_all_dates      8\n",
            "\n",
            "[117 rows x 4 columns]\n"
          ]
        }
      ],
      "source": [
        "# Make sure you're running Python version 3.8 or above\n",
        "# You can un-comment all the code below by selecting all the text and  pressing `CTRL + /` on Windows/Linux or `Command + /` on Mac\n",
        "\n",
        "\n",
        "\n",
        "from contextlib import contextmanager\n",
        "from sys import gettrace, settrace\n",
        "from pandas import DataFrame, Series, date_range\n",
        "from numpy.random import default_rng\n",
        "from pathlib import Path\n",
        "\n",
        "@contextmanager\n",
        "def tracing():\n",
        "    old_trace = gettrace()\n",
        "    depth, frames = 0, []\n",
        "    def trace(frame, event, arg):\n",
        "        nonlocal depth\n",
        "        filename = frame.f_code.co_filename\n",
        "        if filename.startswith('<'): return\n",
        "        filename = Path(filename)\n",
        "        # Use `!pip show pandas` to get your own pandas path!\n",
        "        if not filename.is_relative_to(base_dir := Path('/home/vishu/mambaforge/envs/jlab/lib/python3.10/site-packages/pandas').expanduser()):\n",
        "          return      \n",
        "        if event == 'return':\n",
        "            depth -= 1\n",
        "        if event == 'call':\n",
        "            depth += 1\n",
        "            fr = filename.relative_to(base_dir), frame.f_lineno, frame.f_code.co_name, depth\n",
        "            frames.append(fr)\n",
        "            return trace\n",
        "    settrace(trace)\n",
        "    try:\n",
        "      yield lambda: DataFrame(frames, columns='filename lineno function depth'.split()) \n",
        "    finally:\n",
        "      settrace(old_trace)\n",
        "\n",
        "            \n",
        "rng = default_rng(0)\n",
        "\n",
        "idx1 = date_range('2022-01-01', periods=5, name='date')\n",
        "s1 = Series(rng.integers(-10, +10, size=len(idx1)), index=idx1)\n",
        "\n",
        "idx2 = date_range('2022-01-02', periods=5, name='date')\n",
        "s2 = Series(rng.integers(-10, +10, size=len(idx2)), index=idx2)\n",
        "\n",
        "\n",
        "print(\n",
        "  s1,\n",
        "  s2,\n",
        "  sep='\\n\\n'\n",
        ")\n",
        "\n",
        "with tracing() as get_df:\n",
        "    s1 + s2\n",
        "\n",
        "\n",
        "df = get_df()\n",
        "\n",
        "\n",
        "print(\n",
        "    df, # around 530 function calls to capture s1+s2\n",
        "    df[df['depth']<=5],\n",
        "    df[df['filename'].apply(lambda x: x.is_relative_to('core/indexes'))], # around 116 calls related to index in here!\n",
        "    sep='\\n\\n',\n",
        "      \n",
        ")\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CjGIrLCli6rx"
      },
      "source": [
        "### `.index` is operative metadata with well defined semantics (in terms of data operations)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 145,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zw8HIHr3I49a",
        "outputId": "af0bbcce-6e58-44af-cc5e-7c25032fb4d6"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "r    7\n",
            "x    2\n",
            "m    0\n",
            "Name: series1, dtype: int64\n",
            "────────────────────────────────────────────────────────────\n",
            "s1.attrs={'source': Ellipsis, 'author': Ellipsis}\n",
            "────────────────────────────────────────────────────────────\n",
            "s1.groupby(s1.index).mean().head(3)=a   -0.626632\n",
            "b    0.219697\n",
            "c   -0.590551\n",
            "Name: series1, dtype: float64\n",
            "────────────────────────────────────────────────────────────\n",
            "s1.groupby(s1.index).mean().attrs()={'source': Ellipsis, 'author': Ellipsis}\n",
            "────────────────────────────────────────────────────────────\n",
            "(s1+s2).dropna().head(3).attrs={'source': Ellipsis, 'author': Ellipsis}\n",
            "────────────────────────────────────────────────────────────\n",
            "(s2+s1).dropna().head(3).attrs={'source': Ellipsis, 'date': Ellipsis}\n",
            "────────────────────────────────────────────────────────────\n",
            "(s1+s2).name=None\n",
            "────────────────────────────────────────────────────────────\n",
            "(s2+s1).name=None\n",
            "────────────────────────────────────────────────────────────\n",
            "s2.loc[s1[s1>0].index].attrs={'source': Ellipsis, 'date': Ellipsis}\n",
            "────────────────────────────────────────────────────────────\n",
            "s2.loc[s1[s1>0].index].name=series2\n"
          ]
        }
      ],
      "source": [
        "from pandas import Series\n",
        "from numpy.random import default_rng\n",
        "from string import ascii_lowercase\n",
        "\n",
        "rng = default_rng(0)\n",
        "size=10_000\n",
        "\n",
        "s1 = Series(\n",
        "    data=rng.integers(-10, +10, size=size),\n",
        "    index=rng.choice([*ascii_lowercase], size=size),\n",
        "    name='series1'\n",
        ")\n",
        "#we can attach meta-data to our series using `.attrs()`\n",
        "s1.attrs['source'] = ...\n",
        "s1.attrs['author'] = ...\n",
        "\n",
        "s2 = Series(\n",
        "    data=rng.integers(-10, +10, size=size),\n",
        "    index=rng.choice([*ascii_lowercase], size=size),\n",
        "    name='series2'\n",
        ")\n",
        "s2.attrs['source'] = ...\n",
        "s2.attrs['date'] = ...\n",
        "\n",
        "# And this meta-data will be preserved through operations!\n",
        "\n",
        "print(\n",
        "    s1.head(3),\n",
        "\n",
        "    f's1.attrs={s1.attrs}',\n",
        "\n",
        "    f's1.groupby(s1.index).mean().head(3)={s1.groupby(s1.index).mean().head(3)}',\n",
        "\n",
        "    f's1.groupby(s1.index).mean().attrs()={s1.groupby(s1.index).mean().attrs}',\n",
        "\n",
        "    f'(s1+s2).dropna().head(3).attrs={(s1+s2).dropna().head(3).attrs}',\n",
        "\n",
        "    f'(s2+s1).dropna().head(3).attrs={(s2+s1).dropna().head(3).attrs}',\n",
        " \n",
        "    f'(s1+s2).name={(s1+s2).name}',\n",
        "\n",
        "    f'(s2+s1).name={(s2+s1).name}',\n",
        "\n",
        "    f's2.loc[s1[s1>0].index].attrs={s2.loc[s1[s1>0].index].attrs}',\n",
        "\n",
        "    f's2.loc[s1[s1>0].index].name={s2.loc[s1[s1>0].index].name}',\n",
        "    \n",
        "    \n",
        "    \n",
        "    sep='\\n{}\\n'.format('\\N{box drawings light horizontal}' * 60),\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OA3rljJwnznT"
      },
      "source": [
        "Now, this kind of metadata(authors, date...) has no well-defined semantics in-terms of the operations of the series , in other words, if we add these two series together what we do with the metadata?Do we merge them ? If we have two sources with each representing a string as filename then do we return a set of those strings or return a list of those filenames or we take the bigger of two filenames?If they are two dates or two authors then what do we do?\n",
        "\n",
        "But there are opertaions that we can perform on this pandas series where the metadata that we had for that pandas series namely the attributes and name of that series may or may not be preserved and is not well-defined,whereas, **Pandas Index is always well-defined in-terms of the operations** because the opertations are defined in-terms of Index. Hence, `pandas.index` is the metadata that will be preserved through transformation that we performs on our data because those transformation are defined on how they change the index."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0sNa59Zls62Q"
      },
      "source": [
        "### Rules of promotion in Numpy!\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "fhY10f4oI8XZ"
      },
      "outputs": [],
      "source": [
        "# But before we discuss more about pandas index , let's talk about `numpy` "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 146,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kYetS2RyikVB",
        "outputId": "ea57ecd9-a617-4acd-fe8e-be135c0b0da7"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "False\n",
            "False\n",
            "False\n",
            "[9007199254740992 9007199254740993 9007199254740994 9007199254740995]\n",
            "\n",
            "True\n",
            "True\n",
            "True\n",
            "[9.00719925e+15 9.00719925e+15 9.00719925e+15 9.00719925e+15]\n",
            "\n"
          ]
        }
      ],
      "source": [
        "from numpy import array, equal, allclose, meshgrid, log, exp\n",
        "x = 2**53\n",
        "xs = array([x,x+1,x+2,x+3]) # created a numpy ndarray of 4 integers values\n",
        "# We're just taking a round trip to the original value \n",
        "'''\n",
        "# Run this basic example to see what we're talking about!\n",
        "a = [1,2,3]\n",
        "b = log(exp(a))\n",
        "\n",
        "print(a,\n",
        "      b,\n",
        "      sep='\\n\\n',)\n",
        "'''\n",
        "# We're just taking a round trip to the original value by doing : log(exp('values'))\n",
        "ys = exp(log(xs)) # took the 4 integers value from above then we take the logrithim and exponentiate them!(we get the orignial value back but instead of int we will have float values!)\n",
        "\n",
        "\n",
        "print(\n",
        "    f'{(xs[0]==xs).all()}', # Are all values in our list equal to first value ?                     :-> False\n",
        "    f'{equal(*meshgrid(xs, xs)).all()}', # Is every value equal every other value ?                 :-> False\n",
        "    f'{allclose(*meshgrid(xs, xs), atol=1, rtol=0)}', # Is every value close to every other value ? :-> False\n",
        "    xs,\n",
        "    sep='\\n', end='\\n\\n',\n",
        ")\n",
        "\n",
        "# But we we do some promotion to our int values(xs) and convert them into float values(ys) then:\n",
        "print(\n",
        "    f'{(ys[0]==ys).all()}', # Are all values in our list equal to first value ?                     :-> True\n",
        "    f'{equal(*meshgrid(ys, ys)).all()}', # Is every value equal every other value ?                 :-> True\n",
        "    f'{allclose(*meshgrid(ys, ys), atol=1, rtol=0)}', # Is every value close to every other value ? :-> True\n",
        "    ys,\n",
        "    sep='\\n', end='\\n\\n',\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "b0gZ9G550dsj"
      },
      "source": [
        "Thus , by applying some promotion to our int-values(xs) to get float values(ys), we lost the ability to store the distinction b/w these 4 float values because we are currently working with 2^53 meaning IEEE754 dot precision floating point values, we don't have the ability to precisely store that last digit."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zvvGktfwwbe7"
      },
      "source": [
        "- `all()`: https://numpy.org/doc/stable/reference/generated/numpy.all.html\n",
        "\n",
        "- `allclose()`: https://numpy.org/doc/stable/reference/generated/numpy.allclose.html\n",
        "\n",
        "- `meshgrid()`: https://numpy.org/doc/stable/reference/generated/numpy.meshgrid.html\n",
        "\n",
        "- `IEEE754` : https://en.wikipedia.org/wiki/IEEE_754"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YatCJZPQ11fC"
      },
      "source": [
        "### Promotion with associated loss of precision happens easily in pandas and can be very tricky to avoid!"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 147,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Ey6UJogtuwhl",
        "outputId": "c3148518-962d-48e3-8221-2dfddf623392"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "0    9007199254740992\n",
            "1    9007199254740993\n",
            "2    9007199254740994\n",
            "3    9007199254740995\n",
            "dtype: int64\n",
            "────────────────────────────────────────────────────────────\n",
            "1    9007199254740992\n",
            "2    9007199254740993\n",
            "3    9007199254740994\n",
            "4    9007199254740995\n",
            "dtype: int64\n",
            "────────────────────────────────────────────────────────────\n",
            "(s1.values == s2.values).all()=True\n",
            "────────────────────────────────────────────────────────────\n",
            "(s1.index == s2.index).all()=False\n",
            "────────────────────────────────────────────────────────────\n",
            "0             NaN\n",
            "1    1.801440e+16\n",
            "2    1.801440e+16\n",
            "3    1.801440e+16\n",
            "4             NaN\n",
            "dtype: float64\n",
            "────────────────────────────────────────────────────────────\n",
            "float64\n",
            "────────────────────────────────────────────────────────────\n",
            "0    9.007199e+15\n",
            "1    1.801440e+16\n",
            "2    1.801440e+16\n",
            "3    1.801440e+16\n",
            "4    9.007199e+15\n",
            "dtype: float64\n",
            "────────────────────────────────────────────────────────────\n",
            "float64\n"
          ]
        }
      ],
      "source": [
        "from pandas import Series\n",
        "from numpy import int64\n",
        "\n",
        "x= 2**53\n",
        "\n",
        "# Two same value series just differently indexed..\n",
        "\n",
        "s1 = Series([x, x+1, x+2, x+3], dtype=int64, index=range(4))\n",
        "s2 = Series([x, x+1, x+2, x+3], dtype=int64, index=range(1, 4+1))\n",
        "\n",
        "print(\n",
        "      s1,s2,\n",
        "      f'(s1.values == s2.values).all()={(s1.values == s2.values).all()}', # Do both series contain the same values?     :-> True\n",
        "      f'(s1.index == s2.index).all()={(s1.index == s2.index).all()}',     # Are indicies the same for both series ?     :-> False\n",
        "      s1 + s2,                                                            # This  will do an index-align sum, which will give NaN where thigns are missing\n",
        "      (s1 + s2).dtype,                                                    # Due to NaN it will convert from int64 to Float64\n",
        "      s1.add(s2, fill_value=0),                                           # Maybe if we replace the NaN values by `0`, will it preserve the int64? :-> No!, It will not\n",
        "      s1.add(s2, fill_value=0).dtype,\n",
        "      sep='\\n{}\\n'.format('\\N{box drawings light horizontal}' * 60),\n",
        "    \n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3_VlM_Oa7GNF"
      },
      "source": [
        "### Let's take a look at Numpy Broadcasting to better understand index alignment "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "05CFO-upLBV0"
      },
      "source": [
        "- Numpy Broadcasting is about taking two dissimilar shaped dataset and figuring out how to match them up so a operation performed on them make sense!\n",
        "- `numpy broadcasting`: https://numpy.org/doc/stable/user/basics.broadcasting.html\n",
        "\n",
        "- `.round()`: https://numpy.org/doc/stable/reference/generated/numpy.round_.html"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 148,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dxjknoR2JvCf",
        "outputId": "5ea9dae6-4da6-4434-c8bd-688198fc9351"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "xs.shape=(2, 3, 4)\n",
            "────────────────────────────────────────\n",
            "ys.shape=(4,)\n",
            "────────────────────────────────────────\n",
            "[[[ 0.1 -0.1  0.6  0.1]\n",
            "  [-0.5  0.4  1.3  0.9]\n",
            "  [-0.7 -1.3 -0.6  0. ]]\n",
            "\n",
            " [[-2.3 -0.2 -1.2 -0.7]\n",
            "  [-0.5 -0.3  0.4  1. ]\n",
            "  [-0.1  1.4 -0.7  0.4]]]\n",
            "────────────────────────────────────────\n",
            "[9.00719925e+15 9.00719925e+15 9.00719925e+15 9.00719925e+15]\n",
            "────────────────────────────────────────\n",
            "[[[9.00719925e+15 9.00719925e+15 9.00719925e+15 9.00719925e+15]\n",
            "  [9.00719925e+15 9.00719925e+15 9.00719925e+15 9.00719925e+15]\n",
            "  [9.00719925e+15 9.00719925e+15 9.00719925e+15 9.00719925e+15]]\n",
            "\n",
            " [[9.00719925e+15 9.00719925e+15 9.00719925e+15 9.00719925e+15]\n",
            "  [9.00719925e+15 9.00719925e+15 9.00719925e+15 9.00719925e+15]\n",
            "  [9.00719925e+15 9.00719925e+15 9.00719925e+15 9.00719925e+15]]]\n"
          ]
        }
      ],
      "source": [
        "from numpy import newaxis\n",
        "from numpy.random import default_rng\n",
        "\n",
        "rng = default_rng(0)\n",
        "\n",
        "xs = rng.normal(size=(2, 3, 4)).round(1)  #:-> shape(2,3,4)\n",
        "'''\n",
        "Uncomment `ys` values 1-by-1 to see which returns an error and which dosen't\n",
        "'''\n",
        "#ys = rng.normal(size=(2)).round(1)   #:-> shape(2,)   (broadcasting error) \n",
        "\n",
        "#to avoid broadcast error lets try :\n",
        "\n",
        "#ys = rng.normal(size=(2,3)).round(1) #:-> shape(2,3) (broadcasting error)\n",
        "#ys = rng.normal(size=(3,4)).round(1) #:-> shape(3,4) This will work!\n",
        "# Using `newaxis` to avoid broadcasting error\n",
        "\n",
        "#ys = rng.normal(size=(2)).round(1)[:, newaxis, newaxis] #:-> shape(2,1,1) Will work!\n",
        "\n",
        "print(\n",
        "    f'xs.shape={xs.shape}',\n",
        "    f'ys.shape={ys.shape}',\n",
        "    xs,\n",
        "    ys,\n",
        "    xs + ys, # Return `broadcasting error!` \n",
        "    sep='\\n{}\\n'.format('\\N{box drawings light horizontal}' * 40),\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nzZkTl35QdHL"
      },
      "source": [
        "### Rules of Broadcasting in Numpy"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "XklfLF1tL2Dy"
      },
      "outputs": [],
      "source": [
        "# Here is how rules of `Brodcasting` works:\n",
        "'''\n",
        "Take the dim. of the structure you're looking at (in our example: xs and ys) and right align them and then try and match them up,\n",
        "and when we trying and  matching them up, we're either looking at the exact match or a `1`.\n",
        "'''\n",
        "# Shape:\n",
        "\n",
        "# xs  2 3 4 \n",
        "# ys      2\n",
        "# ys    2 3\n",
        "# ys    3 4\n",
        "# ys  2 1 1"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 149,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EtEOjKIHRDzA",
        "outputId": "bf90e057-9f5d-47a5-8e4e-d2a29c834645"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[[[ 0.1 -0.1  0.6  0.1]\n",
            "  [-0.5  0.4  1.3  0.9]\n",
            "  [-0.7 -1.3 -0.6  0. ]]\n",
            "\n",
            " [[-2.3 -0.2 -1.2 -0.7]\n",
            "  [-0.5 -0.3  0.4  1. ]\n",
            "  [-0.1  1.4 -0.7  0.4]]]\n",
            "────────────────────────────────────────────────────────────\n",
            "xs.shape                          =(2, 3, 4)\n",
            "────────────────────────────────────────────────────────────\n",
            "xs.strides                        =(96, 32, 8)\n",
            "────────────────────────────────────────────────────────────\n",
            "xs[0].strides                     =(32, 8)\n",
            "────────────────────────────────────────────────────────────\n",
            "xs[0, 1].strides                  =(8,)\n",
            "────────────────────────────────────────────────────────────\n",
            "xs[:, 1].strides                  =(8,)\n",
            "────────────────────────────────────────────────────────────\n",
            "ys.strides                        =(8, 0, 0)\n",
            "────────────────────────────────────────────────────────────\n",
            "broadcast_to(zs, xs.shape).strides=(0, 32, 8)\n"
          ]
        }
      ],
      "source": [
        "# Example of Broadcasting rule:\n",
        "\n",
        "from numpy import newaxis, broadcast_to\n",
        "from numpy.random import default_rng\n",
        "\n",
        "rng = default_rng(0)\n",
        "\n",
        "xs = rng.normal(size=(2,3,4)).round(1)\n",
        "ys = rng.normal(size=2).round(1)[:, newaxis, newaxis]\n",
        "zs = rng.normal(size=(3,4)).round(1)\n",
        "\n",
        "\n",
        "print(\n",
        "    xs,\n",
        "    f'xs.shape                          ={xs.shape}',  #:-> Shape(2, 3, 4) 2:layers 3:rows 4:columns\n",
        "    f'xs.strides                        ={xs.strides}', #:-> to move from 1 layer to another we skip 96bytes, to go from row to row we skip 32bytes and to go from column to column we skip 8 bytes\n",
        "    f'xs[0].strides                     ={xs[0].strides}',\n",
        "    f'xs[0, 1].strides                  ={xs[0, 1].strides}',\n",
        "    f'xs[:, 1].strides                  ={xs[0, 1].strides}',\n",
        "    f'ys.strides                        ={ys.strides}',\n",
        "    f'broadcast_to(zs, xs.shape).strides={broadcast_to(zs, xs.shape).strides}',\n",
        "    sep='\\n{}\\n'.format('\\N{box drawings light horizontal}' * 60),\n",
        ")\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gkfKOKMzSwtm"
      },
      "source": [
        "- `strides`: https://numpy.org/doc/stable/reference/generated/numpy.ndarray.strides.html\n",
        "- `broadcast_to`:https://numpy.org/doc/stable/reference/generated/numpy.broadcast_to.html"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JKjHePO8T5II"
      },
      "source": [
        "What we get from this strides is that the outer-most axis (xs.strides=(96,32,8), so `8` is the mostly tightly packed; these are the values that are most contiguous in memory; that's probably why we choose to `right-align` them because we want to start from things that are most contiguous in memory to thigns that are least contiguous in memory so, that we're not jumping around memory too much!\n",
        "\n",
        "\n",
        "When we try to figure out  how indexing works by asking `xs[0].strides `: the indexing work from outermost to innermost ; we're drilling down into the data but we always want to get the thing thats as contiguous as possible because that'll be gonna as fast as possible!\n",
        "\n",
        "**Broadcasting: Works from the right-hand side** \\\\\n",
        "**Indexing: Works from the left-hand side**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2DPBgzm_YWu1"
      },
      "source": [
        "### Rules of \" index alignment \"\n",
        "\n",
        "- It's all about \"matching things up\"!"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 150,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "aWKiPBNvSwXY",
        "outputId": "3df27d40-392c-4ab6-d1d0-5eb37180af03"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "a    7\n",
            "b    2\n",
            "c    0\n",
            "d   -5\n",
            "dtype: int64\n",
            "────────────────────────────────────────────────────────────\n",
            "a    -4\n",
            "b   -10\n",
            "c    -9\n",
            "d   -10\n",
            "dtype: int64\n",
            "────────────────────────────────────────────────────────────\n",
            "s1.index.has_duplicates=False\n",
            "────────────────────────────────────────────────────────────\n",
            "s2.index.has_duplicates=False\n",
            "────────────────────────────────────────────────────────────\n",
            "a     3\n",
            "b    -8\n",
            "c    -9\n",
            "d   -15\n",
            "dtype: int64\n"
          ]
        }
      ],
      "source": [
        "# Without any duplicates in the `.index`\n",
        "\n",
        "\n",
        "from pandas import Series, Index\n",
        "from numpy.random import default_rng\n",
        "from inspect import signature\n",
        "\n",
        "rng = default_rng(0)\n",
        "\n",
        "idx1 = [*'abcd']\n",
        "s1 = Series(rng.integers(-10, +10, size=len(idx1)), index=idx1)\n",
        "\n",
        "idx2 = [*'abcd']\n",
        "s2 = Series(rng.integers(-10, +10, size=len(idx2)), index=idx2)\n",
        "\n",
        "\n",
        "print(\n",
        "    s1,\n",
        "    s2,\n",
        "    f's1.index.has_duplicates={s1.index.has_duplicates}',\n",
        "    f's2.index.has_duplicates={s2.index.has_duplicates}',\n",
        "    s1 + s2,\n",
        "    sep='\\n{}\\n'.format('\\N{box drawings light horizontal}' * 60),\n",
        "\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 151,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-yvwtyj4bh8x",
        "outputId": "e11a845a-768c-4814-c998-c3c7d4760f7d"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "a    7\n",
            "b    2\n",
            "c    0\n",
            "d   -5\n",
            "dtype: int64\n",
            "────────────────────────────────────────────────────────────\n",
            "b    -4\n",
            "c   -10\n",
            "d    -9\n",
            "e   -10\n",
            "dtype: int64\n",
            "────────────────────────────────────────────────────────────\n",
            "s1.index.has_duplicates       = False\n",
            "────────────────────────────────────────────────────────────\n",
            "s2.index.has_duplicates       = False\n",
            "────────────────────────────────────────────────────────────\n",
            "a     NaN\n",
            "b    -2.0\n",
            "c   -10.0\n",
            "d   -14.0\n",
            "e     NaN\n",
            "dtype: float64\n",
            "────────────────────────────────────────────────────────────\n",
            "s1.index.get_indexer(s2.index)= [ 1  2  3 -1]\n",
            "────────────────────────────────────────────────────────────\n",
            "s2.index.get_indexer(s1.index)= [-1  0  1  2]\n",
            "────────────────────────────────────────────────────────────\n",
            "Index([*\"bcde\"]).get_indexer(Index([*\"abcd\"]), method=\"bfill\")=[0 0 1 2]\n",
            "────────────────────────────────────────────────────────────\n",
            "Index([*\"bcde\"]).get_indexer(Index([*\"cdef\"]), method=\"bfill\")=[1 2 3 3]\n",
            "────────────────────────────────────────────────────────────\n",
            "Index([0, 1, 2, 3]).get_indexer(Index([0, 1.1, 2.2, 3.3]), method=\"nearest\")=[0 1 2 3]\n",
            "────────────────────────────────────────────────────────────\n",
            "signature(s1.add)=(other, level=None, fill_value=None, axis=0)\n"
          ]
        }
      ],
      "source": [
        "from pandas import Series, Index\n",
        "from numpy.random import default_rng\n",
        "from inspect import signature\n",
        "\n",
        "rng = default_rng(0)\n",
        "\n",
        "idx1 = [*'abcd']\n",
        "s1 = Series(rng.integers(-10, +10, size=len(idx1)), index=idx1)\n",
        "\n",
        "idx2 = [*'bcde']\n",
        "s2 = Series(rng.integers(-10, +10, size=len(idx2)), index=idx2)\n",
        "\n",
        "\n",
        "print(\n",
        "    s1,\n",
        "    s2,\n",
        "    f's1.index.has_duplicates       = {s1.index.has_duplicates}',\n",
        "    f's2.index.has_duplicates       = {s2.index.has_duplicates}',\n",
        "    s1 + s2,\n",
        "    f's1.index.get_indexer(s2.index)= {s1.index.get_indexer(s2.index)}', # can ask pandas that if we gonna do an operation b/w one index and other then what will happen!\n",
        "                                                                        # Return:-> s1.index.get_indexer(s2.index)=[ 1  2  3 -1] , where `-1` means filling that column with NaN\n",
        "    f's2.index.get_indexer(s1.index)= {s2.index.get_indexer(s1.index)}',\n",
        "    f'Index([*\"bcde\"]).get_indexer(Index([*\"abcd\"]), method=\"bfill\")={Index([*\"bcde\"]).get_indexer(Index([*\"abcd\"]), method=\"bfill\")}',\n",
        "    f'Index([*\"bcde\"]).get_indexer(Index([*\"cdef\"]), method=\"bfill\")={Index([*\"bcde\"]).get_indexer(Index([*\"cdef\"]), method=\"ffill\")}', \n",
        "    f'Index([0, 1, 2, 3]).get_indexer(Index([0, 1.1, 2.2, 3.3]), method=\"nearest\")={Index([0, 1, 2, 3]).get_indexer(Index([0, 1.1, 2.2, 3.3]), method=\"nearest\")}',\n",
        "    f'signature(s1.add)={signature(s1.add)}',                                                                  \n",
        "    sep='\\n{}\\n'.format('\\N{box drawings light horizontal}' * 60),\n",
        "\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hkJQpI4aaa7l"
      },
      "source": [
        "`inspect module`: https://docs.python.org/3/library/inspect.html\n",
        "\n",
        "`index.get_indexer`: https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.Index.get_indexer.html \\\\\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "p91bTXM9lPdA"
      },
      "source": [
        "### Relation b/w `collections.Counter` and `pandas.Series`:\n",
        "\n",
        "-  `collections.Counter` is a first-order approximation of  `pandas.Series`\n",
        "\n",
        "- Supporting only a single-level, unique, hashable object `.index` and non-negative Python int data.\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zRJSzgb6ezIv",
        "outputId": "d66368ab-1321-4ca0-9f3c-998c74845026"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Counter({'a': 1, 'b': 1, 'c': 1, 'd': 1})\n",
            "Counter({'a': 1, 'b': 1, 'c': 1, 'd': 1})\n",
            "Counter({'a': 2, 'b': 2, 'c': 2, 'd': 2})\n"
          ]
        }
      ],
      "source": [
        "from collections import Counter \n",
        "\n",
        "c1 = Counter('abcd')\n",
        "c2 = Counter('abcd')\n",
        "#c2 = Counter('bcde')\n",
        "\n",
        "print(\n",
        "    c1,\n",
        "    c2,\n",
        "    c1 + c2,\n",
        "    sep='\\n',\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qGBIkWU1mAVi",
        "outputId": "41df804a-ded1-4dd8-f27b-0a7c78a2747e"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Counter({'a': 1, 'b': 1, 'c': 1, 'd': 1})\n",
            "Counter({'b': 1, 'c': 1, 'd': 1, 'e': 1})\n",
            "Counter({'b': 2, 'c': 2, 'd': 2, 'a': 1, 'e': 1})\n"
          ]
        }
      ],
      "source": [
        "c1 = Counter('abcd')\n",
        "#c2 = Counter('abcd')\n",
        "c2 = Counter('bcde')\n",
        "\n",
        "print(\n",
        "    c1,\n",
        "    c2,\n",
        "    c1 + c2,\n",
        "    sep='\\n',\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SSEJ_RMzmb2p"
      },
      "source": [
        "If there are duplicates, consider what is a reasonable way to \"match things up\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xRzyJ41QmkHN",
        "outputId": "ab75ccb5-1f6e-41dd-d2a9-eb03eb0981c2"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "a    7\n",
            "b    2\n",
            "c    0\n",
            "d   -5\n",
            "dtype: int64\n",
            "────────────────────────────────────────\n",
            "a    -4\n",
            "a   -10\n",
            "b    -9\n",
            "c   -10\n",
            "dtype: int64\n",
            "────────────────────────────────────────\n",
            "s1.index.has_duplicates=False\n",
            "────────────────────────────────────────\n",
            "s2.index.has_duplicates=True\n",
            "────────────────────────────────────────\n",
            "a     3.0\n",
            "a    -3.0\n",
            "b    -7.0\n",
            "c   -10.0\n",
            "d     NaN\n",
            "dtype: float64\n"
          ]
        }
      ],
      "source": [
        "from pandas import Series, Index \n",
        "from numpy.random import default_rng \n",
        "\n",
        "rng = default_rng(0)\n",
        "\n",
        "idx1 = [*'abcd']\n",
        "s1 = Series(rng.integers(-10, +10, size=len(idx1)), index=idx1)\n",
        "\n",
        "# Case with duplicates on 1 side \n",
        "idx2 = [*'aabc']\n",
        "s2 = Series(rng.integers(-10, +10, size=len(idx2)), index=idx2)\n",
        "\n",
        "print(\n",
        "    s1,\n",
        "\n",
        "    s2,\n",
        "    f's1.index.has_duplicates={s1.index.has_duplicates}',\n",
        "    f's2.index.has_duplicates={s2.index.has_duplicates}',\n",
        "    s1 + s2,\n",
        "    sep='\\n{}\\n'.format('\\N{box drawings light horizontal}' * 40),\n",
        "\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ez4L0OLQoaAX"
      },
      "source": [
        "we can see that when we add s1+s2 and where we've duplicate the index alignment performs a [`cartesian product`](https://en.wikipedia.org/wiki/Cartesian_product) so,a=7(s1) get added to a= -4(s2) to get 3 and a=7(s1) also goes to a=-10(s2) to get -3."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "z_BdbVH_pA92",
        "outputId": "1cca5193-d159-4c3d-c116-29f774042b55"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "a    7\n",
            "a    2\n",
            "c    0\n",
            "d   -5\n",
            "dtype: int64\n",
            "────────────────────────────────────────\n",
            "a    -4\n",
            "a   -10\n",
            "b    -9\n",
            "c   -10\n",
            "dtype: int64\n",
            "────────────────────────────────────────\n",
            "s1.index.has_duplicates=True\n",
            "────────────────────────────────────────\n",
            "s2.index.has_duplicates=True\n",
            "────────────────────────────────────────\n",
            "a     3.0\n",
            "a    -3.0\n",
            "a    -2.0\n",
            "a    -8.0\n",
            "b     NaN\n",
            "c   -10.0\n",
            "d     NaN\n",
            "dtype: float64\n"
          ]
        }
      ],
      "source": [
        "# case with duplicates on both sides\n",
        "\n",
        "\n",
        "from pandas import Series, Index \n",
        "from numpy.random import default_rng \n",
        "\n",
        "rng = default_rng(0)\n",
        "\n",
        "idx1 = [*'aacd']\n",
        "s1 = Series(rng.integers(-10, +10, size=len(idx1)), index=idx1)\n",
        "\n",
        "# Case with duplicates on 1 side \n",
        "idx2 = [*'aabc']\n",
        "s2 = Series(rng.integers(-10, +10, size=len(idx2)), index=idx2)\n",
        "\n",
        "print(\n",
        "    s1,\n",
        "\n",
        "    s2,\n",
        "    f's1.index.has_duplicates={s1.index.has_duplicates}',\n",
        "    f's2.index.has_duplicates={s2.index.has_duplicates}',\n",
        "    s1 + s2,\n",
        "    sep='\\n{}\\n'.format('\\N{box drawings light horizontal}' * 40),\n",
        "\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "M9w3k4F5pPTQ"
      },
      "source": [
        "Here, we will do the cartesian product of everywhere we see duplicates (2 a's in s1 and 2 a's in s2) will result in 4 a's in the results for that a!"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YWJhEBrmpsPY"
      },
      "source": [
        "**Remember**: Pandas `.index` is not guranteed to be **unique**, **sorted**(its not a primary key!), its just some mechanism to look at the data!"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 132
        },
        "id": "y78V4-EqpgdH",
        "outputId": "5c089b78-2440-443c-b634-92e43e8c1465"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "2022-01-01 00:51:00     True\n",
            "2022-01-01 00:52:00    False\n",
            "2022-01-01 00:53:00    False\n",
            "2022-01-01 00:54:00    False\n",
            "2022-01-01 00:55:00    False\n",
            "                       ...  \n",
            "2022-01-22 11:45:00    False\n",
            "2022-01-22 11:46:00    False\n",
            "2022-01-22 11:47:00    False\n",
            "2022-01-22 11:48:00    False\n",
            "2022-01-22 11:49:00     True\n",
            "Freq: T, Length: 30899, dtype: bool\n",
            "────────────────────────────────────────────────────────────\n",
            "DatetimeIndex(['2022-01-01 00:51:00', '2022-01-01 00:52:00',\n",
            "               '2022-01-01 00:53:00', '2022-01-01 00:54:00',\n",
            "               '2022-01-01 00:55:00', '2022-01-01 00:56:00',\n",
            "               '2022-01-01 00:57:00', '2022-01-01 00:58:00',\n",
            "               '2022-01-01 00:59:00', '2022-01-01 01:00:00',\n",
            "               ...\n",
            "               '2022-01-22 11:40:00', '2022-01-22 11:41:00',\n",
            "               '2022-01-22 11:42:00', '2022-01-22 11:43:00',\n",
            "               '2022-01-22 11:44:00', '2022-01-22 11:45:00',\n",
            "               '2022-01-22 11:46:00', '2022-01-22 11:47:00',\n",
            "               '2022-01-22 11:48:00', '2022-01-22 11:49:00'],\n",
            "              dtype='datetime64[ns]', length=30899, freq='T')\n",
            "────────────────────────────────────────────────────────────\n",
            "DatetimeIndex(['2022-01-01 00:51:00', '2022-01-01 01:29:00',\n",
            "               '2022-01-01 02:16:00', '2022-01-01 02:37:00',\n",
            "               '2022-01-01 02:53:00', '2022-01-01 05:15:00',\n",
            "               '2022-01-01 06:22:00', '2022-01-01 07:20:00',\n",
            "               '2022-01-01 08:04:00', '2022-01-01 08:42:00',\n",
            "               ...\n",
            "               '2022-01-22 02:46:00', '2022-01-22 02:59:00',\n",
            "               '2022-01-22 03:32:00', '2022-01-22 04:31:00',\n",
            "               '2022-01-22 05:08:00', '2022-01-22 05:45:00',\n",
            "               '2022-01-22 08:03:00', '2022-01-22 08:37:00',\n",
            "               '2022-01-22 10:46:00', '2022-01-22 11:49:00'],\n",
            "              dtype='datetime64[ns]', length=506, freq=None)\n",
            "────────────────────────────────────────────────────────────\n",
            "(rs.loc[same_idx] == gb.loc[same_idx]).all =<bound method NDFrame._add_numeric_operations.<locals>.all of 2022-01-01 00:51:00    True\n",
            "2022-01-01 01:29:00    True\n",
            "2022-01-01 02:16:00    True\n",
            "2022-01-01 02:37:00    True\n",
            "2022-01-01 02:53:00    True\n",
            "                       ... \n",
            "2022-01-22 05:45:00    True\n",
            "2022-01-22 08:03:00    True\n",
            "2022-01-22 08:37:00    True\n",
            "2022-01-22 10:46:00    True\n",
            "2022-01-22 11:49:00    True\n",
            "Length: 506, dtype: bool>\n",
            "────────────────────────────────────────────────────────────\n",
            "DatetimeIndex(['2022-01-01 00:51:00', '2022-01-01 01:29:00',\n",
            "               '2022-01-01 02:00:00', '2022-01-01 02:16:00',\n",
            "               '2022-01-01 02:35:00', '2022-01-01 02:37:00',\n",
            "               '2022-01-01 02:42:00', '2022-01-01 02:53:00',\n",
            "               '2022-01-01 03:42:00', '2022-01-01 04:21:00',\n",
            "               ...\n",
            "               '2022-01-22 07:15:00', '2022-01-22 07:38:00',\n",
            "               '2022-01-22 08:03:00', '2022-01-22 08:37:00',\n",
            "               '2022-01-22 09:11:00', '2022-01-22 10:10:00',\n",
            "               '2022-01-22 10:21:00', '2022-01-22 10:46:00',\n",
            "               '2022-01-22 10:59:00', '2022-01-22 11:49:00'],\n",
            "              dtype='datetime64[ns]', length=992, freq=None)\n",
            "────────────────────────────────────────────────────────────\n",
            "DatetimeIndex(['2022-01-01 02:00:00', '2022-01-01 02:35:00',\n",
            "               '2022-01-01 02:42:00', '2022-01-01 03:42:00',\n",
            "               '2022-01-01 04:21:00', '2022-01-01 05:46:00',\n",
            "               '2022-01-01 09:15:00', '2022-01-01 11:01:00',\n",
            "               '2022-01-01 11:50:00', '2022-01-01 12:54:00',\n",
            "               ...\n",
            "               '2022-01-21 23:36:00', '2022-01-22 01:08:00',\n",
            "               '2022-01-22 01:58:00', '2022-01-22 06:22:00',\n",
            "               '2022-01-22 07:15:00', '2022-01-22 07:38:00',\n",
            "               '2022-01-22 09:11:00', '2022-01-22 10:10:00',\n",
            "               '2022-01-22 10:21:00', '2022-01-22 10:59:00'],\n",
            "              dtype='datetime64[ns]', length=486, freq=None)\n",
            "────────────────────────────────────────────────────────────\n",
            "(rs.loc[diff_idx] != gb.loc[diff_idx]).all() =True\n"
          ]
        }
      ],
      "source": [
        "# Example of non-unique index\n",
        "\n",
        "from pandas import to_datetime, to_timedelta, Series, DataFrame\n",
        "from numpy import isclose\n",
        "from numpy.random import default_rng \n",
        "\n",
        "rng = default_rng(0)\n",
        "\n",
        "idx = to_datetime('2022-01-01') + to_timedelta(rng.integers(0, 60*60, size=1_000).cumsum(), unit='s')\n",
        "s = Series(rng.normal(loc=1, scale=.1, size=len(idx)).cumprod(), index=idx)\n",
        "\n",
        "# Doing index manipulation\n",
        "print(\n",
        "    s,\n",
        "    s.resample('1T').mean(),# resampling in-terms of `every-minute`\n",
        "    s.groupby(s.index.round('1T')).mean(), # grouping these value by time rounded to the min and then performing the mean\n",
        "    s.resample('1T').agg(set), # same as the first-opertion but we put values into a `set` this time\n",
        "    s.resample('1T').agg(lambda s: {*s.index}), #instead of putting values; we're putting index contents into set!\n",
        "    s.groupby(s.index.round('1T')).agg(lambda s: {*s.index}), # Doing same thing as above with groupby\n",
        "    DataFrame({\n",
        "        'resample': s.resample('1T').agg(lambda s: {*s.index}),\n",
        "        'groupby': s.groupby(s.index.round('1T')).agg(lambda s: {*s.index}),\n",
        "    }), # Putting our two above commands into a DataFrame so that we can check the difference!\n",
        "    DataFrame({\n",
        "        'resample': s.resample('1T').agg(lambda s: {*s.index}),\n",
        "        'groupby': s.groupby(s.index.round('1T')).agg(lambda s: {*s.index}),\n",
        "        \n",
        "    }).apply(lambda s: s['resample'] == s['groupby'], axis='columns'),  # taking the above DataFrame and checking what they have in common \n",
        "    (comp_idx := DataFrame({\n",
        "            'resample': (rs := s.resample('1T').agg(lambda s: {*s.index})),\n",
        "        'groupby': (gb := s.groupby(s.index.round('1T')).agg(lambda s: {*s.index})),\n",
        "        \n",
        "    }).apply(lambda s: s['resample'] == s['groupby'], axis='columns')),  # we're taking the index of the above result by iteself!\n",
        "    comp_idx.where(comp_idx).index,  # and then we can index this thing where its `True`, will show all the places where results matched up!\n",
        "    same_idx := comp_idx[comp_idx].index,  # computing same indicies \n",
        "    f'{(rs.loc[same_idx] == gb.loc[same_idx]).all =}', #looking at `groupby ` and`resample` where the same indicies are on anywhere where they joined on the same value!\n",
        "    common_idx := rs.index.intersection(gb.index),\n",
        "    diff_idx := common_idx.difference(same_idx), # take the orignial indicies of everything they both have and then subtract out everywhere where they are different\n",
        "    f'{(rs.loc[diff_idx] != gb.loc[diff_idx]).all() =}',\n",
        "        \n",
        "\n",
        "\n",
        "    sep='\\n{}\\n'.format('\\N{box drawings light horizontal}' * 60),\n",
        "   \n",
        ")\n",
        "\n",
        "'''\n",
        "The reason,we went through all this operations is for cases like where you've a very large datasets and its not common for us do various index operations on them (and if we're not \n",
        "using index then we're making a lots of copy of raw_Data ; which consumes a lot of memory!). Instead, a better way will be to take two datasets(or more) and figure out what samples \n",
        "are included in each bucket, find all the bucket where they're same and then compute all the places where they used the same bucket and where they used different bucket; then go back \n",
        "to original dataset to figure out whats the corresponding calculation was for those buckets\n",
        "'''\n",
        "\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qwo4uG04rm0J"
      },
      "source": [
        "`.cumsum()`:https://pandas.pydata.org/docs/reference/api/pandas.Series.cumsum.html  \\\n",
        "`.cumprod()`:https://pandas.pydata.org/docs/reference/api/pandas.Series.cumprod.html \\\n",
        "`:=`  a.k.a \"walrus Operator\": https://realpython.com/python-walrus-operator/ "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### Looking at Pandas `DataFrame`"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 152,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "        date ticker      price  volume\n",
            "0 2022-01-01   wqnh  32.147952  -39700\n",
            "1 2022-01-01   ibba  18.696088    8400\n",
            "2 2022-01-01   evqx  67.455290  -44800\n",
            "────────────────────────────────────────────────────────────\n",
            "wqnh\n",
            "────────────────────────────────────────────────────────────\n",
            "2022-01-01 00:00:00\n",
            "────────────────────────────────────────────────────────────\n",
            "wqnh\n",
            "────────────────────────────────────────────────────────────\n",
            "date      2022-01-01 00:00:00\n",
            "ticker                   wqnh\n",
            "price               32.147952\n",
            "volume                 -39700\n",
            "Name: 0, dtype: object\n",
            "────────────────────────────────────────────────────────────\n",
            "0        wqnh\n",
            "1        ibba\n",
            "2        evqx\n",
            "3        npzs\n",
            "4        qooy\n",
            "         ... \n",
            "36495    tfun\n",
            "36496    tnhc\n",
            "36497    nzao\n",
            "36498    laou\n",
            "36499    tznp\n",
            "Name: ticker, Length: 36500, dtype: object\n"
          ]
        }
      ],
      "source": [
        "from pandas import DataFrame, date_range\n",
        "from numpy import repeat, tile\n",
        "from numpy.random import default_rng\n",
        "from string import ascii_lowercase\n",
        "\n",
        "rng = default_rng(0)\n",
        "\n",
        "tickers = rng.choice([*ascii_lowercase], size=(100,4)).view('<U4').ravel()\n",
        "dates = date_range('2022-01-01', '2022-12-31', name='date')\n",
        "prices = (\n",
        "    100 * rng.random(size=len(tickers))\n",
        "    * rng.normal(loc=1, scale=0.01, size=(len(dates), len(tickers))).cumprod(axis=0)\n",
        ").ravel()\n",
        "volumes = rng.integers(-50_000, +50_000, size=len(dates) * len(tickers)).round(-2)\n",
        "\n",
        "df = DataFrame({\n",
        "    'date' : repeat(dates, len(tickers)),\n",
        "    'ticker': tile(tickers, len(dates)),\n",
        "    'price': prices,\n",
        "    'volume': volumes,\n",
        "\n",
        "\n",
        "})#.set_index(['date', 'ticker'])  # If we un-comment this then we can see that we have two 1D data `price` and `volume` on same index `date` and `ticker`\n",
        "\n",
        "print(\n",
        "    df.head(3),\n",
        "    df.loc[0, 'ticker'], # return element at label `0` and ticekr column\n",
        "    df.iloc[0, 0], # return element at position `0` and column`0`\n",
        "    df.loc[0]['ticker'], # same as df.loc[0].loc['ticker']\n",
        "    df.loc[0],\n",
        "    df.loc[:, 'ticker'], # returns everyhting for the `ticker` column and for every particualr row\n",
        "    sep='\\n{}\\n'.format('\\N{box drawings light horizontal}' * 60),\n",
        ")\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "`.tile()` : https://numpy.org/doc/stable/reference/generated/numpy.tile.html \\\n",
        "`.ravel()`: https://numpy.org/doc/stable/reference/generated/numpy.ravel.html#numpy.ravel \n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "A `Series` is singly-indexed one-dimensional data \\\n",
        "A `DataFrame` is doubly-indexed \\\n",
        "The `DataFrame` is a doubly-indexed collection of like-indexed one-dimensional data \\\n",
        "\n",
        "Thus, `Pandas` is really just an indexed 1D data  like `Series` or a collection of liked indexed 1D data like `DataFrame`  \\\n",
        "\n",
        "Real 2D data is for eg an image, where if we roate it then there is a slight change in that image but the fundamental meaning reamins the same  \\\n",
        "whereas in pandas if we rotate the data then we end up with something entirely new!\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 153,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "      0     1     2\n",
            "0  0.13 -0.13  0.64\n",
            "1  0.10 -0.54  0.36\n",
            "2  1.30  0.95 -0.70\n",
            "3 -1.27 -0.62  0.04\n",
            "4 -2.33 -0.22 -1.25\n",
            "────────────────────────────────────────────────────────────\n",
            "0    0.13\n",
            "1   -0.13\n",
            "2    0.64\n",
            "Name: 0, dtype: float64\n",
            "────────────────────────────────────────────────────────────\n",
            "type(df.loc[0]) = <class 'pandas.core.series.Series'>\n",
            "────────────────────────────────────────────────────────────\n",
            "0    0.13\n",
            "1    0.10\n",
            "2    1.30\n",
            "3   -1.27\n",
            "4   -2.33\n",
            "Name: 0, dtype: float64\n",
            "────────────────────────────────────────────────────────────\n",
            "type(df[0]) = <class 'pandas.core.series.Series'>\n",
            "────────────────────────────────────────────────────────────\n",
            "type(df.loc[0]) = <class 'pandas.core.series.Series'>\n",
            "────────────────────────────────────────────────────────────\n",
            "type(df.loc[2]) = <class 'pandas.core.series.Series'>\n"
          ]
        }
      ],
      "source": [
        "from pandas import DataFrame \n",
        "from numpy.random import default_rng \n",
        "\n",
        "rng = default_rng(0)\n",
        "\n",
        "df = DataFrame(rng.normal(size=(5, 3)).round(2))\n",
        "\n",
        "# idx = [0, 0, 1, 1, 2]\n",
        "# df = DataFrame(rng.normal(size=(len(idx),3)).round(2), index=idx)\n",
        "# df.columns = [0, 1, 1]\n",
        "\n",
        "\n",
        "print(\n",
        "    df,\n",
        "    df.loc[0], # access one row of the dataframe ; return to as pandas series\n",
        "    f'{type(df.loc[0]) = }',\n",
        "    df[0], # access one column\n",
        "    f'{type(df[0]) = }',\n",
        "    f'{type(df.loc[0]) = }', # This will return a Pandas `Series`\n",
        "    f'{type(df.loc[2]) = }', # This will return a Pandas `Series`\n",
        "\n",
        "\n",
        "    sep='\\n{}\\n'.format('\\N{box drawings light horizontal}' * 60),\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "But if we change the `index` then let see the change in `f'{type(df.loc[0]) = }'` will show"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 154,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "      0     1     1\n",
            "0 -0.73 -0.54 -0.32\n",
            "0  0.41  1.04 -0.13\n",
            "1  1.37 -0.67  0.35\n",
            "1  0.90  0.09 -0.74\n",
            "2 -0.92 -0.46  0.22\n",
            "────────────────────────────────────────────────────────────\n",
            "df.iloc[0] = 0   -0.73\n",
            "1   -0.54\n",
            "1   -0.32\n",
            "Name: 0, dtype: float64\n",
            "────────────────────────────────────────────────────────────\n",
            "<class 'pandas.core.series.Series'>\n",
            "────────────────────────────────────────────────────────────\n",
            "df.iloc[0].index = Int64Index([0, 1, 1], dtype='int64')\n",
            "────────────────────────────────────────────────────────────\n",
            "df.iloc[0].name = 0\n",
            "────────────────────────────────────────────────────────────\n",
            "df[0] = 0   -0.73\n",
            "0    0.41\n",
            "1    1.37\n",
            "1    0.90\n",
            "2   -0.92\n",
            "Name: 0, dtype: float64\n",
            "────────────────────────────────────────────────────────────\n",
            "df[0].index =Int64Index([0, 0, 1, 1, 2], dtype='int64')\n",
            "────────────────────────────────────────────────────────────\n",
            "df[0].name =0\n",
            "────────────────────────────────────────────────────────────\n",
            "-0.73\n",
            "────────────────────────────────────────────────────────────\n",
            "1    0.90\n",
            "2   -0.92\n",
            "Name: 0, dtype: float64\n"
          ]
        }
      ],
      "source": [
        "from pandas import DataFrame \n",
        "from numpy.random import default_rng \n",
        "\n",
        "rng = default_rng(0)\n",
        "\n",
        "df = DataFrame(rng.normal(size=(5, 3)).round(2))\n",
        "\n",
        "idx = [0, 0, 1, 1, 2]\n",
        "df = DataFrame(rng.normal(size=(len(idx),3)).round(2), index=idx)\n",
        "df.columns = [0, 1, 1]\n",
        "\n",
        "\n",
        "print(\n",
        "    df,\n",
        "    # df.loc[0], # access one row of the dataframe ; return to as pandas series\n",
        "    # f'{type(df.loc[0]) = }',\n",
        "    # df[0], # access one column\n",
        "    # f'{type(df[0]) = }',\n",
        "    # f'{type(df.loc[0]) = }', # This will return a Pandas `DataFrame` instead of a series now because we have two rows with `0`!\n",
        "    # f'{type(df.loc[2]) = }', # This will return a Pandas `Series` because they are not two values at the position !\n",
        "    \n",
        "    \n",
        "    # Why  `.iloc` is a very useful tool in such scenarios :\n",
        "\n",
        "    # - `iloc` is unambiguous, it always represent the value a physical positon , thus `.iloc[0]` always means at physical position `0`\n",
        "    #     and there can't be two physical position `0`.\n",
        "    # - `.iloc` can be thought as an unique range-index starting from  unit`0` to the size of your structure!\n",
        "    \n",
        "    f'{df.iloc[0] = }',\n",
        "    f'{type(df.iloc[0])}',\n",
        "    f'{df.iloc[0].index = }', # This means that if the orginial df is index on the columns[0,1,1] and rows[0,0,1,2] and I asked for row'0'; \n",
        "                                # the indexing of the result will take the column index(That is why its doubly-indexed in Dataframe):-> Use column index when we look at the rows\n",
        "                                # and rows index when we look at the columns\n",
        "    f'{df.iloc[0].name = }',\n",
        "    # df.iloc[2],\n",
        "    # f'{df.iloc[2].index = }',\n",
        "    # f'{df.iloc[2].name = }',\n",
        "    f'{df[0] = }',\n",
        "    f'{df[0].index =}',\n",
        "    f'{df[0].name =}',\n",
        "\n",
        "    df.iloc[0][0], # By label\n",
        "    df[0][3:], # by position\n",
        "\n",
        "    sep='\\n{}\\n'.format('\\N{box drawings light horizontal}' * 60),\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### `SettingWithCopyWarning` in Pandas"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 155,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "   a     b  c\n",
            "0  7  0.10  b\n",
            "1  2 -0.54  q\n",
            "2  0  0.36  o\n",
            "3 -5  1.30  o\n",
            "4 -4  0.95  y\n",
            "────────────────────────────────────────────────────────────\n",
            "a      7\n",
            "b    0.1\n",
            "c      b\n",
            "Name: 0, dtype: object\n",
            "────────────────────────────────────────────────────────────\n",
            "type(df.iloc[0]) = <class 'pandas.core.series.Series'>\n",
            "────────────────────────────────────────────────────────────\n",
            "df.iloc[0].dtype = dtype('O')\n"
          ]
        }
      ],
      "source": [
        "from pandas import DataFrame \n",
        "from numpy.random import default_rng \n",
        "from string import ascii_lowercase \n",
        "from warnings import catch_warnings, filterwarnings \n",
        "\n",
        "\n",
        "rng = default_rng(0)\n",
        "\n",
        "df = DataFrame({\n",
        "    'a' : rng.integers(-10, +10, size=(size := 5)),\n",
        "    'b' : rng.normal(size=size).round(2),\n",
        "    'c' : rng.choice([*ascii_lowercase],size=size),\n",
        "\n",
        "})\n",
        "\n",
        "# Uncomment each at a time to see the behaviour!\n",
        "with catch_warnings():\n",
        "    filterwarnings('error')\n",
        "    # df.iloc[0]['c'] = 'z'                     # This return an error\n",
        "\n",
        "    # df.iloc[0, 'c'] = 'z'                     # This works! \n",
        "    # df.iloc[0, df.columns.get_loc('c')] = 'z' # This works!\n",
        "    # df.loc[df.index.get_loc(0), 'c'] = 'z'    # This works!\n",
        "\n",
        "\n",
        "\n",
        "print(\n",
        "    df,\n",
        "    df.iloc[0],\n",
        "    f'{type(df.iloc[0]) = }',\n",
        "    f'{df.iloc[0].dtype = }',\n",
        "    sep='\\n{}\\n'.format('\\N{box drawings light horizontal}' * 60)\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "   a     b  c\n",
            "0  7  0.10  b\n",
            "1  2 -0.54  q\n",
            "2  0  0.36  o\n",
            "3 -5  1.30  o\n",
            "4 -4  0.95  y\n",
            "────────────────────────────────────────────────────────────\n",
            "BlockManager\n",
            "Items: Index(['a', 'b', 'c'], dtype='object')\n",
            "Axis 1: RangeIndex(start=0, stop=5, step=1)\n",
            "NumericBlock: slice(0, 1, 1), 1 x 5, dtype: int64\n",
            "NumericBlock: slice(1, 2, 1), 1 x 5, dtype: float64\n",
            "ObjectBlock: slice(2, 3, 1), 1 x 5, dtype: object\n",
            "────────────────────────────────────────────────────────────\n",
            "RangeIndex(start=0, stop=5, step=1)\n",
            "────────────────────────────────────────────────────────────\n",
            "Index(['a', 'b', 'c'], dtype='object')\n"
          ]
        }
      ],
      "source": [
        "from pandas import DataFrame \n",
        "from numpy.random import default_rng \n",
        "from string import ascii_lowercase \n",
        "\n",
        "\n",
        "\n",
        "rng = default_rng(0)\n",
        "\n",
        "df = DataFrame({\n",
        "    'a' : rng.integers(-10, +10, size=(size := 5)),\n",
        "    'b' : rng.normal(size=size).round(2),\n",
        "    'c' : rng.choice([*ascii_lowercase],size=size),\n",
        "\n",
        "})\n",
        "\n",
        "print(\n",
        "    df,\n",
        "    df._data,\n",
        "    df.index,   # Also called as `row-index` or the `major-axis`\n",
        "    df.columns, # Also called as `column-index` or the `minor-axis`\n",
        "    sep='\\n{}\\n'.format('\\N{box drawings light horizontal}' * 60)\n",
        "\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### Finally! what are the rules of `index alignment` on `DataFrame`?"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 156,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "            a   b  c\n",
            "date                \n",
            "2022-01-01  7 -10  2\n",
            "2022-01-02  2  -9  8\n",
            "2022-01-03  0 -10  0\n",
            "2022-01-04 -5  -7  2\n",
            "2022-01-05 -4   6  9\n",
            "────────────────────────────────────────────────────────────\n",
            "            a   b   c\n",
            "date                 \n",
            "2022-01-02  4  -5   7\n",
            "2022-01-03  2   6   1\n",
            "2022-01-04  0   3 -10\n",
            "2022-01-05  1 -10   5\n",
            "2022-01-06  8  -3   4\n",
            "────────────────────────────────────────────────────────────\n",
            "              a     b     c\n",
            "date                       \n",
            "2022-01-01  NaN   NaN   NaN\n",
            "2022-01-02  6.0 -14.0  15.0\n",
            "2022-01-03  2.0  -4.0   1.0\n",
            "2022-01-04 -5.0  -4.0  -8.0\n",
            "2022-01-05 -3.0  -4.0  14.0\n",
            "2022-01-06  NaN   NaN   NaN\n"
          ]
        }
      ],
      "source": [
        "from pandas import DataFrame, Series, date_range, concat\n",
        "from numpy.random import default_rng\n",
        "\n",
        "# rng = default_rng(0)\n",
        "# size = 5\n",
        "# data = {'a' : rng.integers(-10, +10, size=size), 'b' : rng.integers(-10, +10, size=size), 'c' : rng.integers(-10, +10, size=size)}\n",
        "\n",
        "# df1 = DataFrame(data, index=date_range('2022-01-01', periods=size, name='date') )\n",
        "# df2 = DataFrame(data, index=date_range('2022-01-02', periods=size, name='date'))\n",
        "\n",
        "\n",
        "#You can write your `DataFrame`` this way or comment this and un-comment the above part! I personally like this way :)\n",
        "rng = default_rng(0)\n",
        "size = 5\n",
        "df1 = DataFrame(\n",
        "    {\n",
        "    'a' : rng.integers(-10, +10, size=size),\n",
        "    'b' : rng.integers(-10, +10, size=size),\n",
        "    'c' : rng.integers(-10, +10, size=size),\n",
        "    },\n",
        "    index=date_range('2022-01-01', periods=size, name='date')\n",
        "    )\n",
        "\n",
        "df2 = DataFrame(\n",
        "    {\n",
        "    'a' : rng.integers(-10, +10, size=size),\n",
        "    'b' : rng.integers(-10, +10, size=size),\n",
        "    'c' : rng.integers(-10, +10, size=size),\n",
        "    # 'd' : rng.integers(-10, +10, size=size),\n",
        "    }, \n",
        "    index=date_range('2022-01-02', periods=size, name='date')\n",
        ")\n",
        "\n",
        "\n",
        "print(\n",
        "    df1,\n",
        "    df2,\n",
        "    df1+df2, # will get NaN's at the top and bottom because we're missing `2022-01-01` in df2 and `2022-01-06` from df1 but since columns are the same; it gonna match columns by name!\n",
        "\n",
        "    sep='\\n{}\\n'.format('\\N{box drawings light horizontal}' * 60),\n",
        "\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 157,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "            a   b  c\n",
            "date                \n",
            "2022-01-01  7 -10  2\n",
            "2022-01-02  2  -9  8\n",
            "2022-01-03  0 -10  0\n",
            "2022-01-04 -5  -7  2\n",
            "2022-01-05 -4   6  9\n",
            "────────────────────────────────────────────────────────────\n",
            "            a   b   c   d\n",
            "date                     \n",
            "2022-01-02  4  -5   7   6\n",
            "2022-01-03  2   6   1  -7\n",
            "2022-01-04  0   3 -10  -9\n",
            "2022-01-05  1 -10   5   7\n",
            "2022-01-06  8  -3   4 -10\n",
            "────────────────────────────────────────────────────────────\n",
            "              a     b     c   d\n",
            "date                           \n",
            "2022-01-01  NaN   NaN   NaN NaN\n",
            "2022-01-02  6.0 -14.0  15.0 NaN\n",
            "2022-01-03  2.0  -4.0   1.0 NaN\n",
            "2022-01-04 -5.0  -4.0  -8.0 NaN\n",
            "2022-01-05 -3.0  -4.0  14.0 NaN\n",
            "2022-01-06  NaN   NaN   NaN NaN\n"
          ]
        }
      ],
      "source": [
        "# adding Extra Column!\n",
        "\n",
        "rng = default_rng(0)\n",
        "size = 5\n",
        "df1 = DataFrame(\n",
        "    {\n",
        "    'a' : rng.integers(-10, +10, size=size),\n",
        "    'b' : rng.integers(-10, +10, size=size),\n",
        "    'c' : rng.integers(-10, +10, size=size),\n",
        "    },\n",
        "    index=date_range('2022-01-01', periods=size, name='date')\n",
        "    )\n",
        "\n",
        "df2 = DataFrame(\n",
        "    {\n",
        "    'a' : rng.integers(-10, +10, size=size),\n",
        "    'b' : rng.integers(-10, +10, size=size),\n",
        "    'c' : rng.integers(-10, +10, size=size),\n",
        "    'd' : rng.integers(-10, +10, size=size),\n",
        "    }, \n",
        "    index=date_range('2022-01-02', periods=size, name='date')\n",
        ")\n",
        "\n",
        "\n",
        "print(\n",
        "    df1,\n",
        "    df2,\n",
        "    df1+df2, # This will result in getting `NaN's` in top and bottom as above but with addition of getting NaN in the extra column `d` ! \n",
        "\n",
        "    sep='\\n{}\\n'.format('\\N{box drawings light horizontal}' * 60),\n",
        "\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 158,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "            a   b  c\n",
            "date                \n",
            "2022-01-01  7 -10  2\n",
            "2022-01-02  2  -9  8\n",
            "2022-01-03  0 -10  0\n",
            "2022-01-04 -5  -7  2\n",
            "2022-01-05 -4   6  9\n",
            "────────────────────────────────────────────────────────────\n",
            "date\n",
            "2022-01-01    4\n",
            "2022-01-02    2\n",
            "2022-01-03    0\n",
            "2022-01-04    1\n",
            "2022-01-05    8\n",
            "Freq: D, dtype: int64\n"
          ]
        }
      ],
      "source": [
        "from pandas import DataFrame, Series, date_range\n",
        "from numpy.random import default_rng \n",
        "\n",
        "rng = default_rng(0)\n",
        "size = 5\n",
        "df = DataFrame(\n",
        "    {\n",
        "    'a' : rng.integers(-10, +10, size=size),\n",
        "    'b' : rng.integers(-10, +10, size=size),\n",
        "    'c' : rng.integers(-10, +10, size=size),\n",
        "    },\n",
        "    index=date_range('2022-01-01', periods=size, name='date')\n",
        "    )\n",
        "\n",
        "s = Series(rng.integers(-10, +10, size=len(df)), index=df.index)\n",
        "\n",
        "print(\n",
        "    df,\n",
        "    s,\n",
        "    # df + s, # took the `row-index` of the `Series` and try to match against the columns ; will give the same result for `s + df`\n",
        "    # df + s.T, # Transpose dosen't work  because the transposition of a pandas `Series` is the `Series` itself! Will show a similar result for `(df.T + s).T`\n",
        "    sep='\\n{}\\n'.format('\\N{box drawings light horizontal}' * 60),\n",
        "\n",
        ")\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 159,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "            a   b  c\n",
            "date                \n",
            "2022-01-01  7 -10  2\n",
            "2022-01-02  2  -9  8\n",
            "2022-01-03  0 -10  0\n",
            "2022-01-04 -5  -7  2\n",
            "2022-01-05 -4   6  9\n",
            "────────────────────────────────────────────────────────────\n",
            "a    4\n",
            "b    2\n",
            "c    0\n",
            "dtype: int64\n",
            "────────────────────────────────────────────────────────────\n",
            "             a  b  c\n",
            "date                \n",
            "2022-01-01  11 -8  2\n",
            "2022-01-02   6 -7  8\n",
            "2022-01-03   4 -8  0\n",
            "2022-01-04  -1 -5  2\n",
            "2022-01-05   0  8  9\n"
          ]
        }
      ],
      "source": [
        "from pandas import DataFrame, Series, date_range\n",
        "from numpy.random import default_rng \n",
        "\n",
        "rng = default_rng(0)\n",
        "size = 5\n",
        "df = DataFrame(\n",
        "    {\n",
        "    'a' : rng.integers(-10, +10, size=size),\n",
        "    'b' : rng.integers(-10, +10, size=size),\n",
        "    'c' : rng.integers(-10, +10, size=size),\n",
        "    },\n",
        "    index=date_range('2022-01-01', periods=size, name='date')\n",
        "    )\n",
        "\n",
        "s = Series(rng.integers(-10, +10, size=len(df.columns)), index=df.columns) # Same `Series` but the index is equal to `columns-indexing` of our `DataFrame`\n",
        "\n",
        "print(\n",
        "    df,\n",
        "    s,\n",
        "    df + s, # Now we add these together!\n",
        "\n",
        "    sep='\\n{}\\n'.format('\\N{box drawings light horizontal}' * 60),\n",
        "\n",
        ")\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 160,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "ticker\n",
            "wqnh    0.998365\n",
            "ibba    0.853698\n",
            "evqx    1.063613\n",
            "npzs    1.009087\n",
            "qooy    0.962868\n",
            "          ...   \n",
            "tfun    1.004738\n",
            "tnhc    1.168481\n",
            "nzao    1.063286\n",
            "laou    0.913663\n",
            "tznp    1.071009\n",
            "Name: factor, Length: 100, dtype: float64\n",
            "────────────────────────────────────────────────────────────\n",
            "date        ticker\n",
            "2022-01-01  arnq       38.749610\n",
            "            awkb       96.332418\n",
            "            awyd       85.292694\n",
            "            bcjv       87.471962\n",
            "            bjqc        9.247532\n",
            "                         ...    \n",
            "2022-12-31  yqjz       48.747306\n",
            "            ysia      123.215568\n",
            "            zglo       85.323680\n",
            "            zhnw       51.300465\n",
            "            zyrz       58.677036\n",
            "Name: price, Length: 36500, dtype: float64\n",
            "────────────────────────────────────────────────────────────\n",
            "date        ticker\n",
            "2022-01-01  arnq       37.132688\n",
            "            awkb       90.844435\n",
            "            awyd       86.337632\n",
            "            bcjv       79.312476\n",
            "            bjqc        9.923136\n",
            "                         ...    \n",
            "2022-12-31  yqjz       53.585677\n",
            "            ysia      118.010350\n",
            "            zglo       74.540744\n",
            "            zhnw       46.382505\n",
            "            zyrz       56.982783\n",
            "Length: 36500, dtype: float64\n"
          ]
        }
      ],
      "source": [
        "from pandas import DataFrame, date_range, IndexSlice, Series \n",
        "from numpy import repeat, tile \n",
        "from numpy.random import default_rng \n",
        "from string import ascii_lowercase \n",
        "\n",
        "rng = default_rng(0)\n",
        "\n",
        "tickers = rng.choice([*ascii_lowercase], size=(100,4)).view('<U4').ravel()\n",
        "dates = date_range('2022-01-01', '2022-12-31', name='date')\n",
        "prices = (\n",
        "    100 * rng.random(size=len(tickers))\n",
        "    * rng.normal(loc=1, scale=0.01, size=(len(dates), len(tickers))).cumprod(axis=0)\n",
        ").ravel()\n",
        "volumes = rng.integers(-50_000, +50_000, size=len(dates) * len(tickers)).round(-2)\n",
        "\n",
        "df = DataFrame({\n",
        "    'date' : repeat(dates, len(tickers)),\n",
        "    'ticker' : tile(tickers, len(dates)),\n",
        "    'price' : prices,\n",
        "    'volume' : volumes,\n",
        "}).set_index(['date', 'ticker']).sort_index()\n",
        "\n",
        "factor = Series(\n",
        "    rng.normal(loc=1, scale=.1, size=len(tickers)),\n",
        "    index=tickers,\n",
        "    name='factor', \n",
        ")\n",
        "factor.index.name = 'ticker'\n",
        "\n",
        "print(\n",
        "    # df,\n",
        "    factor, # Only index by ticker!\n",
        "    df['price'], # This has multi-index, both by `date` and `ticker`\n",
        "    # df.head(3),\n",
        "    # df.loc[IndexSlice[:, 'arnq', :]].sample(3),\n",
        "    # factor.head(3),\n",
        "    df['price'] * factor, # Pandas is smart enough to figure out by matching each one in the factor to everyone in the price\n",
        "                          # Would have taken us so much of PythonCode+looping if we try to built it manually! Thanks Pandas team!!  \n",
        "    sep='\\n{}\\n'.format('\\N{box drawings light horizontal}' * 60),\n",
        "\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 161,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "quarter  symbol\n",
            "2022Q1   abc        7\n",
            "         def        2\n",
            "         xyz        0\n",
            "2022Q2   abc       -5\n",
            "         def       -4\n",
            "         xyz      -10\n",
            "2022Q3   abc       -9\n",
            "         def      -10\n",
            "         xyz       -7\n",
            "2022Q4   abc        6\n",
            "         def        2\n",
            "         xyz        8\n",
            "dtype: int64\n",
            "────────────────────────────────────────────────────────────\n",
            "MultiIndex([('2022Q1', 'abc'),\n",
            "            ('2022Q1', 'def'),\n",
            "            ('2022Q1', 'xyz'),\n",
            "            ('2022Q2', 'abc'),\n",
            "            ('2022Q2', 'def'),\n",
            "            ('2022Q2', 'xyz'),\n",
            "            ('2022Q3', 'abc'),\n",
            "            ('2022Q3', 'def'),\n",
            "            ('2022Q3', 'xyz'),\n",
            "            ('2022Q4', 'abc'),\n",
            "            ('2022Q4', 'def'),\n",
            "            ('2022Q4', 'xyz')],\n",
            "           names=['quarter', 'symbol'])\n",
            "────────────────────────────────────────────────────────────\n",
            "symbol\n",
            "abc    7\n",
            "def    2\n",
            "xyz    0\n",
            "dtype: int64\n",
            "────────────────────────────────────────────────────────────\n",
            "7\n",
            "────────────────────────────────────────────────────────────\n",
            "quarter  symbol\n",
            "2022Q2   abc       -5\n",
            "         xyz      -10\n",
            "dtype: int64\n",
            "────────────────────────────────────────────────────────────\n",
            "quarter  symbol\n",
            "2022Q1   abc        7\n",
            "         xyz        0\n",
            "2022Q2   abc       -5\n",
            "         xyz      -10\n",
            "2022Q3   abc       -9\n",
            "         xyz       -7\n",
            "2022Q4   abc        6\n",
            "         xyz        8\n",
            "dtype: int64\n"
          ]
        }
      ],
      "source": [
        "# Example of Multi-Index !\n",
        "\n",
        "from pandas import Series , MultiIndex, period_range, IndexSlice \n",
        "from numpy.random import default_rng \n",
        "\n",
        "rng = default_rng(0)\n",
        "\n",
        "s = Series(\n",
        "    index=(idx := MultiIndex.from_product([\n",
        "        period_range('2022Q1', periods=4, freq='Q', name='quarter'),\n",
        "        ['abc', 'def', 'xyz'],\n",
        "    ], names=['quarter', 'symbol'])),\n",
        "    data=rng.integers(-10, +10, size=len(idx)),\n",
        ")\n",
        "\n",
        "print(\n",
        "    s,\n",
        "    s.index,\n",
        "    s.loc['2022Q1'],\n",
        "    s.loc['2022Q1', 'abc'],\n",
        "    s.loc[IndexSlice['2022Q2', ['abc', 'xyz'], :]],\n",
        "    s.loc[IndexSlice['2022-03-01':, ['abc', 'xyz'], :]],\n",
        "    sep='\\n{}\\n'.format('\\N{box drawings light horizontal}' * 60),\n",
        "\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### `MultiIndex` is not the only hierarchical `Index`!"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 162,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "date\n",
            "2022-01-01     7\n",
            "2022-01-15     2\n",
            "2022-01-29     0\n",
            "2022-02-12    -5\n",
            "2022-02-26    -4\n",
            "2022-03-12   -10\n",
            "Freq: 14D, dtype: int64\n",
            "────────────────────────────────────────────────────────────\n",
            "DatetimeIndex(['2022-01-01', '2022-01-15', '2022-01-29', '2022-02-12',\n",
            "               '2022-02-26', '2022-03-12'],\n",
            "              dtype='datetime64[ns]', name='date', freq='14D')\n",
            "────────────────────────────────────────────────────────────\n",
            "date\n",
            "2022-01-01     7\n",
            "2022-01-15     2\n",
            "2022-01-29     0\n",
            "2022-02-12    -5\n",
            "2022-02-26    -4\n",
            "2022-03-12   -10\n",
            "Freq: 14D, dtype: int64\n",
            "────────────────────────────────────────────────────────────\n",
            "date\n",
            "2022-01-01    7\n",
            "2022-01-15    2\n",
            "2022-01-29    0\n",
            "Freq: 14D, dtype: int64\n"
          ]
        }
      ],
      "source": [
        "from pandas import Series, date_range \n",
        "from numpy.random import default_rng \n",
        "\n",
        "rng = default_rng(0)\n",
        "\n",
        "s = Series(\n",
        "    index=(idx := date_range('2022-01-01', periods=6, name= 'date', freq='14D')),\n",
        "    data=rng.integers(-10, +10, size=len(idx)),\n",
        ")\n",
        "# `DatetimeIndex` is actually a `MultiIndex` where each level is each fidelity of time measurement [year, month , day and so on..]\n",
        "print(\n",
        "    s,\n",
        "    s.index,\n",
        "    s.loc['2022'], # If we just ask for the year then this will return all the rows for that year just like `MultiIndex` for just one layer of that Index\n",
        "    s.loc['2022-01'], # `DatetimeIndex` is same as MultiIndex with 3-levels : Year-Month-Day ; stored as one particular `Index` and without syntax of `MultiIndex`\n",
        "    sep='\\n{}\\n'.format('\\N{box drawings light horizontal}' * 60),\n",
        "\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### Operations in `pandas` should be thought in-terms of how they operate on or affect the `.index`"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 163,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "date     date      \n",
            "2022-01  2022-01-01     7\n",
            "         2022-01-02     9\n",
            "         2022-01-10    15\n",
            "         2022-01-11    17\n",
            "         2022-01-12    25\n",
            "                       ..\n",
            "2024-09  2024-09-17    25\n",
            "         2024-09-20    26\n",
            "         2024-09-21    27\n",
            "         2024-09-22    36\n",
            "         2024-09-26    42\n",
            "Length: 486, dtype: int64\n"
          ]
        }
      ],
      "source": [
        "from pandas import Series, date_range \n",
        "from numpy.random import default_rng \n",
        "from scipy.stats import zscore, kurtosis\n",
        "\n",
        "rng = default_rng(0)\n",
        "\n",
        "s = Series(\n",
        "    index=(idx := date_range('2022-01-01', periods=1_000, name= 'date', freq='D')),\n",
        "    data=rng.integers(-10, +10, size=len(idx)),\n",
        ")\n",
        "\n",
        "print(\n",
        "    # s, # Series indexd on individual days\n",
        "    # s.groupby(s.index.month).mean(), # Group it by month and take the mean\n",
        "    # s.groupby(s.index.to_period('M')).mean(), # Map actual dates to months!\n",
        "    # s.groupby(s.index.to_period('M')).mean().loc['2022-02-14':'2022-05-01'],\n",
        "    # s.groupby(s.index.to_period('M')).agg(lambda s: s.skew()), # if we want our groups to become the new `index` then we use `.agg()`; \n",
        "    #                                                             #`.agg()` takes the groups & collaspe them into one scalar value(that is indexed on each of the groups)\n",
        "    # s.groupby(s.index.to_period('M')).transform(zscore), # `.transform()` is about preserving  orginal indexing\n",
        "    s.groupby(s.index.to_period('M')).apply(lambda df: df[df > 0].cumsum()), # `.apply` is about creating brand-new `index` and then concat it into a `pandas df` later on!\n",
        "    sep='\\n{}\\n'.format('\\N{box drawings light horizontal}' * 60),\n",
        ")\n",
        "\n",
        "# `.agg()` and `.transform` works on `Series` basis while `.apply` works on `DataFrame` basis"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "                       price        volume\n",
            "ticker date                               \n",
            "arnq   2022-01-31  38.698474   2309.677419\n",
            "       2022-02-28  38.094984  -5392.857143\n",
            "       2022-03-31  38.173743   7319.354839\n",
            "       2022-04-30  40.141961  -8283.333333\n",
            "       2022-05-31  39.877949   3009.677419\n",
            "...                      ...           ...\n",
            "zyrz   2022-08-31  66.316170  -3712.903226\n",
            "       2022-09-30  64.247084   2530.000000\n",
            "       2022-10-31  62.633014   -919.354839\n",
            "       2022-11-30  59.078532  12186.666667\n",
            "       2022-12-31  59.668530  -5664.516129\n",
            "\n",
            "[1200 rows x 2 columns]\n"
          ]
        }
      ],
      "source": [
        "# More on `DataFrame`\n",
        "\n",
        "from turtle import title\n",
        "from pandas import DataFrame , date_range, Series, IndexSlice, to_datetime, MultiIndex \n",
        "from numpy import repeat, tile\n",
        "from numpy.random import default_rng \n",
        "from string import ascii_lowercase\n",
        "from datetime import timedelta \n",
        "\n",
        "rng = default_rng(0)\n",
        "\n",
        "tickers = rng.choice([*ascii_lowercase], size=(100, 4)).view('<U4').ravel()\n",
        "dates = date_range('2022-01-01', '2022-12-31', name='date')\n",
        "prices = (\n",
        "    100 * rng.random(size=len(tickers))\n",
        "    * rng.normal(loc=1, scale=0.01, size=(len(dates), len(tickers))).cumprod(axis=0)\n",
        ").ravel()\n",
        "volumes = rng.integers(-50_000, +50_000, size=len(dates) * len(tickers)).round(-2)\n",
        "\n",
        "df = DataFrame({\n",
        "    'date' : repeat(dates, len(tickers)),\n",
        "    'ticker' : tile(tickers, len(dates)),\n",
        "    'price' : prices,\n",
        "    'volume' : volumes,\n",
        "}).set_index(['date', 'ticker']).sort_index()\n",
        "\n",
        "\n",
        "print(\n",
        "    # df.head(3),\n",
        "    # df.index, df.columns, \n",
        "    # df.axes,              # `.axes ` is just `index` and`columns` in a tuple\n",
        "    # df.unstack(),           # unstack the prescribed level(s) from row to column. \n",
        "    # df.stack(),           # : stack the prescribed level(s) from column to row.\n",
        "    # _df := DataFrame({\n",
        "    #     'sector' : rng.choice(['tech', 'healthcare', 'energy'], size=(size := 5)),\n",
        "    #     to_datetime('2022-01-01'): rng.normal(size=size),\n",
        "    #     to_datetime('2022-01-02'): rng.normal(size=size),\n",
        "    #     to_datetime('2022-01-03'): rng.normal(size=size),\n",
        "    # }, index=rng.choice([*ascii_lowercase], size=(size, 4)).view('<U4').ravel()),\n",
        "\n",
        "    # _df.melt(\n",
        "    #     id_vars=['sector'],\n",
        "    #     value_vars=_df.columns.difference(['sector']),\n",
        "    #     ignore_index=False,\n",
        "    #     var_name='date'\n",
        "\n",
        "    # ).set_index('date', append=True).sort_index(),  # `melt`:take part of my columns except sector column and move it down into my data\n",
        "    # df.pivot,\n",
        "    #df.groupby([df.index.get_level_values('date').to_period('M'), 'ticker']).sum().unstack(),\n",
        "    # df.pivot_table(\n",
        "    #     index=df.index.get_level_values('date').to_period('M'),\n",
        "    #     columns='ticker',\n",
        "    #     aggfunc='sum',\n",
        "    # ),\n",
        "    # df.sort_index(),\n",
        "    # df.sort_index(axis='columns', ascending=False),\n",
        "    # df.sort_values('volume'),\n",
        "    # df.reset_index('ticker'), # `.reset_index()` : takes a layer of the index and pops it back into the data\n",
        "    # df.reset_index('ticker', drop=True),\n",
        "    # df.reset_index('ticker').set_index('ticker'),\n",
        "    # df.reset_index('ticker').set_index('ticker', append=True),\n",
        "    # df.droplevel('ticker'),\n",
        "    # df.reindex(date_range('2022-02-01', periods=4, freq='14D'), level='date'),\n",
        "    # df.set_axis(df.columns.str.title(), axis='columns'),\n",
        "    # df.rename(str.title, axis='columns'),\n",
        "    # df.rename(lambda dt: dt - timedelta(days=1), level='date'),\n",
        "    # df.reorder_levels(['ticker', 'date']),\n",
        "    # df.swaplevel(),\n",
        "    # df.rename_axis(['value'], axis='columns'),\n",
        "    # df.rename_axis(['value'], axis='columns').columns,\n",
        "    # df.stack().index,\n",
        "    # df.rename_axis(['value'], axis='columns').stack().index,\n",
        "    # df.swapaxes('index', 'columns'), # Same as Transpose `df.T`\n",
        "    # df.squeeze(),\n",
        "    # df[['price']].squeeze(),\n",
        "    # df.loc['2022-01-01'].squeeze(axis='index'),\n",
        "    # _s := Series([*rng.integers(10, size=(3, 3))]),\n",
        "    # _s.explode(),\n",
        "    # df.at[('2022-01-01', 'arnq'), 'price'], # same as : `df.loc[('2022-01-01', 'arnq'), 'price']`\n",
        "    # df.iat[0,0], # same as : `df.iloc[0,0]`,\n",
        "    # only diff-> `.at`and `.iat()` give scalar values\n",
        "    # _df1 := DataFrame({\n",
        "    #     'a' : rng.integers(-10, +10, size=(size := 3)),\n",
        "    #     'b' : rng.integers(-10, +10, size=size),\n",
        "    # }, index=date_range('2022-01-01', periods=size)),\n",
        "    # _df2 := DataFrame({\n",
        "    #     'b' : rng.integers(-10, +10, size=(size := 3)),\n",
        "    #     'c' : rng.integers(-10, +10, size=size),\n",
        "    # }, index=date_range('2022-01-01', periods=size)), \n",
        "    # *_df1.align(_df2),\n",
        "    # *_df1.align(_df2, axis='index'),\n",
        "    # _df1.join(_df2, lsuffix='1', rsuffix='2'),\n",
        "    # _df1.join(_df2, lsuffix='1', rsuffix='2',how='outer'),\n",
        "    #df.merge,\n",
        "    # df.abs(),\n",
        "    # df.clip(10, 100),\n",
        "    # df.min(),\n",
        "    # df.min(axis='index'),\n",
        "    # df.min(axis='columns'),\n",
        "    # df.max(),\n",
        "    # df.count(),\n",
        "    # df.nunique(),\n",
        "    # df.mad(),\n",
        "    # df.value_counts(),\n",
        "    # df.nsmallest(1, columns='price'),\n",
        "    # df.nlargest(3, columns=['volume']),\n",
        "    # df.loc[df.nsmallest(3, columns='price').index]['volume'], # what are the volume for three smallest values in price column\n",
        "    # df.idxmax(),\n",
        "    # df.idxmin(),\n",
        "    # df.expanding().max(),\n",
        "    # df.ewm(alpha=.1).mean(),\n",
        "    # df.shift(1),\n",
        "    # df.diff(1),\n",
        "    # df.rolling(3, min_periods=1).mean(),\n",
        "    # df.resample('M').mean(), # will not work on this dataset because its not a `daytimeIndex`\n",
        "    # df.resample('M', level='date').mean(), # this is how make it work for this `DataFrame` but it will `groupby ` resample index!\n",
        "    df.groupby('ticker').apply(\n",
        "        lambda s: s.resample('M',level='date').mean(),  # this preserve the outer index\n",
        "    ),\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "    sep='\\n{}\\n'.format('\\N{box drawings light horizontal}' * 60),\n",
        "\n",
        "\n",
        ")\n",
        "\n",
        "# Bottom line : Instead of Memorzing all this operations, better to understand rules of `indexing` and\n",
        "# then apply them to the operations we need to use! \n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "- `Stack and Unstack ` : <https://towardsdatascience.com/reshaping-a-dataframe-with-pandas-stack-and-unstack-925dc9ce1289>\n",
        "\n",
        "- `.unstack()`: https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.unstack.html\n",
        "- `.stack()` : https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.stack.html#pandas.DataFrame.stack\n",
        "- `.pivot()`: https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.pivot.html#pandas.DataFrame.pivot\n",
        "- `.melt()`: https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.melt.html?highlight=melt#pandas.DataFrame.melt\n",
        "- `.pivot_table()`: https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.pivot_table.html\n",
        "- `.sort_index()`:https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.sort_index.html\n",
        "- `.sort_values()` : https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.sort_values.html\n",
        "- `.reset_index()`: https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.reset_index.html\n",
        "- `.droplevel()`:https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.droplevel.html\n",
        "- `.reindex()`: https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.reindex.html\n",
        "- `,reorder_levels()`: https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.reorder_levels.html\n",
        "- `.swaplevel()`:https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.swaplevel.html\n",
        "- `.rename_axis()` : https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.swaplevel.html\n",
        "- `.swapaxes()`:https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.swapaxes.html\n",
        "- `.squeeze()`:https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.squeeze.html\n",
        "- `.explode()`:https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.explode.html\n",
        "- `.align()`: https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.align.html\n",
        "- `.join()`:https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.join.html\n",
        "- `.merge()`:https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.merge.html\n",
        "- `.mad()`:https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.mad.html\n",
        "- `.value_counts()`:https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.value_counts.html\n",
        "- `.nsmallest()`:https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.nsmallest.html\n",
        "- `.nlargest()`:https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.nlargest.html\n",
        "- `.idxmin()`:https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.idxmin.html\n",
        "- `.idxmax()` :https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.idxmax.html\n",
        "- `.expanding()`:https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.expanding.html\n",
        "- `.ewn()`:https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.ewm.html\n",
        "- `.rolling`:https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.rolling.html\n",
        "- `.shift()`:https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.shift.html\n",
        "- `.diff()`:https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.diff.html\n",
        "- `.resample()`:https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.resample.html\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### PersonalExperience: How a  comma(`,`) can mess up your life! "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 165,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "(            a   b  c\n",
            "date                \n",
            "2022-01-01  7 -10  2\n",
            "2022-01-02  2  -9  8\n",
            "2022-01-03  0 -10  0\n",
            "2022-01-04 -5  -7  2\n",
            "2022-01-05 -4   6  9,)\n",
            "────────────────────────────────────────────────────────────\n",
            "(            a   b   c\n",
            "date                 \n",
            "2022-01-02  4  -5   7\n",
            "2022-01-03  2   6   1\n",
            "2022-01-04  0   3 -10\n",
            "2022-01-05  1 -10   5\n",
            "2022-01-06  8  -3   4,)\n",
            "────────────────────────────────────────────────────────────\n",
            "(            a   b  c\n",
            "date                \n",
            "2022-01-01  7 -10  2\n",
            "2022-01-02  2  -9  8\n",
            "2022-01-03  0 -10  0\n",
            "2022-01-04 -5  -7  2\n",
            "2022-01-05 -4   6  9,             a   b   c\n",
            "date                 \n",
            "2022-01-02  4  -5   7\n",
            "2022-01-03  2   6   1\n",
            "2022-01-04  0   3 -10\n",
            "2022-01-05  1 -10   5\n",
            "2022-01-06  8  -3   4)\n"
          ]
        }
      ],
      "source": [
        "#Let's create two df's same as the ones that we created for our `DataFrame Index alignment` section!\n",
        "\n",
        "from pandas import DataFrame, Series, date_range\n",
        "from numpy.random import default_rng\n",
        "\n",
        "rng = default_rng(0)\n",
        "size = 5\n",
        "df1 = DataFrame(\n",
        "    {\n",
        "    'a' : rng.integers(-10, +10, size=size),\n",
        "    'b' : rng.integers(-10, +10, size=size),\n",
        "    'c' : rng.integers(-10, +10, size=size),\n",
        "    },\n",
        "    index=date_range('2022-01-01', periods=size, name='date')\n",
        "    ),                                                          # this comman will turn our df into a tuple!\n",
        "\n",
        "df2 = DataFrame(\n",
        "    {\n",
        "    'a' : rng.integers(-10, +10, size=size),\n",
        "    'b' : rng.integers(-10, +10, size=size),\n",
        "    'c' : rng.integers(-10, +10, size=size),\n",
        "    # 'd' : rng.integers(-10, +10, size=size),\n",
        "    }, \n",
        "    index=date_range('2022-01-02', periods=size, name='date')\n",
        "),                                                              # this comman will turn our df into a tuple!\n",
        "\n",
        "\n",
        "print(\n",
        "\n",
        "    df1,\n",
        "    df2,\n",
        "    df1+df2,\n",
        "    sep='\\n{}\\n'.format('\\N{box drawings light horizontal}' * 60),\n",
        "\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": []
    }
  ],
  "metadata": {
    "colab": {
      "name": "PandasExtras.ipynb",
      "provenance": [],
      "toc_visible": true
    },
    "kernelspec": {
      "display_name": "Python 3.10.5 ('jlab')",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.10.5"
    },
    "vscode": {
      "interpreter": {
        "hash": "fd2a9a9e662a38e634747b05a0c29058cb1b443461ff579bf134cbf5cee92403"
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}