41-preparo-atributos-genero-nome.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "id": "NWJ6b5VDKBhs"
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Carga dos dados de entrada"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# definir arquivo CSV de entrada\n",
    "#arquivo = 'dados/afastamentos-nomes-sexo.csv'\n",
    "arquivo = 'dados/cadastro-nomes-sexo.csv'\n",
    "#arquivo = 'extra/nome_sexo_pf.csv.bz2'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'dados/cadastro-nomes-sexo-preparado'"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "partes = arquivo.split('.')\n",
    "prefixo_arquivo = partes[0]\n",
    "#sufixo_arquivo = '.'.join(partes[1:])\n",
    "novo_arquivo = prefixo_arquivo + '-preparado'\n",
    "novo_arquivo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>SEXO</th>\n",
       "      <th>QTDE</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PNOME</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>MARIA</th>\n",
       "      <td>F</td>\n",
       "      <td>20389</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>JOSE</th>\n",
       "      <td>M</td>\n",
       "      <td>15925</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ANA</th>\n",
       "      <td>F</td>\n",
       "      <td>11497</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>JOAO</th>\n",
       "      <td>M</td>\n",
       "      <td>7515</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CARLOS</th>\n",
       "      <td>M</td>\n",
       "      <td>7370</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       SEXO   QTDE\n",
       "PNOME             \n",
       "MARIA     F  20389\n",
       "JOSE      M  15925\n",
       "ANA       F  11497\n",
       "JOAO      M   7515\n",
       "CARLOS    M   7370"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv(arquivo, index_col=0)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# remover valores nulos\n",
    "df = df.loc[~df.index.isna()]\n",
    "df.drop(df[df.index.isnull()].index, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "linhas nulas: 0\n"
     ]
    }
   ],
   "source": [
    "total = df[df.index.isnull()].shape[0]\n",
    "print(\"linhas nulas:\", total)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# renomear índice e coluna\n",
    "df.index.rename('NOME', inplace=True)\n",
    "df.rename(columns={\n",
    "    df.columns[0]: 'SEXO',\n",
    "    df.columns[1]: 'QTDE'\n",
    "}, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 390
    },
    "id": "5eczSwIWKBjn",
    "outputId": "7723b62d-9453-4717-af3e-8c64a253abce"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>SEXO</th>\n",
       "      <th>QTDE</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>NOME</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>MARIA</th>\n",
       "      <td>F</td>\n",
       "      <td>20389</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>JOSE</th>\n",
       "      <td>M</td>\n",
       "      <td>15925</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ANA</th>\n",
       "      <td>F</td>\n",
       "      <td>11497</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>JOAO</th>\n",
       "      <td>M</td>\n",
       "      <td>7515</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CARLOS</th>\n",
       "      <td>M</td>\n",
       "      <td>7370</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       SEXO   QTDE\n",
       "NOME              \n",
       "MARIA     F  20389\n",
       "JOSE      M  15925\n",
       "ANA       F  11497\n",
       "JOAO      M   7515\n",
       "CARLOS    M   7370"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Transformações nos dados"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "id": "zuAgZr2GKBj3"
   },
   "outputs": [],
   "source": [
    "df['SEXO'] = df['SEXO'].map({'F': 0, 'M': 1, 'X': 9}).astype('uint8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# remover sexo diferente de 0 ou 1\n",
    "df.drop(df[df['SEXO'] > 1].index, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# converter para tipo de inteiro menor\n",
    "df['QTDE'] = df['QTDE'].astype('uint32')\n",
    "#np.iinfo(np.dtype('uint32'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# criar reverso do nome\n",
    "df['REV'] = df.index\n",
    "df['REV'] = df['REV'].apply(lambda x: x[::-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>SEXO</th>\n",
       "      <th>QTDE</th>\n",
       "      <th>REV</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>NOME</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>MARIA</th>\n",
       "      <td>0</td>\n",
       "      <td>20389</td>\n",
       "      <td>AIRAM</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>JOSE</th>\n",
       "      <td>1</td>\n",
       "      <td>15925</td>\n",
       "      <td>ESOJ</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ANA</th>\n",
       "      <td>0</td>\n",
       "      <td>11497</td>\n",
       "      <td>ANA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>JOAO</th>\n",
       "      <td>1</td>\n",
       "      <td>7515</td>\n",
       "      <td>OAOJ</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CARLOS</th>\n",
       "      <td>1</td>\n",
       "      <td>7370</td>\n",
       "      <td>SOLRAC</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        SEXO   QTDE     REV\n",
       "NOME                       \n",
       "MARIA      0  20389   AIRAM\n",
       "JOSE       1  15925    ESOJ\n",
       "ANA        0  11497     ANA\n",
       "JOAO       1   7515    OAOJ\n",
       "CARLOS     1   7370  SOLRAC"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "df['LETRAS'] = 0\n",
    "df['LETRAS'] = df['REV'].apply(lambda x: len(x))\n",
    "maior = df['LETRAS'].max()\n",
    "maior"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "MAIOR_QTDE_LETRAS = 16 # fixado para o maior arquivo\n",
    "\n",
    "for i in range(MAIOR_QTDE_LETRAS):\n",
    "    df['L' + str(i)] = df['REV'].apply(\n",
    "        lambda x: ord(x[i]) - 64 if len(x) > i else 0\n",
    "    ).astype('uint8')"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {
    "id": "UyDGsCtvKBj8"
   },
   "source": [
    "for i in range(MAIOR_QTDE_LETRAS):\n",
    "    col = 'L' + str(i)\n",
    "    #df[col] = df['REV'].apply(lambda x: x[i] if len(x) > i else ' ')\n",
    "    #df.loc[df[col].isnull(), col] = ' '\n",
    "    df[col] = df[col].apply(lambda x: ord(x) - 64 if x != ' ' else 0).astype('uint8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# remover coluna reversa\n",
    "df.drop(['REV'], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "VL2wi7EgKBkC",
    "outputId": "f74b5db2-7650-4ace-dba8-77bd81166a1f"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SEXO     uint8\n",
       "QTDE    uint32\n",
       "L0       uint8\n",
       "L1       uint8\n",
       "L2       uint8\n",
       "L3       uint8\n",
       "L4       uint8\n",
       "L5       uint8\n",
       "L6       uint8\n",
       "L7       uint8\n",
       "L8       uint8\n",
       "L9       uint8\n",
       "L10      uint8\n",
       "L11      uint8\n",
       "L12      uint8\n",
       "L13      uint8\n",
       "L14      uint8\n",
       "L15      uint8\n",
       "dtype: object"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.dtypes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Análise dos resultados"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Amostra dos dados"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 390
    },
    "id": "H_wvkeoJKBkH",
    "outputId": "6387995b-a1ed-4da7-af3b-f85435592ab7"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>SEXO</th>\n",
       "      <th>QTDE</th>\n",
       "      <th>L0</th>\n",
       "      <th>L1</th>\n",
       "      <th>L2</th>\n",
       "      <th>L3</th>\n",
       "      <th>L4</th>\n",
       "      <th>L5</th>\n",
       "      <th>L6</th>\n",
       "      <th>L7</th>\n",
       "      <th>L8</th>\n",
       "      <th>L9</th>\n",
       "      <th>L10</th>\n",
       "      <th>L11</th>\n",
       "      <th>L12</th>\n",
       "      <th>L13</th>\n",
       "      <th>L14</th>\n",
       "      <th>L15</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>NOME</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>MARIA</th>\n",
       "      <td>0</td>\n",
       "      <td>20389</td>\n",
       "      <td>1</td>\n",
       "      <td>9</td>\n",
       "      <td>18</td>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>JOSE</th>\n",
       "      <td>1</td>\n",
       "      <td>15925</td>\n",
       "      <td>5</td>\n",
       "      <td>19</td>\n",
       "      <td>15</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ANA</th>\n",
       "      <td>0</td>\n",
       "      <td>11497</td>\n",
       "      <td>1</td>\n",
       "      <td>14</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>JOAO</th>\n",
       "      <td>1</td>\n",
       "      <td>7515</td>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>15</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CARLOS</th>\n",
       "      <td>1</td>\n",
       "      <td>7370</td>\n",
       "      <td>19</td>\n",
       "      <td>15</td>\n",
       "      <td>12</td>\n",
       "      <td>18</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PAULO</th>\n",
       "      <td>1</td>\n",
       "      <td>7218</td>\n",
       "      <td>15</td>\n",
       "      <td>12</td>\n",
       "      <td>21</td>\n",
       "      <td>1</td>\n",
       "      <td>16</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ANTONIO</th>\n",
       "      <td>1</td>\n",
       "      <td>6982</td>\n",
       "      <td>15</td>\n",
       "      <td>9</td>\n",
       "      <td>14</td>\n",
       "      <td>15</td>\n",
       "      <td>20</td>\n",
       "      <td>14</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LUIZ</th>\n",
       "      <td>1</td>\n",
       "      <td>6139</td>\n",
       "      <td>26</td>\n",
       "      <td>9</td>\n",
       "      <td>21</td>\n",
       "      <td>12</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>FRANCISCO</th>\n",
       "      <td>1</td>\n",
       "      <td>5795</td>\n",
       "      <td>15</td>\n",
       "      <td>3</td>\n",
       "      <td>19</td>\n",
       "      <td>9</td>\n",
       "      <td>3</td>\n",
       "      <td>14</td>\n",
       "      <td>1</td>\n",
       "      <td>18</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MARCELO</th>\n",
       "      <td>1</td>\n",
       "      <td>4979</td>\n",
       "      <td>15</td>\n",
       "      <td>12</td>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "      <td>18</td>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           SEXO   QTDE  L0  L1  L2  L3  L4  L5  L6  L7  L8  L9  L10  L11  L12  \\\n",
       "NOME                                                                            \n",
       "MARIA         0  20389   1   9  18   1  13   0   0   0   0   0    0    0    0   \n",
       "JOSE          1  15925   5  19  15  10   0   0   0   0   0   0    0    0    0   \n",
       "ANA           0  11497   1  14   1   0   0   0   0   0   0   0    0    0    0   \n",
       "JOAO          1   7515  15   1  15  10   0   0   0   0   0   0    0    0    0   \n",
       "CARLOS        1   7370  19  15  12  18   1   3   0   0   0   0    0    0    0   \n",
       "PAULO         1   7218  15  12  21   1  16   0   0   0   0   0    0    0    0   \n",
       "ANTONIO       1   6982  15   9  14  15  20  14   1   0   0   0    0    0    0   \n",
       "LUIZ          1   6139  26   9  21  12   0   0   0   0   0   0    0    0    0   \n",
       "FRANCISCO     1   5795  15   3  19   9   3  14   1  18   6   0    0    0    0   \n",
       "MARCELO       1   4979  15  12   5   3  18   1  13   0   0   0    0    0    0   \n",
       "\n",
       "           L13  L14  L15  \n",
       "NOME                      \n",
       "MARIA        0    0    0  \n",
       "JOSE         0    0    0  \n",
       "ANA          0    0    0  \n",
       "JOAO         0    0    0  \n",
       "CARLOS       0    0    0  \n",
       "PAULO        0    0    0  \n",
       "ANTONIO      0    0    0  \n",
       "LUIZ         0    0    0  \n",
       "FRANCISCO    0    0    0  \n",
       "MARCELO      0    0    0  "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "id": "wV1JflwpKBkL"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total de linhas: 41146\n"
     ]
    }
   ],
   "source": [
    "print(\"total de linhas:\", df.shape[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Quantidades por sexo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CONT</th>\n",
       "      <th>PERC</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SEXO</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>21902</td>\n",
       "      <td>53.22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>19244</td>\n",
       "      <td>46.77</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       CONT   PERC\n",
       "SEXO              \n",
       "0     21902  53.22\n",
       "1     19244  46.77"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df2 = df[['SEXO', 'L0']].groupby(by=['SEXO']).count().rename({'L0': 'CONT'}, axis=1)\n",
    "total = df2['CONT'].sum()\n",
    "df2['PERC'] = df2['CONT'].apply(lambda x: int(x / total * 10000) / 100)\n",
    "df2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Gravação do arquivo final"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv(novo_arquivo + '.csv.bz2', compression='bz2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_pickle(novo_arquivo + '.pkl.bz2')"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "61-modelo-genero-nome.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}