1_2_censobrDataExtraction.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "vscode": {
     "languageId": "plaintext"
    }
   },
   "source": [
    "### Preprocessing census data\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Change this to the path of your CensoBR cache directory\n",
    "CENSOBR_CACHE_DIR = (\n",
    "    \"/Users/claudio/Library/Caches/org.R-project.R/R/censobr/data_release_v0.3.0/\"\n",
    ")\n",
    "tracts_basico = pd.read_parquet(CENSOBR_CACHE_DIR + \"2010_tracts_Basico_v0.3.0.parquet\")\n",
    "tracts_domicilio_renda = pd.read_parquet(\n",
    "    CENSOBR_CACHE_DIR + \"2010_tracts_DomicilioRenda_v0.3.0.parquet\"\n",
    ")\n",
    "tracts_pessoa = pd.read_parquet(CENSOBR_CACHE_DIR + \"2010_tracts_Pessoa_v0.3.0.parquet\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "tract_basico = tracts_basico[[\"code_tract\", \"V002\"]]  # Population\n",
    "tract_income = tracts_domicilio_renda[[\"code_tract\", \"V003\"]]  # Income (Reais)\n",
    "tract_idade = tracts_pessoa[\n",
    "    [\n",
    "        \"code_tract\",\n",
    "        \"pessoa13_V021\",\n",
    "        \"pessoa13_V022\",\n",
    "        \"pessoa13_V023\",\n",
    "        \"pessoa13_V024\",\n",
    "        \"pessoa13_V025\",\n",
    "        \"pessoa13_V026\",\n",
    "        \"pessoa13_V027\",\n",
    "        \"pessoa13_V028\",\n",
    "        \"pessoa13_V029\",\n",
    "        \"pessoa13_V030\",\n",
    "        \"pessoa13_V031\",\n",
    "        \"pessoa13_V032\",\n",
    "        \"pessoa13_V033\",\n",
    "        \"pessoa13_V034\",\n",
    "        \"pessoa13_V035\",\n",
    "        \"pessoa13_V036\",\n",
    "        \"pessoa13_V037\",\n",
    "        \"pessoa13_V038\",\n",
    "        \"pessoa13_V039\",\n",
    "        \"pessoa13_V040\",\n",
    "        \"pessoa13_V041\",\n",
    "        \"pessoa13_V042\",\n",
    "        \"pessoa13_V043\",\n",
    "        \"pessoa13_V044\",\n",
    "        \"pessoa13_V045\",\n",
    "        \"pessoa13_V046\",\n",
    "        \"pessoa13_V047\",\n",
    "        \"pessoa13_V048\",\n",
    "        \"pessoa13_V049\",\n",
    "        \"pessoa13_V050\",\n",
    "        \"pessoa13_V051\",\n",
    "        \"pessoa13_V052\",\n",
    "    ]\n",
    "]  # Population by age"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "tracts_df_temp = pd.merge(tract_basico, tract_income, on=\"code_tract\", how=\"left\")\n",
    "tracts_df = pd.merge(tracts_df_temp, tract_idade, on=\"code_tract\", how=\"left\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "vscode": {
     "languageId": "javascript"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "        code_tract    V002      V003  pessoa13_V021  pessoa13_V022  \\\n",
      "0  120001305000001   957.0  601805.0            0.0            8.0   \n",
      "1  120001305000002  1203.0  385033.0            0.0           29.0   \n",
      "2  120001305000003  1700.0  531794.0            0.0           41.0   \n",
      "3  120001305000004   182.0   58853.0            0.0            2.0   \n",
      "4  120001305000005   305.0  112979.0            0.0            5.0   \n",
      "\n",
      "   pessoa13_V023  pessoa13_V024  pessoa13_V025  pessoa13_V026  pessoa13_V027  \\\n",
      "0            0.0            0.0            0.0            1.0            0.0   \n",
      "1            1.0            2.0            3.0            4.0            0.0   \n",
      "2            4.0            2.0            4.0            3.0            1.0   \n",
      "3            0.0            0.0            0.0            0.0            0.0   \n",
      "4            1.0            1.0            0.0            0.0            0.0   \n",
      "\n",
      "   ...  pessoa13_V044  pessoa13_V045  pessoa13_V046  pessoa13_V047  \\\n",
      "0  ...           16.0           22.0           20.0           26.0   \n",
      "1  ...           29.0           41.0           23.0           36.0   \n",
      "2  ...           47.0           39.0           31.0           35.0   \n",
      "3  ...            4.0            6.0            4.0            4.0   \n",
      "4  ...            5.0            6.0            4.0            7.0   \n",
      "\n",
      "   pessoa13_V048  pessoa13_V049  pessoa13_V050  pessoa13_V051  pessoa13_V052  \\\n",
      "0           17.0           23.0           22.0           14.0           17.0   \n",
      "1           31.0           36.0           35.0           27.0           23.0   \n",
      "2           27.0           51.0           21.0           37.0           34.0   \n",
      "3            3.0            2.0            2.0            4.0            7.0   \n",
      "4            3.0            5.0            5.0            9.0            9.0   \n",
      "\n",
      "    income_pc  \n",
      "0  628.845350  \n",
      "1  320.060682  \n",
      "2  312.820000  \n",
      "3  323.368132  \n",
      "4  370.422951  \n",
      "\n",
      "[5 rows x 36 columns]\n"
     ]
    }
   ],
   "source": [
    "tracts_df = tracts_df.assign(income_pc=tracts_df['V003'] / tracts_df['V002'])\n",
    "print(tracts_df.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "vscode": {
     "languageId": "javascript"
    }
   },
   "outputs": [],
   "source": [
    "tracts_df.to_csv(\"data/censobr_tract_income.csv\", index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "tracts_df.to_parquet(\"data/censobr_tract_income.parquet\", index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".env",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}