3.4-Monitorar_Dados_TFDV.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "l4X1m3rey3E4"
      },
      "source": [
        "Tensor Flow Data Validation: https://www.tensorflow.org/tfx/tutorials/data_validation/tfdv_basic\n",
        "\n",
        "Tensor Flow Extension: https://www.tensorflow.org/tfx\n",
        "\n",
        "Census Income Data Set: http://archive.ics.uci.edu/ml/datasets/Census+Income"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "0haIuG--ysuy"
      },
      "outputs": [],
      "source": [
        "#!pip install tensorflow_data_validation"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "si0uIUQYhhON"
      },
      "outputs": [],
      "source": [
        "import tensorflow as tf\n",
        "import tensorflow_data_validation as tfdv\n",
        "import pandas as pd"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "GHi-tkOeBOis"
      },
      "outputs": [],
      "source": [
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "from tensorflow_metadata.proto.v0 import schema_pb2\n",
        "\n",
        "print('TFDV Version: {}'.format(tfdv.__version__))\n",
        "print('Tensorflow Version: {}'.format(tf.__version__))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "WKTfuT2rga-_"
      },
      "outputs": [],
      "source": [
        "df = pd.read_csv('http://archive.ics.uci.edu/ml/machine-learning-databases/adult/adult.data', skipinitialspace=True, header=None)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "WTjkLl_Y2ghi"
      },
      "outputs": [],
      "source": [
        "train_df, eval_df = train_test_split(df, test_size=0.2, shuffle=False)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "uxDFD6dR0PYH"
      },
      "outputs": [],
      "source": [
        "train_df.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "yyrmQLCm0a5V"
      },
      "outputs": [],
      "source": [
        "eval_df.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "4KrY55Vk2osB"
      },
      "outputs": [],
      "source": [
        "eval_df.info()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "90r28C8hqaXD"
      },
      "outputs": [],
      "source": [
        "novos_dados = [[46,'NaN',257473,'Bachelors',8,'Married-civ-spouse','Plumber','Husband','Other','Male',1000,0,41,'Australia','>50K'],\n",
        "               [0,'Private',257473,'Masters',8,'Married-civ-spouse','Adm-clerical','Wife','Asian','Female',0,0,40,'Pakistan','>50K'],\n",
        "               [1000,'Private',257473,'Masters',8,'Married-civ-spouse','Prof-specialty','Husband','Black','Male',0,0,20,'Cameroon','<=50K'],\n",
        "               [25,'?',257473,'Masters',8,'Married-civ-spouse','gamer','Husband','Asian','Female',0,0,50,'Mongolia','<=50K']]\n",
        "\n",
        "eval_df2 = eval_df.append(novos_dados, ignore_index=True)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "eyc7z41l3Q6p"
      },
      "outputs": [],
      "source": [
        "eval_df2.info()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "nkHyCaLo2y3x"
      },
      "outputs": [],
      "source": [
        "eval_df2.tail(5)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Duwwrsvf_9bK"
      },
      "source": [
        "## Geração e visualização do dataset de treino"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "_vTx9Qkk4yGc"
      },
      "outputs": [],
      "source": [
        "estatistica_treino = tfdv.generate_statistics_from_dataframe(train_df)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "1D1wP3mm5ebW"
      },
      "outputs": [],
      "source": [
        "tfdv.visualize_statistics(estatistica_treino)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "W9skjM-M44Jz"
      },
      "outputs": [],
      "source": [
        "schema = tfdv.infer_schema(statistics=estatistica_treino)\n",
        "\n",
        "tfdv.display_schema(schema)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5oj_GIprgPU1"
      },
      "source": [
        "## Geração e visualização do dataset de score"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "bzZy1x3c6Mi0",
        "scrolled": true
      },
      "outputs": [],
      "source": [
        "estatistica_score = tfdv.generate_statistics_from_dataframe(eval_df2)\n",
        "\n",
        "tfdv.visualize_statistics(\n",
        "    lhs_statistics=estatistica_score,\n",
        "    rhs_statistics=estatistica_treino,\n",
        "    lhs_name='SCORE',\n",
        "    rhs_name='TREINO'\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "QvXtFZSTqaXI"
      },
      "outputs": [],
      "source": [
        "eval_df2 = eval_df2[eval_df2[0] > 16]\n",
        "eval_df2 = eval_df2[eval_df2[0] < 91]\n",
        "\n",
        "eval_df2.dropna(inplace=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "zYomYtzkqaXI"
      },
      "outputs": [],
      "source": [
        "estatistica_score2 = tfdv.generate_statistics_from_dataframe(eval_df2)\n",
        "\n",
        "tfdv.visualize_statistics(\n",
        "    lhs_statistics=estatistica_score2,\n",
        "    rhs_statistics=estatistica_treino,\n",
        "    lhs_name='SCORE',\n",
        "    rhs_name='TREINO'\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "J3Lvnr-YgPU2"
      },
      "source": [
        "## Encontrar anomalias"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "OR5dBqpW6ky2"
      },
      "outputs": [],
      "source": [
        "anomalia =  tfdv.validate_statistics(statistics=estatistica_score2, schema=schema)\n",
        "\n",
        "tfdv.display_anomalies(anomalia)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "cFsxWJerqaXK"
      },
      "outputs": [],
      "source": [
        "country_feature = tfdv.get_feature(schema, '13')\n",
        "country_feature.distribution_constraints.min_domain_mass = 0.9\n",
        "\n",
        "occupation_feature = tfdv.get_feature(schema, '6')\n",
        "occupation_feature.distribution_constraints.min_domain_mass = 0.9"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "hTCWS04p6lDh"
      },
      "outputs": [],
      "source": [
        "race_domain = tfdv.get_domain(schema, '8')\n",
        "race_domain.value.append('Asian')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "5iGxnqlsqaXL"
      },
      "outputs": [],
      "source": [
        "tfdv.set_domain(schema, '0', schema_pb2.IntDomain(name='0', min=17, max=90))\n",
        "tfdv.display_schema(schema)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "CZDDNQ1iqaXM"
      },
      "outputs": [],
      "source": [
        "anomalias_atualizadas = tfdv.validate_statistics(estatistica_score2, schema)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Pve-LlZGqaXN"
      },
      "outputs": [],
      "source": [
        "tfdv.display_anomalies(anomalias_atualizadas)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "lv4q6w3prf_1"
      },
      "outputs": [],
      "source": [
        "workclass_domain = tfdv.get_domain(schema, '1')\n",
        "workclass_domain.value.append('NaN')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "6qakYgNPr2TL"
      },
      "outputs": [],
      "source": [
        "anomalias_atualizadas = tfdv.validate_statistics(estatistica_score2, schema)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "V2Ho0kaUr2aB"
      },
      "outputs": [],
      "source": [
        "tfdv.display_anomalies(anomalias_atualizadas)"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.8"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}