01_AutomatizeFunctions.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyPn/ebDDdTXH3b5HG0rDvpZ",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/luasampaio/data-engineering/blob/main/1_AutomatizeFunctions.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Removendo dados duplicados"
      ],
      "metadata": {
        "id": "KShD5kf3d43A"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "- Luciana Sampaio\n",
        "- 09.01.25\n"
      ],
      "metadata": {
        "id": "lKrKFzt5sIhp"
      }
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "c-39lNc_sH3F"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd"
      ],
      "metadata": {
        "id": "Dw2gAy-Ud5aJ"
      },
      "execution_count": 24,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": 25,
      "metadata": {
        "id": "5503XBDWclwX"
      },
      "outputs": [],
      "source": [
        "def remove_duplicates(df, subset=None):\n",
        "    return df.drop_duplicates(subset=subset)"
      ]
    },
    {
      "source": [
        "# Example dataset with duplicates, ensuring consistent lengths\n",
        "data = {'Name': ['Alice', 'Bob', 'Alice', 'David','Luciana','Angelica'],\n",
        "        'Age': [25, 30, 25, 22,23,32],\n",
        "        'Salary': [50000, 60000, 50000, 45000, 45000, 45000]}\n",
        "df = pd.DataFrame(data)"
      ],
      "cell_type": "code",
      "metadata": {
        "id": "RaM2jcmweBjz"
      },
      "execution_count": 26,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5jY1PkPDeFvH",
        "outputId": "ec2af0f1-135d-46ef-d9a8-02cb1abbfd02"
      },
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'Name': ['Alice', 'Bob', 'Alice', 'David', 'Luciana', 'Angelica'],\n",
              " 'Age': [25, 30, 25, 22, 23, 32],\n",
              " 'Salary': [50000, 60000, 50000, 45000, 45000, 45000]}"
            ]
          },
          "metadata": {},
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "cleaned_df = remove_duplicates(df, subset=['Name'])\n",
        "print(cleaned_df)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SrxJRwnCeLyH",
        "outputId": "807710fa-a724-4260-e83a-0fd33194287e"
      },
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "       Name  Age  Salary\n",
            "0     Alice   25   50000\n",
            "1       Bob   30   60000\n",
            "3     David   22   45000\n",
            "4   Luciana   23   45000\n",
            "5  Angelica   32   45000\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Automating Data Type Transformation"
      ],
      "metadata": {
        "id": "mEAH52Eah_nX"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def transform_data_types(df, col_types):\n",
        "    for col, dtype in col_types.items():\n",
        "        df[col] = df[col].astype(dtype)\n",
        "    return df"
      ],
      "metadata": {
        "id": "H1QsExI_elHv"
      },
      "execution_count": 29,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(df.dtypes)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wKYvu-dhiVUH",
        "outputId": "cfcede53-c075-4737-f719-a90dc025e121"
      },
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Name      object\n",
            "Age        int64\n",
            "Salary     int64\n",
            "dtype: object\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Definição de tipo de dados\n"
      ],
      "metadata": {
        "id": "_6AnOboMix9q"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "col_types = {'Age': 'int', 'Salary': 'float'}"
      ],
      "metadata": {
        "id": "oCLJxO69iZUE"
      },
      "execution_count": 31,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "cleaned_df = transform_data_types(df, col_types)\n"
      ],
      "metadata": {
        "id": "5LNSmtHsigHu"
      },
      "execution_count": 32,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(cleaned_df.dtypes)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KGq9ykAqioo6",
        "outputId": "4b6de9c4-e95d-41c6-fbe8-69e8edc63a25"
      },
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Name       object\n",
            "Age         int64\n",
            "Salary    float64\n",
            "dtype: object\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Observação:\n",
        "- Adicione uma etapa no pipeline para lidar com valores ausentes antes de converter os tipos"
      ],
      "metadata": {
        "id": "z8ksGZ-yjSHJ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "\n",
        "# Função para lidar com valores ausentes\n",
        "def handle_missing_values(df, method='mean', fill_value=None):\n",
        "    if method == 'mean':\n",
        "        for col in df.select_dtypes(include=[np.number]).columns:\n",
        "            df.loc[:, col] = df[col].fillna(df[col].mean())\n",
        "    elif method == 'median':\n",
        "        for col in df.select_dtypes(include=[np.number]).columns:\n",
        "            df.loc[:, col] = df[col].fillna(df[col].median())\n",
        "    elif method == 'mode':\n",
        "        for col in df.columns:\n",
        "            df.loc[:, col] = df[col].fillna(df[col].mode()[0])\n",
        "    elif method == 'constant':\n",
        "        df = df.fillna(fill_value)\n",
        "    elif method == 'drop':\n",
        "        df = df.dropna()\n",
        "    else:\n",
        "        raise ValueError(\"Método inválido para lidar com valores ausentes.\")\n",
        "    return df\n",
        "\n",
        "# Função para remover duplicatas\n",
        "def remove_duplicates(df, subset=None):\n",
        "    return df.drop_duplicates(subset=subset)\n",
        "\n",
        "# Função para transformar tipos de dados\n",
        "def transform_data_types(df, col_types):\n",
        "    for col, dtype in col_types.items():\n",
        "        # Preencher valores ausentes com um valor padrão antes da conversão\n",
        "        if dtype == 'int':\n",
        "            df.loc[:, col] = df[col].fillna(0).astype(dtype)\n",
        "        elif dtype == 'float':\n",
        "            df.loc[:, col] = df[col].fillna(0.0).astype(dtype)\n",
        "        else:\n",
        "            df.loc[:, col] = df[col].astype(dtype)\n",
        "    return df\n",
        "\n",
        "# Pipeline de limpeza de dados\n",
        "def data_cleaning_pipeline(df, missing_values_method='mean', fill_value=None, subset=None, col_types=None):\n",
        "    # Lidar com valores ausentes\n",
        "    df = handle_missing_values(df, method=missing_values_method, fill_value=fill_value)\n",
        "\n",
        "    # Remover duplicatas\n",
        "    df = remove_duplicates(df, subset=subset)\n",
        "\n",
        "    # Transformar tipos de dados\n",
        "    if col_types:\n",
        "        df = transform_data_types(df, col_types)\n",
        "\n",
        "    return df\n",
        "\n",
        "# Exemplo de conjunto de dados com vários problemas\n",
        "data = {\n",
        "    'Nome': ['Alice', 'Bob', None, 'Alice'],\n",
        "    'Idade': ['25', None, '30', '22'],\n",
        "    'Salário': [50000, 60000, None, 50000]\n",
        "}\n",
        "\n",
        "df = pd.DataFrame(data)\n",
        "\n",
        "# Defina os tipos de dados e execute o pipeline\n",
        "col_types = {'Idade': 'int', 'Salário': 'float'}\n",
        "cleaned_df = data_cleaning_pipeline(df, missing_values_method='mean', subset=['Nome'], col_types=col_types)\n",
        "\n",
        "print(cleaned_df)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mTNxGuUWliaz",
        "outputId": "ab3c0d44-4c7a-4e71-85dc-dbed21014b41"
      },
      "execution_count": 34,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "    Nome Idade       Salário\n",
            "0  Alice    25  50000.000000\n",
            "1    Bob     0  60000.000000\n",
            "2   None    30  53333.333333\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "SWyQTfPFmDoM"
      },
      "execution_count": 34,
      "outputs": []
    }
  ]
}