34_Em_Desenvolvimento.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyPYT6z0KBJIJssy002iUQkg",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/luasampaio/data-engineering/blob/main/34_Em_Desenvolvimento.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [],
      "metadata": {
        "id": "csbu_aDhC6U_"
      }
    },
    {
      "source": [
        "\n",
        "from pyspark.sql import SparkSession\n",
        "import requests\n",
        "import pandas as pd\n",
        "\n",
        "# Create a SparkSession\n",
        "spark = SparkSession.builder.appName(\"LucianaSampaioApp\").getOrCreate()\n",
        "\n",
        "# URL para o conteúdo bruto do arquivo CSV\n",
        "url = 'https://raw.githubusercontent.com/luasampaio/datasets/main/products.csv'"
      ],
      "cell_type": "code",
      "metadata": {
        "id": "ZoDu_bCAN74w"
      },
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Lê o arquivo CSV diretamente do URL using pandas\n",
        "pandas_df = pd.read_csv(url)"
      ],
      "metadata": {
        "id": "Wg1fVwPHOC9r"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Create a Spark DataFrame from the pandas DataFrame\n",
        "df = spark.createDataFrame(pandas_df)"
      ],
      "metadata": {
        "id": "dQQ-47ZEQB9d"
      },
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "display(df)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "id": "TXMl6tIAQDcT",
        "outputId": "691e19ec-7280-4d0a-bed1-44f8b6a99d13"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "DataFrame[product_id: bigint, product_name: string, brand_id: bigint, category_id: bigint, model_year: bigint, list_price: double]"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Definir as colunas que o DataFrame precisa ter\n",
        "required_columns = {\"product_id\", \"product_name\"}\n",
        "\n",
        "# Comparar com as colunas reais do DataFrame\n",
        "missing_columns = required_columns - set(df.columns)\n",
        "\n",
        "# Se houver colunas ausentes, gerar um erro\n",
        "if missing_columns:\n",
        "    raise ValueError(f\"❌ Erro: Colunas ausentes no DataFrame: {missing_columns}\")\n"
      ],
      "metadata": {
        "id": "Fu-sZRwbdVzt"
      },
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df.show(7)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0Z2PJfeVQFjo",
        "outputId": "544612ce-98d6-48c8-f31d-26c89b621cb0"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+----------+--------------------+--------+-----------+----------+----------+\n",
            "|product_id|        product_name|brand_id|category_id|model_year|list_price|\n",
            "+----------+--------------------+--------+-----------+----------+----------+\n",
            "|         1|     Trek 820 - 2016|       9|          6|      2016|    379.99|\n",
            "|         2|Ritchey Timberwol...|       5|          6|      2016|    749.99|\n",
            "|         3|Surly Wednesday F...|       8|          6|      2016|    999.99|\n",
            "|         4|Trek Fuel EX 8 29...|       9|          6|      2016|   2899.99|\n",
            "|         5|Heller Shagamaw F...|       3|          6|      2016|   1320.99|\n",
            "|         6|Surly Ice Cream T...|       8|          6|      2016|    469.99|\n",
            "|         7|Trek Slash 8 27.5...|       9|          6|      2016|   3999.99|\n",
            "+----------+--------------------+--------+-----------+----------+----------+\n",
            "only showing top 7 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df.printSchema()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZSifwE2uQJkF",
        "outputId": "df67e51d-2cbb-4fff-ed15-6089e55b071b"
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "root\n",
            " |-- product_id: long (nullable = true)\n",
            " |-- product_name: string (nullable = true)\n",
            " |-- brand_id: long (nullable = true)\n",
            " |-- category_id: long (nullable = true)\n",
            " |-- model_year: long (nullable = true)\n",
            " |-- list_price: double (nullable = true)\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Adicionando a data de ingestão\n"
      ],
      "metadata": {
        "id": "K8cBti-0aAtv"
      }
    },
    {
      "source": [
        "from pyspark.sql.functions import to_timestamp, current_timestamp, lit, concat_ws, col\n",
        "\n",
        "\n",
        "# Adicionar colunas extras\n",
        "df = df.withColumn(\"ingestion_date\", current_timestamp())"
      ],
      "cell_type": "code",
      "metadata": {
        "id": "FqL330PVZOva"
      },
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df.show(7)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "z8ZsaGDZZSba",
        "outputId": "09589e69-b605-4d83-b220-869c49bbc944"
      },
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+----------+--------------------+--------+-----------+----------+----------+--------------------+\n",
            "|product_id|        product_name|brand_id|category_id|model_year|list_price|      ingestion_date|\n",
            "+----------+--------------------+--------+-----------+----------+----------+--------------------+\n",
            "|         1|     Trek 820 - 2016|       9|          6|      2016|    379.99|2025-02-04 14:59:...|\n",
            "|         2|Ritchey Timberwol...|       5|          6|      2016|    749.99|2025-02-04 14:59:...|\n",
            "|         3|Surly Wednesday F...|       8|          6|      2016|    999.99|2025-02-04 14:59:...|\n",
            "|         4|Trek Fuel EX 8 29...|       9|          6|      2016|   2899.99|2025-02-04 14:59:...|\n",
            "|         5|Heller Shagamaw F...|       3|          6|      2016|   1320.99|2025-02-04 14:59:...|\n",
            "|         6|Surly Ice Cream T...|       8|          6|      2016|    469.99|2025-02-04 14:59:...|\n",
            "|         7|Trek Slash 8 27.5...|       9|          6|      2016|   3999.99|2025-02-04 14:59:...|\n",
            "+----------+--------------------+--------+-----------+----------+----------+--------------------+\n",
            "only showing top 7 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Uma segunda opção usando o elemento col (Functions)"
      ],
      "metadata": {
        "id": "Kvwx7gMsg5J7"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "df = df.withColumn(\"ingestion_date\", to_timestamp(col(\"ingestion_date\")))"
      ],
      "metadata": {
        "id": "uskimafpde3B"
      },
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df.show(7)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RHburIXNgz4V",
        "outputId": "ed51b755-d2de-4cea-b217-2f9509112e28"
      },
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+----------+--------------------+--------+-----------+----------+----------+--------------------+\n",
            "|product_id|        product_name|brand_id|category_id|model_year|list_price|      ingestion_date|\n",
            "+----------+--------------------+--------+-----------+----------+----------+--------------------+\n",
            "|         1|     Trek 820 - 2016|       9|          6|      2016|    379.99|2025-02-04 15:10:...|\n",
            "|         2|Ritchey Timberwol...|       5|          6|      2016|    749.99|2025-02-04 15:10:...|\n",
            "|         3|Surly Wednesday F...|       8|          6|      2016|    999.99|2025-02-04 15:10:...|\n",
            "|         4|Trek Fuel EX 8 29...|       9|          6|      2016|   2899.99|2025-02-04 15:10:...|\n",
            "|         5|Heller Shagamaw F...|       3|          6|      2016|   1320.99|2025-02-04 15:10:...|\n",
            "|         6|Surly Ice Cream T...|       8|          6|      2016|    469.99|2025-02-04 15:10:...|\n",
            "|         7|Trek Slash 8 27.5...|       9|          6|      2016|   3999.99|2025-02-04 15:10:...|\n",
            "+----------+--------------------+--------+-----------+----------+----------+--------------------+\n",
            "only showing top 7 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "h5XPNVyzhDwK"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [],
      "metadata": {
        "id": "cSeHg21sC4cd"
      }
    }
  ]
}