20_ntb_pySpark_tratativa_dados.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyOF6pjbYCN6EoOKrs+Te7YA",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/luasampaio/data-engineering/blob/main/20_ntb_pySpark_tratativa_dados.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Loading data"
      ],
      "metadata": {
        "id": "utRUSFb7ksHa"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "from pyspark.sql import SparkSession\n",
        "import pyspark .sql.functions as F\n",
        "import os"
      ],
      "metadata": {
        "id": "WadEjunqx3lR"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "source": [
        "\n",
        "\n",
        "\n",
        "spark = (\n",
        "    SparkSession.builder.appName('PySpark - LucianaSampaio')\n",
        "    .config('spark.sql.repl.eagerEval.enabled', True)\n",
        "    .getOrCreate()\n",
        ")\n",
        "\n"
      ],
      "cell_type": "code",
      "metadata": {
        "id": "zmwU9zVZt8ZT"
      },
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive')\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "F_ymNzpQyA_-",
        "outputId": "232df778-bf4b-4172-ff8e-0b8207daafba"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "df = spark.read.csv('/content/drive/MyDrive/.Dataset/test.csv', sep=',', encoding='UTF-8', header=True, inferSchema=True)\n",
        "df\n",
        ""
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 529
        },
        "id": "Wo9YG4tHuQKB",
        "outputId": "eab35c88-2871-498f-ee51-6da3564bfd9d"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "+-----------+------+--------------------+------+----+-----+-----+----------------+-------+-----+--------+\n",
              "|PassengerId|Pclass|                Name|   Sex| Age|SibSp|Parch|          Ticket|   Fare|Cabin|Embarked|\n",
              "+-----------+------+--------------------+------+----+-----+-----+----------------+-------+-----+--------+\n",
              "|        892|     3|    Kelly, Mr. James|  male|34.5|    0|    0|          330911| 7.8292| NULL|       Q|\n",
              "|        893|     3|Wilkes, Mrs. Jame...|female|47.0|    1|    0|          363272|    7.0| NULL|       S|\n",
              "|        894|     2|Myles, Mr. Thomas...|  male|62.0|    0|    0|          240276| 9.6875| NULL|       Q|\n",
              "|        895|     3|    Wirz, Mr. Albert|  male|27.0|    0|    0|          315154| 8.6625| NULL|       S|\n",
              "|        896|     3|Hirvonen, Mrs. Al...|female|22.0|    1|    1|         3101298|12.2875| NULL|       S|\n",
              "|        897|     3|Svensson, Mr. Joh...|  male|14.0|    0|    0|            7538|  9.225| NULL|       S|\n",
              "|        898|     3|Connolly, Miss. Kate|female|30.0|    0|    0|          330972| 7.6292| NULL|       Q|\n",
              "|        899|     2|Caldwell, Mr. Alb...|  male|26.0|    1|    1|          248738|   29.0| NULL|       S|\n",
              "|        900|     3|Abrahim, Mrs. Jos...|female|18.0|    0|    0|            2657| 7.2292| NULL|       C|\n",
              "|        901|     3|Davies, Mr. John ...|  male|21.0|    2|    0|       A/4 48871|  24.15| NULL|       S|\n",
              "|        902|     3|    Ilieff, Mr. Ylio|  male|NULL|    0|    0|          349220| 7.8958| NULL|       S|\n",
              "|        903|     1|Jones, Mr. Charle...|  male|46.0|    0|    0|             694|   26.0| NULL|       S|\n",
              "|        904|     1|Snyder, Mrs. John...|female|23.0|    1|    0|           21228|82.2667|  B45|       S|\n",
              "|        905|     2|Howard, Mr. Benjamin|  male|63.0|    1|    0|           24065|   26.0| NULL|       S|\n",
              "|        906|     1|Chaffee, Mrs. Her...|female|47.0|    1|    0|     W.E.P. 5734| 61.175|  E31|       S|\n",
              "|        907|     2|del Carlo, Mrs. S...|female|24.0|    1|    0|   SC/PARIS 2167|27.7208| NULL|       C|\n",
              "|        908|     2|   Keane, Mr. Daniel|  male|35.0|    0|    0|          233734|  12.35| NULL|       Q|\n",
              "|        909|     3|   Assaf, Mr. Gerios|  male|21.0|    0|    0|            2692|  7.225| NULL|       C|\n",
              "|        910|     3|Ilmakangas, Miss....|female|27.0|    1|    0|STON/O2. 3101270|  7.925| NULL|       S|\n",
              "|        911|     3|\"Assaf Khalil, Mr...|female|45.0|    0|    0|            2696|  7.225| NULL|       C|\n",
              "+-----------+------+--------------------+------+----+-----+-----+----------------+-------+-----+--------+\n",
              "only showing top 20 rows"
            ],
            "text/html": [
              "<table border='1'>\n",
              "<tr><th>PassengerId</th><th>Pclass</th><th>Name</th><th>Sex</th><th>Age</th><th>SibSp</th><th>Parch</th><th>Ticket</th><th>Fare</th><th>Cabin</th><th>Embarked</th></tr>\n",
              "<tr><td>892</td><td>3</td><td>Kelly, Mr. James</td><td>male</td><td>34.5</td><td>0</td><td>0</td><td>330911</td><td>7.8292</td><td>NULL</td><td>Q</td></tr>\n",
              "<tr><td>893</td><td>3</td><td>Wilkes, Mrs. Jame...</td><td>female</td><td>47.0</td><td>1</td><td>0</td><td>363272</td><td>7.0</td><td>NULL</td><td>S</td></tr>\n",
              "<tr><td>894</td><td>2</td><td>Myles, Mr. Thomas...</td><td>male</td><td>62.0</td><td>0</td><td>0</td><td>240276</td><td>9.6875</td><td>NULL</td><td>Q</td></tr>\n",
              "<tr><td>895</td><td>3</td><td>Wirz, Mr. Albert</td><td>male</td><td>27.0</td><td>0</td><td>0</td><td>315154</td><td>8.6625</td><td>NULL</td><td>S</td></tr>\n",
              "<tr><td>896</td><td>3</td><td>Hirvonen, Mrs. Al...</td><td>female</td><td>22.0</td><td>1</td><td>1</td><td>3101298</td><td>12.2875</td><td>NULL</td><td>S</td></tr>\n",
              "<tr><td>897</td><td>3</td><td>Svensson, Mr. Joh...</td><td>male</td><td>14.0</td><td>0</td><td>0</td><td>7538</td><td>9.225</td><td>NULL</td><td>S</td></tr>\n",
              "<tr><td>898</td><td>3</td><td>Connolly, Miss. Kate</td><td>female</td><td>30.0</td><td>0</td><td>0</td><td>330972</td><td>7.6292</td><td>NULL</td><td>Q</td></tr>\n",
              "<tr><td>899</td><td>2</td><td>Caldwell, Mr. Alb...</td><td>male</td><td>26.0</td><td>1</td><td>1</td><td>248738</td><td>29.0</td><td>NULL</td><td>S</td></tr>\n",
              "<tr><td>900</td><td>3</td><td>Abrahim, Mrs. Jos...</td><td>female</td><td>18.0</td><td>0</td><td>0</td><td>2657</td><td>7.2292</td><td>NULL</td><td>C</td></tr>\n",
              "<tr><td>901</td><td>3</td><td>Davies, Mr. John ...</td><td>male</td><td>21.0</td><td>2</td><td>0</td><td>A/4 48871</td><td>24.15</td><td>NULL</td><td>S</td></tr>\n",
              "<tr><td>902</td><td>3</td><td>Ilieff, Mr. Ylio</td><td>male</td><td>NULL</td><td>0</td><td>0</td><td>349220</td><td>7.8958</td><td>NULL</td><td>S</td></tr>\n",
              "<tr><td>903</td><td>1</td><td>Jones, Mr. Charle...</td><td>male</td><td>46.0</td><td>0</td><td>0</td><td>694</td><td>26.0</td><td>NULL</td><td>S</td></tr>\n",
              "<tr><td>904</td><td>1</td><td>Snyder, Mrs. John...</td><td>female</td><td>23.0</td><td>1</td><td>0</td><td>21228</td><td>82.2667</td><td>B45</td><td>S</td></tr>\n",
              "<tr><td>905</td><td>2</td><td>Howard, Mr. Benjamin</td><td>male</td><td>63.0</td><td>1</td><td>0</td><td>24065</td><td>26.0</td><td>NULL</td><td>S</td></tr>\n",
              "<tr><td>906</td><td>1</td><td>Chaffee, Mrs. Her...</td><td>female</td><td>47.0</td><td>1</td><td>0</td><td>W.E.P. 5734</td><td>61.175</td><td>E31</td><td>S</td></tr>\n",
              "<tr><td>907</td><td>2</td><td>del Carlo, Mrs. S...</td><td>female</td><td>24.0</td><td>1</td><td>0</td><td>SC/PARIS 2167</td><td>27.7208</td><td>NULL</td><td>C</td></tr>\n",
              "<tr><td>908</td><td>2</td><td>Keane, Mr. Daniel</td><td>male</td><td>35.0</td><td>0</td><td>0</td><td>233734</td><td>12.35</td><td>NULL</td><td>Q</td></tr>\n",
              "<tr><td>909</td><td>3</td><td>Assaf, Mr. Gerios</td><td>male</td><td>21.0</td><td>0</td><td>0</td><td>2692</td><td>7.225</td><td>NULL</td><td>C</td></tr>\n",
              "<tr><td>910</td><td>3</td><td>Ilmakangas, Miss....</td><td>female</td><td>27.0</td><td>1</td><td>0</td><td>STON/O2. 3101270</td><td>7.925</td><td>NULL</td><td>S</td></tr>\n",
              "<tr><td>911</td><td>3</td><td>&quot;Assaf Khalil, Mr...</td><td>female</td><td>45.0</td><td>0</td><td>0</td><td>2696</td><td>7.225</td><td>NULL</td><td>C</td></tr>\n",
              "</table>\n",
              "only showing top 20 rows\n"
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "display(df)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 529
        },
        "id": "2fQq7HOmyS-9",
        "outputId": "2c4a4252-abb4-43bf-aa24-907657b08a39"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "DataFrame[PassengerId: int, Pclass: int, Name: string, Sex: string, Age: double, SibSp: int, Parch: int, Ticket: string, Fare: double, Cabin: string, Embarked: string]"
            ],
            "text/html": [
              "<table border='1'>\n",
              "<tr><th>PassengerId</th><th>Pclass</th><th>Name</th><th>Sex</th><th>Age</th><th>SibSp</th><th>Parch</th><th>Ticket</th><th>Fare</th><th>Cabin</th><th>Embarked</th></tr>\n",
              "<tr><td>892</td><td>3</td><td>Kelly, Mr. James</td><td>male</td><td>34.5</td><td>0</td><td>0</td><td>330911</td><td>7.8292</td><td>NULL</td><td>Q</td></tr>\n",
              "<tr><td>893</td><td>3</td><td>Wilkes, Mrs. Jame...</td><td>female</td><td>47.0</td><td>1</td><td>0</td><td>363272</td><td>7.0</td><td>NULL</td><td>S</td></tr>\n",
              "<tr><td>894</td><td>2</td><td>Myles, Mr. Thomas...</td><td>male</td><td>62.0</td><td>0</td><td>0</td><td>240276</td><td>9.6875</td><td>NULL</td><td>Q</td></tr>\n",
              "<tr><td>895</td><td>3</td><td>Wirz, Mr. Albert</td><td>male</td><td>27.0</td><td>0</td><td>0</td><td>315154</td><td>8.6625</td><td>NULL</td><td>S</td></tr>\n",
              "<tr><td>896</td><td>3</td><td>Hirvonen, Mrs. Al...</td><td>female</td><td>22.0</td><td>1</td><td>1</td><td>3101298</td><td>12.2875</td><td>NULL</td><td>S</td></tr>\n",
              "<tr><td>897</td><td>3</td><td>Svensson, Mr. Joh...</td><td>male</td><td>14.0</td><td>0</td><td>0</td><td>7538</td><td>9.225</td><td>NULL</td><td>S</td></tr>\n",
              "<tr><td>898</td><td>3</td><td>Connolly, Miss. Kate</td><td>female</td><td>30.0</td><td>0</td><td>0</td><td>330972</td><td>7.6292</td><td>NULL</td><td>Q</td></tr>\n",
              "<tr><td>899</td><td>2</td><td>Caldwell, Mr. Alb...</td><td>male</td><td>26.0</td><td>1</td><td>1</td><td>248738</td><td>29.0</td><td>NULL</td><td>S</td></tr>\n",
              "<tr><td>900</td><td>3</td><td>Abrahim, Mrs. Jos...</td><td>female</td><td>18.0</td><td>0</td><td>0</td><td>2657</td><td>7.2292</td><td>NULL</td><td>C</td></tr>\n",
              "<tr><td>901</td><td>3</td><td>Davies, Mr. John ...</td><td>male</td><td>21.0</td><td>2</td><td>0</td><td>A/4 48871</td><td>24.15</td><td>NULL</td><td>S</td></tr>\n",
              "<tr><td>902</td><td>3</td><td>Ilieff, Mr. Ylio</td><td>male</td><td>NULL</td><td>0</td><td>0</td><td>349220</td><td>7.8958</td><td>NULL</td><td>S</td></tr>\n",
              "<tr><td>903</td><td>1</td><td>Jones, Mr. Charle...</td><td>male</td><td>46.0</td><td>0</td><td>0</td><td>694</td><td>26.0</td><td>NULL</td><td>S</td></tr>\n",
              "<tr><td>904</td><td>1</td><td>Snyder, Mrs. John...</td><td>female</td><td>23.0</td><td>1</td><td>0</td><td>21228</td><td>82.2667</td><td>B45</td><td>S</td></tr>\n",
              "<tr><td>905</td><td>2</td><td>Howard, Mr. Benjamin</td><td>male</td><td>63.0</td><td>1</td><td>0</td><td>24065</td><td>26.0</td><td>NULL</td><td>S</td></tr>\n",
              "<tr><td>906</td><td>1</td><td>Chaffee, Mrs. Her...</td><td>female</td><td>47.0</td><td>1</td><td>0</td><td>W.E.P. 5734</td><td>61.175</td><td>E31</td><td>S</td></tr>\n",
              "<tr><td>907</td><td>2</td><td>del Carlo, Mrs. S...</td><td>female</td><td>24.0</td><td>1</td><td>0</td><td>SC/PARIS 2167</td><td>27.7208</td><td>NULL</td><td>C</td></tr>\n",
              "<tr><td>908</td><td>2</td><td>Keane, Mr. Daniel</td><td>male</td><td>35.0</td><td>0</td><td>0</td><td>233734</td><td>12.35</td><td>NULL</td><td>Q</td></tr>\n",
              "<tr><td>909</td><td>3</td><td>Assaf, Mr. Gerios</td><td>male</td><td>21.0</td><td>0</td><td>0</td><td>2692</td><td>7.225</td><td>NULL</td><td>C</td></tr>\n",
              "<tr><td>910</td><td>3</td><td>Ilmakangas, Miss....</td><td>female</td><td>27.0</td><td>1</td><td>0</td><td>STON/O2. 3101270</td><td>7.925</td><td>NULL</td><td>S</td></tr>\n",
              "<tr><td>911</td><td>3</td><td>&quot;Assaf Khalil, Mr...</td><td>female</td><td>45.0</td><td>0</td><td>0</td><td>2696</td><td>7.225</td><td>NULL</td><td>C</td></tr>\n",
              "</table>\n",
              "only showing top 20 rows\n"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df.head(6)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RXRb2vjTyd0b",
        "outputId": "7bd2a3b1-ef00-4f0c-91b3-8b5fcc87e818"
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[Row(PassengerId=892, Pclass=3, Name='Kelly, Mr. James', Sex='male', Age=34.5, SibSp=0, Parch=0, Ticket='330911', Fare=7.8292, Cabin=None, Embarked='Q'),\n",
              " Row(PassengerId=893, Pclass=3, Name='Wilkes, Mrs. James (Ellen Needs)', Sex='female', Age=47.0, SibSp=1, Parch=0, Ticket='363272', Fare=7.0, Cabin=None, Embarked='S'),\n",
              " Row(PassengerId=894, Pclass=2, Name='Myles, Mr. Thomas Francis', Sex='male', Age=62.0, SibSp=0, Parch=0, Ticket='240276', Fare=9.6875, Cabin=None, Embarked='Q'),\n",
              " Row(PassengerId=895, Pclass=3, Name='Wirz, Mr. Albert', Sex='male', Age=27.0, SibSp=0, Parch=0, Ticket='315154', Fare=8.6625, Cabin=None, Embarked='S'),\n",
              " Row(PassengerId=896, Pclass=3, Name='Hirvonen, Mrs. Alexander (Helga E Lindqvist)', Sex='female', Age=22.0, SibSp=1, Parch=1, Ticket='3101298', Fare=12.2875, Cabin=None, Embarked='S'),\n",
              " Row(PassengerId=897, Pclass=3, Name='Svensson, Mr. Johan Cervin', Sex='male', Age=14.0, SibSp=0, Parch=0, Ticket='7538', Fare=9.225, Cabin=None, Embarked='S')]"
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9clL4JW9yhy1",
        "outputId": "a0459a74-5b44-4bf7-9e8e-0fa183b6e927"
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-----------+------+--------------------+------+----+-----+-----+----------------+-------+-----+--------+\n",
            "|PassengerId|Pclass|                Name|   Sex| Age|SibSp|Parch|          Ticket|   Fare|Cabin|Embarked|\n",
            "+-----------+------+--------------------+------+----+-----+-----+----------------+-------+-----+--------+\n",
            "|        892|     3|    Kelly, Mr. James|  male|34.5|    0|    0|          330911| 7.8292| NULL|       Q|\n",
            "|        893|     3|Wilkes, Mrs. Jame...|female|47.0|    1|    0|          363272|    7.0| NULL|       S|\n",
            "|        894|     2|Myles, Mr. Thomas...|  male|62.0|    0|    0|          240276| 9.6875| NULL|       Q|\n",
            "|        895|     3|    Wirz, Mr. Albert|  male|27.0|    0|    0|          315154| 8.6625| NULL|       S|\n",
            "|        896|     3|Hirvonen, Mrs. Al...|female|22.0|    1|    1|         3101298|12.2875| NULL|       S|\n",
            "|        897|     3|Svensson, Mr. Joh...|  male|14.0|    0|    0|            7538|  9.225| NULL|       S|\n",
            "|        898|     3|Connolly, Miss. Kate|female|30.0|    0|    0|          330972| 7.6292| NULL|       Q|\n",
            "|        899|     2|Caldwell, Mr. Alb...|  male|26.0|    1|    1|          248738|   29.0| NULL|       S|\n",
            "|        900|     3|Abrahim, Mrs. Jos...|female|18.0|    0|    0|            2657| 7.2292| NULL|       C|\n",
            "|        901|     3|Davies, Mr. John ...|  male|21.0|    2|    0|       A/4 48871|  24.15| NULL|       S|\n",
            "|        902|     3|    Ilieff, Mr. Ylio|  male|NULL|    0|    0|          349220| 7.8958| NULL|       S|\n",
            "|        903|     1|Jones, Mr. Charle...|  male|46.0|    0|    0|             694|   26.0| NULL|       S|\n",
            "|        904|     1|Snyder, Mrs. John...|female|23.0|    1|    0|           21228|82.2667|  B45|       S|\n",
            "|        905|     2|Howard, Mr. Benjamin|  male|63.0|    1|    0|           24065|   26.0| NULL|       S|\n",
            "|        906|     1|Chaffee, Mrs. Her...|female|47.0|    1|    0|     W.E.P. 5734| 61.175|  E31|       S|\n",
            "|        907|     2|del Carlo, Mrs. S...|female|24.0|    1|    0|   SC/PARIS 2167|27.7208| NULL|       C|\n",
            "|        908|     2|   Keane, Mr. Daniel|  male|35.0|    0|    0|          233734|  12.35| NULL|       Q|\n",
            "|        909|     3|   Assaf, Mr. Gerios|  male|21.0|    0|    0|            2692|  7.225| NULL|       C|\n",
            "|        910|     3|Ilmakangas, Miss....|female|27.0|    1|    0|STON/O2. 3101270|  7.925| NULL|       S|\n",
            "|        911|     3|\"Assaf Khalil, Mr...|female|45.0|    0|    0|            2696|  7.225| NULL|       C|\n",
            "+-----------+------+--------------------+------+----+-----+-----+----------------+-------+-----+--------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Idade Média\n",
        "\n",
        "df.select(F.mean(\"age\")).show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hcwI67EPy6vC",
        "outputId": "d9fe3402-28ed-4a39-8933-a09a79e55ef6"
      },
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+------------------+\n",
            "|          avg(age)|\n",
            "+------------------+\n",
            "|30.272590361445783|\n",
            "+------------------+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# prompt: df.printschema()\n",
        "\n",
        "df.printSchema()\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CNXgPFXzzIab",
        "outputId": "6443d928-eaab-47f5-a1ce-aae18e1b68f8"
      },
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "root\n",
            " |-- PassengerId: integer (nullable = true)\n",
            " |-- Pclass: integer (nullable = true)\n",
            " |-- Name: string (nullable = true)\n",
            " |-- Sex: string (nullable = true)\n",
            " |-- Age: double (nullable = true)\n",
            " |-- SibSp: integer (nullable = true)\n",
            " |-- Parch: integer (nullable = true)\n",
            " |-- Ticket: string (nullable = true)\n",
            " |-- Fare: double (nullable = true)\n",
            " |-- Cabin: string (nullable = true)\n",
            " |-- Embarked: string (nullable = true)\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Convert para inteiro a Idade"
      ],
      "metadata": {
        "id": "HI_huFe42QPo"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "dfInt =  df.withColumn(\"age\", df[\"age\"].cast(\"int\"))"
      ],
      "metadata": {
        "id": "aMHFAXZMzqId"
      },
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "dfInt.printSchema()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "a2DN7o8e2GNX",
        "outputId": "5a3306c8-d96f-4ba4-9974-ce1627703c35"
      },
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "root\n",
            " |-- PassengerId: integer (nullable = true)\n",
            " |-- Pclass: integer (nullable = true)\n",
            " |-- Name: string (nullable = true)\n",
            " |-- Sex: string (nullable = true)\n",
            " |-- age: integer (nullable = true)\n",
            " |-- SibSp: integer (nullable = true)\n",
            " |-- Parch: integer (nullable = true)\n",
            " |-- Ticket: string (nullable = true)\n",
            " |-- Fare: double (nullable = true)\n",
            " |-- Cabin: string (nullable = true)\n",
            " |-- Embarked: string (nullable = true)\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "display(dfInt)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 529
        },
        "id": "OGUVxC0X2KDg",
        "outputId": "43b5b9ba-9d2b-47b5-b78a-323a7b2a265b"
      },
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "+-----------+------+--------------------+------+----+-----+-----+----------------+-------+-----+--------+\n",
              "|PassengerId|Pclass|                Name|   Sex| age|SibSp|Parch|          Ticket|   Fare|Cabin|Embarked|\n",
              "+-----------+------+--------------------+------+----+-----+-----+----------------+-------+-----+--------+\n",
              "|        892|     3|    Kelly, Mr. James|  male|  34|    0|    0|          330911| 7.8292| NULL|       Q|\n",
              "|        893|     3|Wilkes, Mrs. Jame...|female|  47|    1|    0|          363272|    7.0| NULL|       S|\n",
              "|        894|     2|Myles, Mr. Thomas...|  male|  62|    0|    0|          240276| 9.6875| NULL|       Q|\n",
              "|        895|     3|    Wirz, Mr. Albert|  male|  27|    0|    0|          315154| 8.6625| NULL|       S|\n",
              "|        896|     3|Hirvonen, Mrs. Al...|female|  22|    1|    1|         3101298|12.2875| NULL|       S|\n",
              "|        897|     3|Svensson, Mr. Joh...|  male|  14|    0|    0|            7538|  9.225| NULL|       S|\n",
              "|        898|     3|Connolly, Miss. Kate|female|  30|    0|    0|          330972| 7.6292| NULL|       Q|\n",
              "|        899|     2|Caldwell, Mr. Alb...|  male|  26|    1|    1|          248738|   29.0| NULL|       S|\n",
              "|        900|     3|Abrahim, Mrs. Jos...|female|  18|    0|    0|            2657| 7.2292| NULL|       C|\n",
              "|        901|     3|Davies, Mr. John ...|  male|  21|    2|    0|       A/4 48871|  24.15| NULL|       S|\n",
              "|        902|     3|    Ilieff, Mr. Ylio|  male|NULL|    0|    0|          349220| 7.8958| NULL|       S|\n",
              "|        903|     1|Jones, Mr. Charle...|  male|  46|    0|    0|             694|   26.0| NULL|       S|\n",
              "|        904|     1|Snyder, Mrs. John...|female|  23|    1|    0|           21228|82.2667|  B45|       S|\n",
              "|        905|     2|Howard, Mr. Benjamin|  male|  63|    1|    0|           24065|   26.0| NULL|       S|\n",
              "|        906|     1|Chaffee, Mrs. Her...|female|  47|    1|    0|     W.E.P. 5734| 61.175|  E31|       S|\n",
              "|        907|     2|del Carlo, Mrs. S...|female|  24|    1|    0|   SC/PARIS 2167|27.7208| NULL|       C|\n",
              "|        908|     2|   Keane, Mr. Daniel|  male|  35|    0|    0|          233734|  12.35| NULL|       Q|\n",
              "|        909|     3|   Assaf, Mr. Gerios|  male|  21|    0|    0|            2692|  7.225| NULL|       C|\n",
              "|        910|     3|Ilmakangas, Miss....|female|  27|    1|    0|STON/O2. 3101270|  7.925| NULL|       S|\n",
              "|        911|     3|\"Assaf Khalil, Mr...|female|  45|    0|    0|            2696|  7.225| NULL|       C|\n",
              "+-----------+------+--------------------+------+----+-----+-----+----------------+-------+-----+--------+\n",
              "only showing top 20 rows"
            ],
            "text/html": [
              "<table border='1'>\n",
              "<tr><th>PassengerId</th><th>Pclass</th><th>Name</th><th>Sex</th><th>age</th><th>SibSp</th><th>Parch</th><th>Ticket</th><th>Fare</th><th>Cabin</th><th>Embarked</th></tr>\n",
              "<tr><td>892</td><td>3</td><td>Kelly, Mr. James</td><td>male</td><td>34</td><td>0</td><td>0</td><td>330911</td><td>7.8292</td><td>NULL</td><td>Q</td></tr>\n",
              "<tr><td>893</td><td>3</td><td>Wilkes, Mrs. Jame...</td><td>female</td><td>47</td><td>1</td><td>0</td><td>363272</td><td>7.0</td><td>NULL</td><td>S</td></tr>\n",
              "<tr><td>894</td><td>2</td><td>Myles, Mr. Thomas...</td><td>male</td><td>62</td><td>0</td><td>0</td><td>240276</td><td>9.6875</td><td>NULL</td><td>Q</td></tr>\n",
              "<tr><td>895</td><td>3</td><td>Wirz, Mr. Albert</td><td>male</td><td>27</td><td>0</td><td>0</td><td>315154</td><td>8.6625</td><td>NULL</td><td>S</td></tr>\n",
              "<tr><td>896</td><td>3</td><td>Hirvonen, Mrs. Al...</td><td>female</td><td>22</td><td>1</td><td>1</td><td>3101298</td><td>12.2875</td><td>NULL</td><td>S</td></tr>\n",
              "<tr><td>897</td><td>3</td><td>Svensson, Mr. Joh...</td><td>male</td><td>14</td><td>0</td><td>0</td><td>7538</td><td>9.225</td><td>NULL</td><td>S</td></tr>\n",
              "<tr><td>898</td><td>3</td><td>Connolly, Miss. Kate</td><td>female</td><td>30</td><td>0</td><td>0</td><td>330972</td><td>7.6292</td><td>NULL</td><td>Q</td></tr>\n",
              "<tr><td>899</td><td>2</td><td>Caldwell, Mr. Alb...</td><td>male</td><td>26</td><td>1</td><td>1</td><td>248738</td><td>29.0</td><td>NULL</td><td>S</td></tr>\n",
              "<tr><td>900</td><td>3</td><td>Abrahim, Mrs. Jos...</td><td>female</td><td>18</td><td>0</td><td>0</td><td>2657</td><td>7.2292</td><td>NULL</td><td>C</td></tr>\n",
              "<tr><td>901</td><td>3</td><td>Davies, Mr. John ...</td><td>male</td><td>21</td><td>2</td><td>0</td><td>A/4 48871</td><td>24.15</td><td>NULL</td><td>S</td></tr>\n",
              "<tr><td>902</td><td>3</td><td>Ilieff, Mr. Ylio</td><td>male</td><td>NULL</td><td>0</td><td>0</td><td>349220</td><td>7.8958</td><td>NULL</td><td>S</td></tr>\n",
              "<tr><td>903</td><td>1</td><td>Jones, Mr. Charle...</td><td>male</td><td>46</td><td>0</td><td>0</td><td>694</td><td>26.0</td><td>NULL</td><td>S</td></tr>\n",
              "<tr><td>904</td><td>1</td><td>Snyder, Mrs. John...</td><td>female</td><td>23</td><td>1</td><td>0</td><td>21228</td><td>82.2667</td><td>B45</td><td>S</td></tr>\n",
              "<tr><td>905</td><td>2</td><td>Howard, Mr. Benjamin</td><td>male</td><td>63</td><td>1</td><td>0</td><td>24065</td><td>26.0</td><td>NULL</td><td>S</td></tr>\n",
              "<tr><td>906</td><td>1</td><td>Chaffee, Mrs. Her...</td><td>female</td><td>47</td><td>1</td><td>0</td><td>W.E.P. 5734</td><td>61.175</td><td>E31</td><td>S</td></tr>\n",
              "<tr><td>907</td><td>2</td><td>del Carlo, Mrs. S...</td><td>female</td><td>24</td><td>1</td><td>0</td><td>SC/PARIS 2167</td><td>27.7208</td><td>NULL</td><td>C</td></tr>\n",
              "<tr><td>908</td><td>2</td><td>Keane, Mr. Daniel</td><td>male</td><td>35</td><td>0</td><td>0</td><td>233734</td><td>12.35</td><td>NULL</td><td>Q</td></tr>\n",
              "<tr><td>909</td><td>3</td><td>Assaf, Mr. Gerios</td><td>male</td><td>21</td><td>0</td><td>0</td><td>2692</td><td>7.225</td><td>NULL</td><td>C</td></tr>\n",
              "<tr><td>910</td><td>3</td><td>Ilmakangas, Miss....</td><td>female</td><td>27</td><td>1</td><td>0</td><td>STON/O2. 3101270</td><td>7.925</td><td>NULL</td><td>S</td></tr>\n",
              "<tr><td>911</td><td>3</td><td>&quot;Assaf Khalil, Mr...</td><td>female</td><td>45</td><td>0</td><td>0</td><td>2696</td><td>7.225</td><td>NULL</td><td>C</td></tr>\n",
              "</table>\n",
              "only showing top 20 rows\n"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Adiciona DataLog"
      ],
      "metadata": {
        "id": "S4K8fwxs3JRI"
      }
    },
    {
      "source": [
        "from pyspark.sql.functions import current_timestamp, to_date, date_format\n",
        "# Adiciona a coluna `datalog` com o timestamp atual\n",
        "df = dfInt.withColumn(\"datalog\", current_timestamp())\n",
        "\n",
        "# Converte a coluna `datalog` para apenas data\n",
        "df = df.withColumn(\"datalog\", to_date(\"datalog\")) # Changed dfInt to df"
      ],
      "cell_type": "code",
      "metadata": {
        "id": "tl9Qq6gX3G1r"
      },
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "display(df)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 529
        },
        "id": "F7vdaipM3OjF",
        "outputId": "41e29875-b6b6-4ca3-fe82-3a70423f422d"
      },
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "+-----------+------+--------------------+------+----+-----+-----+----------------+-------+-----+--------+----------+\n",
              "|PassengerId|Pclass|                Name|   Sex| age|SibSp|Parch|          Ticket|   Fare|Cabin|Embarked|   datalog|\n",
              "+-----------+------+--------------------+------+----+-----+-----+----------------+-------+-----+--------+----------+\n",
              "|        892|     3|    Kelly, Mr. James|  male|  34|    0|    0|          330911| 7.8292| NULL|       Q|2025-01-21|\n",
              "|        893|     3|Wilkes, Mrs. Jame...|female|  47|    1|    0|          363272|    7.0| NULL|       S|2025-01-21|\n",
              "|        894|     2|Myles, Mr. Thomas...|  male|  62|    0|    0|          240276| 9.6875| NULL|       Q|2025-01-21|\n",
              "|        895|     3|    Wirz, Mr. Albert|  male|  27|    0|    0|          315154| 8.6625| NULL|       S|2025-01-21|\n",
              "|        896|     3|Hirvonen, Mrs. Al...|female|  22|    1|    1|         3101298|12.2875| NULL|       S|2025-01-21|\n",
              "|        897|     3|Svensson, Mr. Joh...|  male|  14|    0|    0|            7538|  9.225| NULL|       S|2025-01-21|\n",
              "|        898|     3|Connolly, Miss. Kate|female|  30|    0|    0|          330972| 7.6292| NULL|       Q|2025-01-21|\n",
              "|        899|     2|Caldwell, Mr. Alb...|  male|  26|    1|    1|          248738|   29.0| NULL|       S|2025-01-21|\n",
              "|        900|     3|Abrahim, Mrs. Jos...|female|  18|    0|    0|            2657| 7.2292| NULL|       C|2025-01-21|\n",
              "|        901|     3|Davies, Mr. John ...|  male|  21|    2|    0|       A/4 48871|  24.15| NULL|       S|2025-01-21|\n",
              "|        902|     3|    Ilieff, Mr. Ylio|  male|NULL|    0|    0|          349220| 7.8958| NULL|       S|2025-01-21|\n",
              "|        903|     1|Jones, Mr. Charle...|  male|  46|    0|    0|             694|   26.0| NULL|       S|2025-01-21|\n",
              "|        904|     1|Snyder, Mrs. John...|female|  23|    1|    0|           21228|82.2667|  B45|       S|2025-01-21|\n",
              "|        905|     2|Howard, Mr. Benjamin|  male|  63|    1|    0|           24065|   26.0| NULL|       S|2025-01-21|\n",
              "|        906|     1|Chaffee, Mrs. Her...|female|  47|    1|    0|     W.E.P. 5734| 61.175|  E31|       S|2025-01-21|\n",
              "|        907|     2|del Carlo, Mrs. S...|female|  24|    1|    0|   SC/PARIS 2167|27.7208| NULL|       C|2025-01-21|\n",
              "|        908|     2|   Keane, Mr. Daniel|  male|  35|    0|    0|          233734|  12.35| NULL|       Q|2025-01-21|\n",
              "|        909|     3|   Assaf, Mr. Gerios|  male|  21|    0|    0|            2692|  7.225| NULL|       C|2025-01-21|\n",
              "|        910|     3|Ilmakangas, Miss....|female|  27|    1|    0|STON/O2. 3101270|  7.925| NULL|       S|2025-01-21|\n",
              "|        911|     3|\"Assaf Khalil, Mr...|female|  45|    0|    0|            2696|  7.225| NULL|       C|2025-01-21|\n",
              "+-----------+------+--------------------+------+----+-----+-----+----------------+-------+-----+--------+----------+\n",
              "only showing top 20 rows"
            ],
            "text/html": [
              "<table border='1'>\n",
              "<tr><th>PassengerId</th><th>Pclass</th><th>Name</th><th>Sex</th><th>age</th><th>SibSp</th><th>Parch</th><th>Ticket</th><th>Fare</th><th>Cabin</th><th>Embarked</th><th>datalog</th></tr>\n",
              "<tr><td>892</td><td>3</td><td>Kelly, Mr. James</td><td>male</td><td>34</td><td>0</td><td>0</td><td>330911</td><td>7.8292</td><td>NULL</td><td>Q</td><td>2025-01-21</td></tr>\n",
              "<tr><td>893</td><td>3</td><td>Wilkes, Mrs. Jame...</td><td>female</td><td>47</td><td>1</td><td>0</td><td>363272</td><td>7.0</td><td>NULL</td><td>S</td><td>2025-01-21</td></tr>\n",
              "<tr><td>894</td><td>2</td><td>Myles, Mr. Thomas...</td><td>male</td><td>62</td><td>0</td><td>0</td><td>240276</td><td>9.6875</td><td>NULL</td><td>Q</td><td>2025-01-21</td></tr>\n",
              "<tr><td>895</td><td>3</td><td>Wirz, Mr. Albert</td><td>male</td><td>27</td><td>0</td><td>0</td><td>315154</td><td>8.6625</td><td>NULL</td><td>S</td><td>2025-01-21</td></tr>\n",
              "<tr><td>896</td><td>3</td><td>Hirvonen, Mrs. Al...</td><td>female</td><td>22</td><td>1</td><td>1</td><td>3101298</td><td>12.2875</td><td>NULL</td><td>S</td><td>2025-01-21</td></tr>\n",
              "<tr><td>897</td><td>3</td><td>Svensson, Mr. Joh...</td><td>male</td><td>14</td><td>0</td><td>0</td><td>7538</td><td>9.225</td><td>NULL</td><td>S</td><td>2025-01-21</td></tr>\n",
              "<tr><td>898</td><td>3</td><td>Connolly, Miss. Kate</td><td>female</td><td>30</td><td>0</td><td>0</td><td>330972</td><td>7.6292</td><td>NULL</td><td>Q</td><td>2025-01-21</td></tr>\n",
              "<tr><td>899</td><td>2</td><td>Caldwell, Mr. Alb...</td><td>male</td><td>26</td><td>1</td><td>1</td><td>248738</td><td>29.0</td><td>NULL</td><td>S</td><td>2025-01-21</td></tr>\n",
              "<tr><td>900</td><td>3</td><td>Abrahim, Mrs. Jos...</td><td>female</td><td>18</td><td>0</td><td>0</td><td>2657</td><td>7.2292</td><td>NULL</td><td>C</td><td>2025-01-21</td></tr>\n",
              "<tr><td>901</td><td>3</td><td>Davies, Mr. John ...</td><td>male</td><td>21</td><td>2</td><td>0</td><td>A/4 48871</td><td>24.15</td><td>NULL</td><td>S</td><td>2025-01-21</td></tr>\n",
              "<tr><td>902</td><td>3</td><td>Ilieff, Mr. Ylio</td><td>male</td><td>NULL</td><td>0</td><td>0</td><td>349220</td><td>7.8958</td><td>NULL</td><td>S</td><td>2025-01-21</td></tr>\n",
              "<tr><td>903</td><td>1</td><td>Jones, Mr. Charle...</td><td>male</td><td>46</td><td>0</td><td>0</td><td>694</td><td>26.0</td><td>NULL</td><td>S</td><td>2025-01-21</td></tr>\n",
              "<tr><td>904</td><td>1</td><td>Snyder, Mrs. John...</td><td>female</td><td>23</td><td>1</td><td>0</td><td>21228</td><td>82.2667</td><td>B45</td><td>S</td><td>2025-01-21</td></tr>\n",
              "<tr><td>905</td><td>2</td><td>Howard, Mr. Benjamin</td><td>male</td><td>63</td><td>1</td><td>0</td><td>24065</td><td>26.0</td><td>NULL</td><td>S</td><td>2025-01-21</td></tr>\n",
              "<tr><td>906</td><td>1</td><td>Chaffee, Mrs. Her...</td><td>female</td><td>47</td><td>1</td><td>0</td><td>W.E.P. 5734</td><td>61.175</td><td>E31</td><td>S</td><td>2025-01-21</td></tr>\n",
              "<tr><td>907</td><td>2</td><td>del Carlo, Mrs. S...</td><td>female</td><td>24</td><td>1</td><td>0</td><td>SC/PARIS 2167</td><td>27.7208</td><td>NULL</td><td>C</td><td>2025-01-21</td></tr>\n",
              "<tr><td>908</td><td>2</td><td>Keane, Mr. Daniel</td><td>male</td><td>35</td><td>0</td><td>0</td><td>233734</td><td>12.35</td><td>NULL</td><td>Q</td><td>2025-01-21</td></tr>\n",
              "<tr><td>909</td><td>3</td><td>Assaf, Mr. Gerios</td><td>male</td><td>21</td><td>0</td><td>0</td><td>2692</td><td>7.225</td><td>NULL</td><td>C</td><td>2025-01-21</td></tr>\n",
              "<tr><td>910</td><td>3</td><td>Ilmakangas, Miss....</td><td>female</td><td>27</td><td>1</td><td>0</td><td>STON/O2. 3101270</td><td>7.925</td><td>NULL</td><td>S</td><td>2025-01-21</td></tr>\n",
              "<tr><td>911</td><td>3</td><td>&quot;Assaf Khalil, Mr...</td><td>female</td><td>45</td><td>0</td><td>0</td><td>2696</td><td>7.225</td><td>NULL</td><td>C</td><td>2025-01-21</td></tr>\n",
              "</table>\n",
              "only showing top 20 rows\n"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df.count()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2xCPEWEb3T7M",
        "outputId": "e58c8bef-acff-4dcf-993b-99498dfd3ada"
      },
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "418"
            ]
          },
          "metadata": {},
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Adicionado dia da Semana"
      ],
      "metadata": {
        "id": "g0S7pJJo3qGY"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Adiciona a coluna `dia_semana` com o nome do dia da semana\n",
        "df = df.withColumn(\"dia_semana\", date_format(\"datalog\", \"EEEE\"))\n",
        "\n",
        "# Exibe o esquema do DataFrame para confirmar as novas colunas\n",
        "df.printSchema()\n",
        "\n",
        "# Mostra as primeiras linhas do DataFrame\n",
        "df.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "aTRmV5jH3ove",
        "outputId": "45bd281f-9c99-4d9a-b597-a7eda55920d2"
      },
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "root\n",
            " |-- PassengerId: integer (nullable = true)\n",
            " |-- Pclass: integer (nullable = true)\n",
            " |-- Name: string (nullable = true)\n",
            " |-- Sex: string (nullable = true)\n",
            " |-- age: integer (nullable = true)\n",
            " |-- SibSp: integer (nullable = true)\n",
            " |-- Parch: integer (nullable = true)\n",
            " |-- Ticket: string (nullable = true)\n",
            " |-- Fare: double (nullable = true)\n",
            " |-- Cabin: string (nullable = true)\n",
            " |-- Embarked: string (nullable = true)\n",
            " |-- datalog: date (nullable = false)\n",
            " |-- dia_semana: string (nullable = false)\n",
            "\n",
            "+-----------+------+--------------------+------+----+-----+-----+----------------+-------+-----+--------+----------+----------+\n",
            "|PassengerId|Pclass|                Name|   Sex| age|SibSp|Parch|          Ticket|   Fare|Cabin|Embarked|   datalog|dia_semana|\n",
            "+-----------+------+--------------------+------+----+-----+-----+----------------+-------+-----+--------+----------+----------+\n",
            "|        892|     3|    Kelly, Mr. James|  male|  34|    0|    0|          330911| 7.8292| NULL|       Q|2025-01-21|   Tuesday|\n",
            "|        893|     3|Wilkes, Mrs. Jame...|female|  47|    1|    0|          363272|    7.0| NULL|       S|2025-01-21|   Tuesday|\n",
            "|        894|     2|Myles, Mr. Thomas...|  male|  62|    0|    0|          240276| 9.6875| NULL|       Q|2025-01-21|   Tuesday|\n",
            "|        895|     3|    Wirz, Mr. Albert|  male|  27|    0|    0|          315154| 8.6625| NULL|       S|2025-01-21|   Tuesday|\n",
            "|        896|     3|Hirvonen, Mrs. Al...|female|  22|    1|    1|         3101298|12.2875| NULL|       S|2025-01-21|   Tuesday|\n",
            "|        897|     3|Svensson, Mr. Joh...|  male|  14|    0|    0|            7538|  9.225| NULL|       S|2025-01-21|   Tuesday|\n",
            "|        898|     3|Connolly, Miss. Kate|female|  30|    0|    0|          330972| 7.6292| NULL|       Q|2025-01-21|   Tuesday|\n",
            "|        899|     2|Caldwell, Mr. Alb...|  male|  26|    1|    1|          248738|   29.0| NULL|       S|2025-01-21|   Tuesday|\n",
            "|        900|     3|Abrahim, Mrs. Jos...|female|  18|    0|    0|            2657| 7.2292| NULL|       C|2025-01-21|   Tuesday|\n",
            "|        901|     3|Davies, Mr. John ...|  male|  21|    2|    0|       A/4 48871|  24.15| NULL|       S|2025-01-21|   Tuesday|\n",
            "|        902|     3|    Ilieff, Mr. Ylio|  male|NULL|    0|    0|          349220| 7.8958| NULL|       S|2025-01-21|   Tuesday|\n",
            "|        903|     1|Jones, Mr. Charle...|  male|  46|    0|    0|             694|   26.0| NULL|       S|2025-01-21|   Tuesday|\n",
            "|        904|     1|Snyder, Mrs. John...|female|  23|    1|    0|           21228|82.2667|  B45|       S|2025-01-21|   Tuesday|\n",
            "|        905|     2|Howard, Mr. Benjamin|  male|  63|    1|    0|           24065|   26.0| NULL|       S|2025-01-21|   Tuesday|\n",
            "|        906|     1|Chaffee, Mrs. Her...|female|  47|    1|    0|     W.E.P. 5734| 61.175|  E31|       S|2025-01-21|   Tuesday|\n",
            "|        907|     2|del Carlo, Mrs. S...|female|  24|    1|    0|   SC/PARIS 2167|27.7208| NULL|       C|2025-01-21|   Tuesday|\n",
            "|        908|     2|   Keane, Mr. Daniel|  male|  35|    0|    0|          233734|  12.35| NULL|       Q|2025-01-21|   Tuesday|\n",
            "|        909|     3|   Assaf, Mr. Gerios|  male|  21|    0|    0|            2692|  7.225| NULL|       C|2025-01-21|   Tuesday|\n",
            "|        910|     3|Ilmakangas, Miss....|female|  27|    1|    0|STON/O2. 3101270|  7.925| NULL|       S|2025-01-21|   Tuesday|\n",
            "|        911|     3|\"Assaf Khalil, Mr...|female|  45|    0|    0|            2696|  7.225| NULL|       C|2025-01-21|   Tuesday|\n",
            "+-----------+------+--------------------+------+----+-----+-----+----------------+-------+-----+--------+----------+----------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Criando Rank"
      ],
      "metadata": {
        "id": "5qbHdDRR4mrf"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from pyspark.sql.window import Window\n",
        "from pyspark.sql.functions import row_number\n",
        "\n",
        "window_spec = Window.orderBy(\"age\")\n",
        "\n",
        "df_with_rank = df.withColumn(\"rank\", row_number().over(window_spec))\n",
        "\n",
        "df_with_rank.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9tTwRJ7y3YVO",
        "outputId": "2ae9f703-f60a-4318-ad14-15c4903e85cf"
      },
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-----------+------+--------------------+------+----+-----+-----+-----------+-------+-----+--------+----------+----------+----+\n",
            "|PassengerId|Pclass|                Name|   Sex| age|SibSp|Parch|     Ticket|   Fare|Cabin|Embarked|   datalog|dia_semana|rank|\n",
            "+-----------+------+--------------------+------+----+-----+-----+-----------+-------+-----+--------+----------+----------+----+\n",
            "|        983|     3|  Pedersen, Mr. Olaf|  male|NULL|    0|    0|     345498|  7.775| NULL|       S|2025-01-21|   Tuesday|   1|\n",
            "|       1055|     3|  Pearce, Mr. Ernest|  male|NULL|    0|    0|     343271|    7.0| NULL|       S|2025-01-21|   Tuesday|   2|\n",
            "|        985|     3|   Guest, Mr. Robert|  male|NULL|    0|    0|     376563|   8.05| NULL|       S|2025-01-21|   Tuesday|   3|\n",
            "|        946|     2|Mangiavacchi, Mr....|  male|NULL|    0|    0|SC/A.3 2861|15.5792| NULL|       C|2025-01-21|   Tuesday|   4|\n",
            "|        994|     3|  Foley, Mr. William|  male|NULL|    0|    0|     365235|   7.75| NULL|       Q|2025-01-21|   Tuesday|   5|\n",
            "|        921|     3|   Samaan, Mr. Elias|  male|NULL|    2|    0|       2662|21.6792| NULL|       C|2025-01-21|   Tuesday|   6|\n",
            "|        999|     3|    Ryan, Mr. Edward|  male|NULL|    0|    0|     383162|   7.75| NULL|       Q|2025-01-21|   Tuesday|   7|\n",
            "|        928|     3| Roth, Miss. Sarah A|female|NULL|    0|    0|     342712|   8.05| NULL|       S|2025-01-21|   Tuesday|   8|\n",
            "|       1000|     3|\"Willer, Mr. Aaro...|  male|NULL|    0|    0|       3410| 8.7125| NULL|       S|2025-01-21|   Tuesday|   9|\n",
            "|        902|     3|    Ilieff, Mr. Ylio|  male|NULL|    0|    0|     349220| 7.8958| NULL|       S|2025-01-21|   Tuesday|  10|\n",
            "|       1052|     3|  Smyth, Miss. Julia|female|NULL|    0|    0|     335432| 7.7333| NULL|       Q|2025-01-21|   Tuesday|  11|\n",
            "|       1043|     3|Matinoff, Mr. Nicola|  male|NULL|    0|    0|     349255| 7.8958| NULL|       C|2025-01-21|   Tuesday|  12|\n",
            "|       1003|     3|Shine, Miss. Elle...|female|NULL|    0|    0|     330968| 7.7792| NULL|       Q|2025-01-21|   Tuesday|  13|\n",
            "|        950|     3|Davison, Mr. Thom...|  male|NULL|    1|    0|     386525|   16.1| NULL|       S|2025-01-21|   Tuesday|  14|\n",
            "|       1008|     3|    Thomas, Mr. John|  male|NULL|    0|    0|       2681| 6.4375| NULL|       C|2025-01-21|   Tuesday|  15|\n",
            "|        968|     3|    Miles, Mr. Frank|  male|NULL|    0|    0|     359306|   8.05| NULL|       S|2025-01-21|   Tuesday|  16|\n",
            "|        975|     3|Demetri, Mr. Marinko|  male|NULL|    0|    0|     349238| 7.8958| NULL|       S|2025-01-21|   Tuesday|  17|\n",
            "|        976|     2|Lamb, Mr. John Jo...|  male|NULL|    0|    0|     240261|10.7083| NULL|       Q|2025-01-21|   Tuesday|  18|\n",
            "|        977|     3|  Khalil, Mr. Betros|  male|NULL|    1|    0|       2660|14.4542| NULL|       C|2025-01-21|   Tuesday|  19|\n",
            "|       1013|     3|   Kiernan, Mr. John|  male|NULL|    1|    0|     367227|   7.75| NULL|       Q|2025-01-21|   Tuesday|  20|\n",
            "+-----------+------+--------------------+------+----+-----+-----+-----------+-------+-----+--------+----------+----------+----+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "\n",
        "from pyspark.sql import SparkSession\n",
        "import pyspark.sql.functions as F\n",
        "import os\n",
        "import pandas as pd\n",
        "from google.colab import drive\n",
        "from pyspark.sql.functions import current_timestamp, to_date, date_format\n",
        "from pyspark.sql.window import Window\n",
        "from pyspark.sql.functions import row_number\n",
        "\n",
        "# Loading data\n",
        "\n",
        "spark = (\n",
        "    SparkSession.builder.appName('PySpark - LucianaSampaio')\n",
        "    .config('spark.sql.repl.eagerEval.enabled', True)\n",
        "    .getOrCreate()\n",
        ")\n",
        "\n",
        "drive.mount('/content/drive')\n",
        "\n",
        "# Specify the temporary file path\n",
        "temp_file_path = \"/tmp/temp_data.csv\"\n",
        "\n",
        "df = spark.read.csv('/content/drive/MyDrive/.Dataset/test.csv', sep=',', encoding='UTF-8', header=True, inferSchema=True)\n",
        "\n",
        "# Save to temporary file\n",
        "df.write.csv(temp_file_path, header=True)\n",
        "\n",
        "# Load from the temporary file\n",
        "df = spark.read.csv(temp_file_path, header=True, inferSchema=True)\n",
        "\n",
        "df.printSchema()\n",
        "\n",
        "# Idade Média\n",
        "df.select(F.mean(\"age\")).show()\n",
        "\n",
        "\n",
        "# Convert para inteiro a Idade\n",
        "dfInt =  df.withColumn(\"age\", df[\"age\"].cast(\"int\"))\n",
        "dfInt.printSchema()\n",
        "\n",
        "# Adiciona DataLog\n",
        "# Adiciona a coluna `datalog` com o timestamp atual\n",
        "df = dfInt.withColumn(\"datalog\", current_timestamp())\n",
        "\n",
        "# Converte a coluna `datalog` para apenas data\n",
        "df = df.withColumn(\"datalog\", to_date(\"datalog\"))\n",
        "df.show()\n",
        "\n",
        "df.count()\n",
        "\n",
        "# Adicionado dia da Semana\n",
        "# Adiciona a coluna `dia_semana` com o nome do dia da semana\n",
        "df = df.withColumn(\"dia_semana\", date_format(\"datalog\", \"EEEE\"))\n",
        "\n",
        "\n",
        "# Exibe o esquema do DataFrame para confirmar as novas colunas\n",
        "df.printSchema()\n",
        "\n",
        "# Mostra as primeiras linhas do DataFrame\n",
        "df.show()\n",
        "# Criando Rank\n",
        "\n",
        "window_spec = Window.orderBy(\"age\")\n",
        "\n",
        "df_with_rank = df.withColumn(\"rank\", row_number().over(window_spec))\n",
        "\n",
        "df_with_rank.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5Vibdcix5CRt",
        "outputId": "90e672be-cea6-49cd-e136-b6723e381d33"
      },
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n",
            "root\n",
            " |-- PassengerId: integer (nullable = true)\n",
            " |-- Pclass: integer (nullable = true)\n",
            " |-- Name: string (nullable = true)\n",
            " |-- Sex: string (nullable = true)\n",
            " |-- Age: double (nullable = true)\n",
            " |-- SibSp: integer (nullable = true)\n",
            " |-- Parch: integer (nullable = true)\n",
            " |-- Ticket: string (nullable = true)\n",
            " |-- Fare: double (nullable = true)\n",
            " |-- Cabin: string (nullable = true)\n",
            " |-- Embarked: string (nullable = true)\n",
            "\n",
            "+------------------+\n",
            "|          avg(age)|\n",
            "+------------------+\n",
            "|30.272590361445783|\n",
            "+------------------+\n",
            "\n",
            "root\n",
            " |-- PassengerId: integer (nullable = true)\n",
            " |-- Pclass: integer (nullable = true)\n",
            " |-- Name: string (nullable = true)\n",
            " |-- Sex: string (nullable = true)\n",
            " |-- age: integer (nullable = true)\n",
            " |-- SibSp: integer (nullable = true)\n",
            " |-- Parch: integer (nullable = true)\n",
            " |-- Ticket: string (nullable = true)\n",
            " |-- Fare: double (nullable = true)\n",
            " |-- Cabin: string (nullable = true)\n",
            " |-- Embarked: string (nullable = true)\n",
            "\n",
            "+-----------+------+--------------------+------+----+-----+-----+----------------+-------+-----+--------+----------+\n",
            "|PassengerId|Pclass|                Name|   Sex| age|SibSp|Parch|          Ticket|   Fare|Cabin|Embarked|   datalog|\n",
            "+-----------+------+--------------------+------+----+-----+-----+----------------+-------+-----+--------+----------+\n",
            "|        892|     3|    Kelly, Mr. James|  male|  34|    0|    0|          330911| 7.8292| NULL|       Q|2025-01-21|\n",
            "|        893|     3|Wilkes, Mrs. Jame...|female|  47|    1|    0|          363272|    7.0| NULL|       S|2025-01-21|\n",
            "|        894|     2|Myles, Mr. Thomas...|  male|  62|    0|    0|          240276| 9.6875| NULL|       Q|2025-01-21|\n",
            "|        895|     3|    Wirz, Mr. Albert|  male|  27|    0|    0|          315154| 8.6625| NULL|       S|2025-01-21|\n",
            "|        896|     3|Hirvonen, Mrs. Al...|female|  22|    1|    1|         3101298|12.2875| NULL|       S|2025-01-21|\n",
            "|        897|     3|Svensson, Mr. Joh...|  male|  14|    0|    0|            7538|  9.225| NULL|       S|2025-01-21|\n",
            "|        898|     3|Connolly, Miss. Kate|female|  30|    0|    0|          330972| 7.6292| NULL|       Q|2025-01-21|\n",
            "|        899|     2|Caldwell, Mr. Alb...|  male|  26|    1|    1|          248738|   29.0| NULL|       S|2025-01-21|\n",
            "|        900|     3|Abrahim, Mrs. Jos...|female|  18|    0|    0|            2657| 7.2292| NULL|       C|2025-01-21|\n",
            "|        901|     3|Davies, Mr. John ...|  male|  21|    2|    0|       A/4 48871|  24.15| NULL|       S|2025-01-21|\n",
            "|        902|     3|    Ilieff, Mr. Ylio|  male|NULL|    0|    0|          349220| 7.8958| NULL|       S|2025-01-21|\n",
            "|        903|     1|Jones, Mr. Charle...|  male|  46|    0|    0|             694|   26.0| NULL|       S|2025-01-21|\n",
            "|        904|     1|Snyder, Mrs. John...|female|  23|    1|    0|           21228|82.2667|  B45|       S|2025-01-21|\n",
            "|        905|     2|Howard, Mr. Benjamin|  male|  63|    1|    0|           24065|   26.0| NULL|       S|2025-01-21|\n",
            "|        906|     1|Chaffee, Mrs. Her...|female|  47|    1|    0|     W.E.P. 5734| 61.175|  E31|       S|2025-01-21|\n",
            "|        907|     2|del Carlo, Mrs. S...|female|  24|    1|    0|   SC/PARIS 2167|27.7208| NULL|       C|2025-01-21|\n",
            "|        908|     2|   Keane, Mr. Daniel|  male|  35|    0|    0|          233734|  12.35| NULL|       Q|2025-01-21|\n",
            "|        909|     3|   Assaf, Mr. Gerios|  male|  21|    0|    0|            2692|  7.225| NULL|       C|2025-01-21|\n",
            "|        910|     3|Ilmakangas, Miss....|female|  27|    1|    0|STON/O2. 3101270|  7.925| NULL|       S|2025-01-21|\n",
            "|        911|     3|\"Assaf Khalil, Mr...|female|  45|    0|    0|            2696|  7.225| NULL|       C|2025-01-21|\n",
            "+-----------+------+--------------------+------+----+-----+-----+----------------+-------+-----+--------+----------+\n",
            "only showing top 20 rows\n",
            "\n",
            "root\n",
            " |-- PassengerId: integer (nullable = true)\n",
            " |-- Pclass: integer (nullable = true)\n",
            " |-- Name: string (nullable = true)\n",
            " |-- Sex: string (nullable = true)\n",
            " |-- age: integer (nullable = true)\n",
            " |-- SibSp: integer (nullable = true)\n",
            " |-- Parch: integer (nullable = true)\n",
            " |-- Ticket: string (nullable = true)\n",
            " |-- Fare: double (nullable = true)\n",
            " |-- Cabin: string (nullable = true)\n",
            " |-- Embarked: string (nullable = true)\n",
            " |-- datalog: date (nullable = false)\n",
            " |-- dia_semana: string (nullable = false)\n",
            "\n",
            "+-----------+------+--------------------+------+----+-----+-----+----------------+-------+-----+--------+----------+----------+\n",
            "|PassengerId|Pclass|                Name|   Sex| age|SibSp|Parch|          Ticket|   Fare|Cabin|Embarked|   datalog|dia_semana|\n",
            "+-----------+------+--------------------+------+----+-----+-----+----------------+-------+-----+--------+----------+----------+\n",
            "|        892|     3|    Kelly, Mr. James|  male|  34|    0|    0|          330911| 7.8292| NULL|       Q|2025-01-21|   Tuesday|\n",
            "|        893|     3|Wilkes, Mrs. Jame...|female|  47|    1|    0|          363272|    7.0| NULL|       S|2025-01-21|   Tuesday|\n",
            "|        894|     2|Myles, Mr. Thomas...|  male|  62|    0|    0|          240276| 9.6875| NULL|       Q|2025-01-21|   Tuesday|\n",
            "|        895|     3|    Wirz, Mr. Albert|  male|  27|    0|    0|          315154| 8.6625| NULL|       S|2025-01-21|   Tuesday|\n",
            "|        896|     3|Hirvonen, Mrs. Al...|female|  22|    1|    1|         3101298|12.2875| NULL|       S|2025-01-21|   Tuesday|\n",
            "|        897|     3|Svensson, Mr. Joh...|  male|  14|    0|    0|            7538|  9.225| NULL|       S|2025-01-21|   Tuesday|\n",
            "|        898|     3|Connolly, Miss. Kate|female|  30|    0|    0|          330972| 7.6292| NULL|       Q|2025-01-21|   Tuesday|\n",
            "|        899|     2|Caldwell, Mr. Alb...|  male|  26|    1|    1|          248738|   29.0| NULL|       S|2025-01-21|   Tuesday|\n",
            "|        900|     3|Abrahim, Mrs. Jos...|female|  18|    0|    0|            2657| 7.2292| NULL|       C|2025-01-21|   Tuesday|\n",
            "|        901|     3|Davies, Mr. John ...|  male|  21|    2|    0|       A/4 48871|  24.15| NULL|       S|2025-01-21|   Tuesday|\n",
            "|        902|     3|    Ilieff, Mr. Ylio|  male|NULL|    0|    0|          349220| 7.8958| NULL|       S|2025-01-21|   Tuesday|\n",
            "|        903|     1|Jones, Mr. Charle...|  male|  46|    0|    0|             694|   26.0| NULL|       S|2025-01-21|   Tuesday|\n",
            "|        904|     1|Snyder, Mrs. John...|female|  23|    1|    0|           21228|82.2667|  B45|       S|2025-01-21|   Tuesday|\n",
            "|        905|     2|Howard, Mr. Benjamin|  male|  63|    1|    0|           24065|   26.0| NULL|       S|2025-01-21|   Tuesday|\n",
            "|        906|     1|Chaffee, Mrs. Her...|female|  47|    1|    0|     W.E.P. 5734| 61.175|  E31|       S|2025-01-21|   Tuesday|\n",
            "|        907|     2|del Carlo, Mrs. S...|female|  24|    1|    0|   SC/PARIS 2167|27.7208| NULL|       C|2025-01-21|   Tuesday|\n",
            "|        908|     2|   Keane, Mr. Daniel|  male|  35|    0|    0|          233734|  12.35| NULL|       Q|2025-01-21|   Tuesday|\n",
            "|        909|     3|   Assaf, Mr. Gerios|  male|  21|    0|    0|            2692|  7.225| NULL|       C|2025-01-21|   Tuesday|\n",
            "|        910|     3|Ilmakangas, Miss....|female|  27|    1|    0|STON/O2. 3101270|  7.925| NULL|       S|2025-01-21|   Tuesday|\n",
            "|        911|     3|\"Assaf Khalil, Mr...|female|  45|    0|    0|            2696|  7.225| NULL|       C|2025-01-21|   Tuesday|\n",
            "+-----------+------+--------------------+------+----+-----+-----+----------------+-------+-----+--------+----------+----------+\n",
            "only showing top 20 rows\n",
            "\n",
            "+-----------+------+--------------------+------+----+-----+-----+-----------+-------+-----+--------+----------+----------+----+\n",
            "|PassengerId|Pclass|                Name|   Sex| age|SibSp|Parch|     Ticket|   Fare|Cabin|Embarked|   datalog|dia_semana|rank|\n",
            "+-----------+------+--------------------+------+----+-----+-----+-----------+-------+-----+--------+----------+----------+----+\n",
            "|        983|     3|  Pedersen, Mr. Olaf|  male|NULL|    0|    0|     345498|  7.775| NULL|       S|2025-01-21|   Tuesday|   1|\n",
            "|       1055|     3|  Pearce, Mr. Ernest|  male|NULL|    0|    0|     343271|    7.0| NULL|       S|2025-01-21|   Tuesday|   2|\n",
            "|        985|     3|   Guest, Mr. Robert|  male|NULL|    0|    0|     376563|   8.05| NULL|       S|2025-01-21|   Tuesday|   3|\n",
            "|        946|     2|Mangiavacchi, Mr....|  male|NULL|    0|    0|SC/A.3 2861|15.5792| NULL|       C|2025-01-21|   Tuesday|   4|\n",
            "|        994|     3|  Foley, Mr. William|  male|NULL|    0|    0|     365235|   7.75| NULL|       Q|2025-01-21|   Tuesday|   5|\n",
            "|        921|     3|   Samaan, Mr. Elias|  male|NULL|    2|    0|       2662|21.6792| NULL|       C|2025-01-21|   Tuesday|   6|\n",
            "|        999|     3|    Ryan, Mr. Edward|  male|NULL|    0|    0|     383162|   7.75| NULL|       Q|2025-01-21|   Tuesday|   7|\n",
            "|        928|     3| Roth, Miss. Sarah A|female|NULL|    0|    0|     342712|   8.05| NULL|       S|2025-01-21|   Tuesday|   8|\n",
            "|       1000|     3|\"Willer, Mr. Aaro...|  male|NULL|    0|    0|       3410| 8.7125| NULL|       S|2025-01-21|   Tuesday|   9|\n",
            "|        902|     3|    Ilieff, Mr. Ylio|  male|NULL|    0|    0|     349220| 7.8958| NULL|       S|2025-01-21|   Tuesday|  10|\n",
            "|       1052|     3|  Smyth, Miss. Julia|female|NULL|    0|    0|     335432| 7.7333| NULL|       Q|2025-01-21|   Tuesday|  11|\n",
            "|       1043|     3|Matinoff, Mr. Nicola|  male|NULL|    0|    0|     349255| 7.8958| NULL|       C|2025-01-21|   Tuesday|  12|\n",
            "|       1003|     3|Shine, Miss. Elle...|female|NULL|    0|    0|     330968| 7.7792| NULL|       Q|2025-01-21|   Tuesday|  13|\n",
            "|        950|     3|Davison, Mr. Thom...|  male|NULL|    1|    0|     386525|   16.1| NULL|       S|2025-01-21|   Tuesday|  14|\n",
            "|       1008|     3|    Thomas, Mr. John|  male|NULL|    0|    0|       2681| 6.4375| NULL|       C|2025-01-21|   Tuesday|  15|\n",
            "|        968|     3|    Miles, Mr. Frank|  male|NULL|    0|    0|     359306|   8.05| NULL|       S|2025-01-21|   Tuesday|  16|\n",
            "|        975|     3|Demetri, Mr. Marinko|  male|NULL|    0|    0|     349238| 7.8958| NULL|       S|2025-01-21|   Tuesday|  17|\n",
            "|        976|     2|Lamb, Mr. John Jo...|  male|NULL|    0|    0|     240261|10.7083| NULL|       Q|2025-01-21|   Tuesday|  18|\n",
            "|        977|     3|  Khalil, Mr. Betros|  male|NULL|    1|    0|       2660|14.4542| NULL|       C|2025-01-21|   Tuesday|  19|\n",
            "|       1013|     3|   Kiernan, Mr. John|  male|NULL|    1|    0|     367227|   7.75| NULL|       Q|2025-01-21|   Tuesday|  20|\n",
            "+-----------+------+--------------------+------+----+-----+-----+-----------+-------+-----+--------+----------+----------+----+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    }
  ]
}