data_cleaning_code.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
        "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
        "execution": {
          "iopub.execute_input": "2020-11-24T15:33:44.617819Z",
          "iopub.status.busy": "2020-11-24T15:33:44.616734Z",
          "iopub.status.idle": "2020-11-24T15:33:48.545917Z",
          "shell.execute_reply": "2020-11-24T15:33:48.545192Z"
        },
        "papermill": {
          "duration": 3.947576,
          "end_time": "2020-11-24T15:33:48.546057",
          "exception": false,
          "start_time": "2020-11-24T15:33:44.598481",
          "status": "completed"
        },
        "tags": [],
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4HjbX5J_IiMy",
        "outputId": "d3c0c821-6a1f-4dce-8375-6e07f577b2b0"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['mbarara63702.csv',\n",
              " 'kabale63726.csv',\n",
              " 'kasese63674.csv',\n",
              " 'arua63602.csv',\n",
              " 'gulu63630.csv',\n",
              " 'masindi63654.csv',\n",
              " 'tororo63684.csv',\n",
              " 'jinja63682.csv',\n",
              " 'entebbe63602.csv',\n",
              " 'soroti63658.csv',\n",
              " 'kampala63680.csv']"
            ]
          },
          "metadata": {},
          "execution_count": 29
        }
      ],
      "source": [
        "# importing libraries and data\n",
        "import numpy as np\n",
        "import os\n",
        "import pandas as pd\n",
        "#read the path\n",
        "file_path=(\"/content/uganda\")\n",
        "#list all the files from the directory\n",
        "file_list = os.listdir(file_path)\n",
        "file_list"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "file_list = os.listdir(file_path)\n",
        "\n",
        "dfs = [] # collect the files in an array\n",
        "\n",
        "for file in file_list:\n",
        "    file_full_path = os.path.join(file_path, file)\n",
        "    df_temp = pd.read_csv(file_full_path)\n",
        "    dfs.append(df_temp)\n",
        "\n",
        "df_append = pd.concat(dfs, ignore_index=True)\n",
        "df_append.head(5)"
      ],
      "metadata": {
        "id": "542yCpRCMWOK",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 293
        },
        "outputId": "0398f1bb-3d03-45af-958c-74574adf9907"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "               Date    TC   TdC    Hr ddd  ffkmh  Precmm  Prechrs   Nt   Nh  \\\n",
              "0  31/12/2020 15:00  24.0  18.0  69.0  15   14.8     0.0        3  6.0  3.0   \n",
              "1  31/12/2020 06:00  20.0  16.9  82.0   5   11.1     0.0       24  6.0  1.0   \n",
              "2  30/12/2020 15:00  27.9  15.2  46.0   1   11.1     0.0        3  7.0  2.0   \n",
              "3  30/12/2020 12:00  28.2  15.2  45.0   4   18.5     0.0        6  6.0  3.0   \n",
              "4  30/12/2020 09:00  26.4  16.7  55.0  16    9.3     0.0        3  6.0  6.0   \n",
              "\n",
              "   HKm  Viskm  Prec_rate  Classification  P0hPa  \n",
              "0  0.6   20.0        0.0               2    NaN  \n",
              "1  0.6   20.0        0.0               2    NaN  \n",
              "2  0.6   30.0        0.0               2    NaN  \n",
              "3  0.6   30.0        0.0               2    NaN  \n",
              "4  0.6   20.0        0.0               2    NaN  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-340b8dea-e66d-44df-afbb-c9f7e6a25ee9\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Date</th>\n",
              "      <th>TC</th>\n",
              "      <th>TdC</th>\n",
              "      <th>Hr</th>\n",
              "      <th>ddd</th>\n",
              "      <th>ffkmh</th>\n",
              "      <th>Precmm</th>\n",
              "      <th>Prechrs</th>\n",
              "      <th>Nt</th>\n",
              "      <th>Nh</th>\n",
              "      <th>HKm</th>\n",
              "      <th>Viskm</th>\n",
              "      <th>Prec_rate</th>\n",
              "      <th>Classification</th>\n",
              "      <th>P0hPa</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>31/12/2020 15:00</td>\n",
              "      <td>24.0</td>\n",
              "      <td>18.0</td>\n",
              "      <td>69.0</td>\n",
              "      <td>15</td>\n",
              "      <td>14.8</td>\n",
              "      <td>0.0</td>\n",
              "      <td>3</td>\n",
              "      <td>6.0</td>\n",
              "      <td>3.0</td>\n",
              "      <td>0.6</td>\n",
              "      <td>20.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>2</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>31/12/2020 06:00</td>\n",
              "      <td>20.0</td>\n",
              "      <td>16.9</td>\n",
              "      <td>82.0</td>\n",
              "      <td>5</td>\n",
              "      <td>11.1</td>\n",
              "      <td>0.0</td>\n",
              "      <td>24</td>\n",
              "      <td>6.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.6</td>\n",
              "      <td>20.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>2</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>30/12/2020 15:00</td>\n",
              "      <td>27.9</td>\n",
              "      <td>15.2</td>\n",
              "      <td>46.0</td>\n",
              "      <td>1</td>\n",
              "      <td>11.1</td>\n",
              "      <td>0.0</td>\n",
              "      <td>3</td>\n",
              "      <td>7.0</td>\n",
              "      <td>2.0</td>\n",
              "      <td>0.6</td>\n",
              "      <td>30.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>2</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>30/12/2020 12:00</td>\n",
              "      <td>28.2</td>\n",
              "      <td>15.2</td>\n",
              "      <td>45.0</td>\n",
              "      <td>4</td>\n",
              "      <td>18.5</td>\n",
              "      <td>0.0</td>\n",
              "      <td>6</td>\n",
              "      <td>6.0</td>\n",
              "      <td>3.0</td>\n",
              "      <td>0.6</td>\n",
              "      <td>30.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>2</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>30/12/2020 09:00</td>\n",
              "      <td>26.4</td>\n",
              "      <td>16.7</td>\n",
              "      <td>55.0</td>\n",
              "      <td>16</td>\n",
              "      <td>9.3</td>\n",
              "      <td>0.0</td>\n",
              "      <td>3</td>\n",
              "      <td>6.0</td>\n",
              "      <td>6.0</td>\n",
              "      <td>0.6</td>\n",
              "      <td>20.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>2</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-340b8dea-e66d-44df-afbb-c9f7e6a25ee9')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-340b8dea-e66d-44df-afbb-c9f7e6a25ee9 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-340b8dea-e66d-44df-afbb-c9f7e6a25ee9');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-9d79d4ed-e449-4c3e-936c-dc3ec2c8e21a\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-9d79d4ed-e449-4c3e-936c-dc3ec2c8e21a')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-9d79d4ed-e449-4c3e-936c-dc3ec2c8e21a button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "summary": "{\n  \"name\": \"# print(\\\"Number of rows:\\\", num_rows)\",\n  \"rows\": 5,\n  \"fields\": [\n    {\n      \"column\": \"Date\",\n      \"properties\": {\n        \"dtype\": \"object\",\n        \"num_unique_values\": 5,\n        \"samples\": [\n          \"31/12/2020 06:00\",\n          \"30/12/2020 09:00\",\n          \"30/12/2020 15:00\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"TC\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 3.397057550292605,\n        \"min\": 20.0,\n        \"max\": 28.2,\n        \"num_unique_values\": 5,\n        \"samples\": [\n          20.0,\n          26.4,\n          27.9\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"TdC\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 1.202081528017131,\n        \"min\": 15.2,\n        \"max\": 18.0,\n        \"num_unique_values\": 4,\n        \"samples\": [\n          16.9,\n          16.7,\n          18.0\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Hr\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 15.883954167649817,\n        \"min\": 45.0,\n        \"max\": 82.0,\n        \"num_unique_values\": 5,\n        \"samples\": [\n          82.0,\n          55.0,\n          46.0\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"ddd\",\n      \"properties\": {\n        \"dtype\": \"date\",\n        \"min\": 1,\n        \"max\": 16,\n        \"num_unique_values\": 5,\n        \"samples\": [\n          5,\n          16,\n          1\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"ffkmh\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 3.687546609874918,\n        \"min\": 9.3,\n        \"max\": 18.5,\n        \"num_unique_values\": 4,\n        \"samples\": [\n          11.1,\n          9.3,\n          14.8\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Precmm\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.0,\n        \"min\": 0.0,\n        \"max\": 0.0,\n        \"num_unique_values\": 1,\n        \"samples\": [\n          0.0\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Prechrs\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 9,\n        \"min\": 3,\n        \"max\": 24,\n        \"num_unique_values\": 3,\n        \"samples\": [\n          3\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Nt\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.4472135954999579,\n        \"min\": 6.0,\n        \"max\": 7.0,\n        \"num_unique_values\": 2,\n        \"samples\": [\n          7.0\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Nh\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 1.8708286933869707,\n        \"min\": 1.0,\n        \"max\": 6.0,\n        \"num_unique_values\": 4,\n        \"samples\": [\n          1.0\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"HKm\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.0,\n        \"min\": 0.6,\n        \"max\": 0.6,\n        \"num_unique_values\": 1,\n        \"samples\": [\n          0.6\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Viskm\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 5.477225575051661,\n        \"min\": 20.0,\n        \"max\": 30.0,\n        \"num_unique_values\": 2,\n        \"samples\": [\n          30.0\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Prec_rate\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.0,\n        \"min\": 0.0,\n        \"max\": 0.0,\n        \"num_unique_values\": 1,\n        \"samples\": [\n          0.0\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Classification\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0,\n        \"min\": 2,\n        \"max\": 2,\n        \"num_unique_values\": 1,\n        \"samples\": [\n          2\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"P0hPa\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": null,\n        \"min\": null,\n        \"max\": null,\n        \"num_unique_values\": 0,\n        \"samples\": [],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 30
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "OvBjGv6XIiM5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 293
        },
        "outputId": "b409da8e-d479-48f5-c0b0-be9eef74dfcf"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                   Date    TC   TdC    Hr ddd  ffkmh  Precmm  Prechrs   Nt  \\\n",
              "2084   11/03/2022 12:00  30.5  14.2  37.0   7   18.5     0.0        6  5.0   \n",
              "17479  23/06/2022 15:00  24.5  18.2  68.0   9   18.5     0.0        3  7.0   \n",
              "5227   21/06/2021 15:00  29.7  16.8  46.0   5    3.7     0.0        3  2.0   \n",
              "15354  10/10/2020 12:00  26.3  17.4  58.0   1    9.3     0.0        6  3.0   \n",
              "19606  16/08/2022 15:00  23.0  19.6  81.0   9   11.1     0.0        3  7.0   \n",
              "\n",
              "        Nh  HKm  Viskm  Prec_rate  Classification  P0hPa  \n",
              "2084   5.0  0.6   30.0        0.0               2    NaN  \n",
              "17479  1.0  0.6   15.0        0.0               2  884.4  \n",
              "5227   1.0  0.6   10.0        0.0               1  904.1  \n",
              "15354  2.0  0.6   10.0        0.0               2  885.3  \n",
              "19606  2.0  0.3   20.0        0.0               2  887.9  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-bb8f632e-9bd2-4d54-ac79-1074b31eaac5\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Date</th>\n",
              "      <th>TC</th>\n",
              "      <th>TdC</th>\n",
              "      <th>Hr</th>\n",
              "      <th>ddd</th>\n",
              "      <th>ffkmh</th>\n",
              "      <th>Precmm</th>\n",
              "      <th>Prechrs</th>\n",
              "      <th>Nt</th>\n",
              "      <th>Nh</th>\n",
              "      <th>HKm</th>\n",
              "      <th>Viskm</th>\n",
              "      <th>Prec_rate</th>\n",
              "      <th>Classification</th>\n",
              "      <th>P0hPa</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>2084</th>\n",
              "      <td>11/03/2022 12:00</td>\n",
              "      <td>30.5</td>\n",
              "      <td>14.2</td>\n",
              "      <td>37.0</td>\n",
              "      <td>7</td>\n",
              "      <td>18.5</td>\n",
              "      <td>0.0</td>\n",
              "      <td>6</td>\n",
              "      <td>5.0</td>\n",
              "      <td>5.0</td>\n",
              "      <td>0.6</td>\n",
              "      <td>30.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>2</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17479</th>\n",
              "      <td>23/06/2022 15:00</td>\n",
              "      <td>24.5</td>\n",
              "      <td>18.2</td>\n",
              "      <td>68.0</td>\n",
              "      <td>9</td>\n",
              "      <td>18.5</td>\n",
              "      <td>0.0</td>\n",
              "      <td>3</td>\n",
              "      <td>7.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.6</td>\n",
              "      <td>15.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>2</td>\n",
              "      <td>884.4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5227</th>\n",
              "      <td>21/06/2021 15:00</td>\n",
              "      <td>29.7</td>\n",
              "      <td>16.8</td>\n",
              "      <td>46.0</td>\n",
              "      <td>5</td>\n",
              "      <td>3.7</td>\n",
              "      <td>0.0</td>\n",
              "      <td>3</td>\n",
              "      <td>2.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.6</td>\n",
              "      <td>10.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1</td>\n",
              "      <td>904.1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15354</th>\n",
              "      <td>10/10/2020 12:00</td>\n",
              "      <td>26.3</td>\n",
              "      <td>17.4</td>\n",
              "      <td>58.0</td>\n",
              "      <td>1</td>\n",
              "      <td>9.3</td>\n",
              "      <td>0.0</td>\n",
              "      <td>6</td>\n",
              "      <td>3.0</td>\n",
              "      <td>2.0</td>\n",
              "      <td>0.6</td>\n",
              "      <td>10.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>2</td>\n",
              "      <td>885.3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19606</th>\n",
              "      <td>16/08/2022 15:00</td>\n",
              "      <td>23.0</td>\n",
              "      <td>19.6</td>\n",
              "      <td>81.0</td>\n",
              "      <td>9</td>\n",
              "      <td>11.1</td>\n",
              "      <td>0.0</td>\n",
              "      <td>3</td>\n",
              "      <td>7.0</td>\n",
              "      <td>2.0</td>\n",
              "      <td>0.3</td>\n",
              "      <td>20.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>2</td>\n",
              "      <td>887.9</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-bb8f632e-9bd2-4d54-ac79-1074b31eaac5')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-bb8f632e-9bd2-4d54-ac79-1074b31eaac5 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-bb8f632e-9bd2-4d54-ac79-1074b31eaac5');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-df7f6ab0-126e-49e0-960a-0308fd245ebd\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-df7f6ab0-126e-49e0-960a-0308fd245ebd')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-df7f6ab0-126e-49e0-960a-0308fd245ebd button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "df_append_shuffled",
              "summary": "{\n  \"name\": \"df_append_shuffled\",\n  \"rows\": 22264,\n  \"fields\": [\n    {\n      \"column\": \"Date\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 4923,\n        \"samples\": [\n          \"25/01/2022 09:00\",\n          \"19/05/2022 18:00\",\n          \"11/08/2021 12:00\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"TC\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 3.7243313662273883,\n        \"min\": 10.0,\n        \"max\": 36.0,\n        \"num_unique_values\": 267,\n        \"samples\": [\n          30.6,\n          27.0,\n          34.6\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"TdC\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 2.4251835634960535,\n        \"min\": 0.8,\n        \"max\": 39.8,\n        \"num_unique_values\": 243,\n        \"samples\": [\n          18.4,\n          17.6,\n          7.6\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Hr\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 17.05313192534546,\n        \"min\": 13.0,\n        \"max\": 184.0,\n        \"num_unique_values\": 121,\n        \"samples\": [\n          57.0,\n          80.0,\n          81.0\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"ddd\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 38,\n        \"samples\": [\n          6,\n          \"t3B\",\n          3\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"ffkmh\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 9.975007052312568,\n        \"min\": 0.0,\n        \"max\": 324.2,\n        \"num_unique_values\": 77,\n        \"samples\": [\n          14.8,\n          55.6,\n          1.9\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Precmm\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 7.938601785236579,\n        \"min\": 0.0,\n        \"max\": 461.0,\n        \"num_unique_values\": 130,\n        \"samples\": [\n          30.0,\n          0.889646465,\n          0.901585603\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Prechrs\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 8,\n        \"min\": 1,\n        \"max\": 24,\n        \"num_unique_values\": 7,\n        \"samples\": [\n          6,\n          3,\n          2\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Nt\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 1.96341136275825,\n        \"min\": 0.0,\n        \"max\": 9.0,\n        \"num_unique_values\": 14,\n        \"samples\": [\n          5.143307087,\n          4.212121212,\n          5.0\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Nh\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 1.8576754794224783,\n        \"min\": 0.0,\n        \"max\": 9.0,\n        \"num_unique_values\": 40,\n        \"samples\": [\n          3.668439716,\n          3.099307159,\n          4.385542169\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"HKm\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.525367682835202,\n        \"min\": 0.0,\n        \"max\": 2.5,\n        \"num_unique_values\": 36,\n        \"samples\": [\n          0.823999837,\n          0.2,\n          0.630859375\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Viskm\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 8.37422084487806,\n        \"min\": 0.0,\n        \"max\": 50.0,\n        \"num_unique_values\": 64,\n        \"samples\": [\n          20.40441324,\n          18.05627706,\n          30.0\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Prec_rate\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 3.1434029304572007,\n        \"min\": 0.0,\n        \"max\": 461.0,\n        \"num_unique_values\": 176,\n        \"samples\": [\n          0.416666667,\n          1.083333333,\n          2.833333333\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Classification\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0,\n        \"min\": 0,\n        \"max\": 3,\n        \"num_unique_values\": 4,\n        \"samples\": [\n          1,\n          3,\n          2\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"P0hPa\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 10.61841872006337,\n        \"min\": 180.5,\n        \"max\": 1007.1,\n        \"num_unique_values\": 331,\n        \"samples\": [\n          886.0,\n          910.7,\n          883.1\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 33
        }
      ],
      "source": [
        "# shuffle data\n",
        "df_append_shuffled = df_append.sample(frac=1, random_state=42)\n",
        "df_append_shuffled.head(5)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install datasets\n",
        "from datasets import Dataset\n",
        "df_append_shuffled.to_csv('/content/sample_data/dataset.csv', index=False)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SUSiAWOFcJWh",
        "outputId": "1c664892-e73b-4177-f2ff-19e2127d03fb"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: datasets in /usr/local/lib/python3.10/dist-packages (2.19.0)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from datasets) (3.13.4)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from datasets) (1.25.2)\n",
            "Requirement already satisfied: pyarrow>=12.0.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (14.0.2)\n",
            "Requirement already satisfied: pyarrow-hotfix in /usr/local/lib/python3.10/dist-packages (from datasets) (0.6)\n",
            "Requirement already satisfied: dill<0.3.9,>=0.3.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (0.3.8)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from datasets) (2.0.3)\n",
            "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (2.31.0)\n",
            "Requirement already satisfied: tqdm>=4.62.1 in /usr/local/lib/python3.10/dist-packages (from datasets) (4.66.2)\n",
            "Requirement already satisfied: xxhash in /usr/local/lib/python3.10/dist-packages (from datasets) (3.4.1)\n",
            "Requirement already satisfied: multiprocess in /usr/local/lib/python3.10/dist-packages (from datasets) (0.70.16)\n",
            "Requirement already satisfied: fsspec[http]<=2024.3.1,>=2023.1.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (2023.6.0)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from datasets) (3.9.5)\n",
            "Requirement already satisfied: huggingface-hub>=0.21.2 in /usr/local/lib/python3.10/dist-packages (from datasets) (0.22.2)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from datasets) (24.0)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from datasets) (6.0.1)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.3.1)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (23.2.0)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.4.1)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (6.0.5)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.9.4)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (4.0.3)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.21.2->datasets) (4.11.0)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets) (3.7)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets) (2024.2.2)\n",
            "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2023.4)\n",
            "Requirement already satisfied: tzdata>=2022.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2024.1)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.2->pandas->datasets) (1.16.0)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#read the path\n",
        "data2 = pd.read_csv(\"/content/sample_data/dataset.csv\")\n",
        "\n",
        "data2['P0hPa'] = data2['P0hPa'].fillna(data2['P0hPa'].mean())\n",
        "# print(data2['P0hPa'].mean()) #a line to print the mean value\n",
        "data2.to_csv('/content/sample_data/dataset_after_full_preprocessig.csv', index=False) #export the dataset file and assign a name and location"
      ],
      "metadata": {
        "id": "mmGq-u0fhCL5"
      },
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "celltoolbar": "Raw Cell Format",
    "kernelspec": {
      "display_name": "Python 3 (ipykernel)",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.10.9"
    },
    "papermill": {
      "duration": 55.320944,
      "end_time": "2020-11-24T15:34:35.258832",
      "environment_variables": {},
      "exception": null,
      "input_path": "__notebook__.ipynb",
      "output_path": "__notebook__.ipynb",
      "parameters": {},
      "start_time": "2020-11-24T15:33:39.937888",
      "version": "2.1.0"
    },
    "colab": {
      "provenance": []
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}