local_0_setup_workshop.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b2316e4b-357c-4946-b4a5-3a7579900543",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# setup darts\n",
    "!pip install -r requirements.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7c579e11-d731-4bfe-8db7-bcf8702d5a50",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Download data from data source to local\n",
    "import io\n",
    "import os\n",
    "import requests\n",
    "import shutil\n",
    "import zipfile\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "23f948de-7146-4c19-9664-978e473241b5",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_dir = \"data\"\n",
    "fc_dir = os.path.join(data_dir, \"forecasting\")\n",
    "anom_dir = os.path.join(data_dir, \"anomaly_detection\")\n",
    "pdm_dir = os.path.join(data_dir, \"pdm\")\n",
    "for dir_path in [data_dir, fc_dir, anom_dir, pdm_dir]:\n",
    "    if not os.path.exists(dir_path):\n",
    "        os.mkdir(dir_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5fbe36ac-27a1-4e96-92b9-ea08c1c6cced",
   "metadata": {},
   "source": [
    "## Data Download and Preprocessing"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8c926816-1dc7-4d7d-a57d-cb95f89ff94a",
   "metadata": {},
   "source": [
    "### Electricity Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fd04e4f3-85dd-4f8f-a590-181552189e10",
   "metadata": {},
   "outputs": [],
   "source": [
    "def download_weather_data():\n",
    "    \"\"\"Concatenate the yearly csv files into a single dataframe and reshape it\"\"\"\n",
    "    # download the csv from the url\n",
    "    base_url = \"https://data.stadt-zuerich.ch/dataset/ugz_meteodaten_stundenmittelwerte/download/\"\n",
    "    filenames = [f\"ugz_ogd_meteo_h1_{year}.csv\" for year in range(2015, 2023)]\n",
    "    df = pd.concat([pd.read_csv(base_url + fname) for fname in filenames])\n",
    "    # retain only one weather station\n",
    "    df = df.loc[df[\"Standort\"] == \"Zch_Stampfenbachstrasse\"]\n",
    "    # pivot the df to get all measurements as columns\n",
    "    df[\"param_name\"] = df[\"Parameter\"] + \" [\" + df[\"Einheit\"] + \"]\"\n",
    "    df = df.pivot(index=\"Datum\", columns=\"param_name\", values=\"Wert\")\n",
    "    # convert time index to from CET to UTC and extract the required time range\n",
    "    df.index = pd.DatetimeIndex(pd.to_datetime(df.index, utc=True)).tz_localize(\n",
    "        None\n",
    "    )\n",
    "    df = df.loc[\n",
    "        (pd.Timestamp(\"2015-01-01\") <= df.index)\n",
    "        & (df.index <= pd.Timestamp(\"2022-08-31\"))\n",
    "    ]\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "08c31893-394c-4b39-aa86-22b0cdac532d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def download_electricity_data():\n",
    "    # URL of the file\n",
    "    energy_url = \"https://data.stadt-zuerich.ch/dataset/ewz_stromabgabe_netzebenen_stadt_zuerich/download/ewz_stromabgabe_netzebenen_stadt_zuerich.csv\"\n",
    "    \n",
    "    file_path = os.path.join(fc_dir, energy_url.split(\"/\")[-1])\n",
    "    processed_path = file_path.replace(\".csv\", \"_proc.csv\")\n",
    "    \n",
    "    if not os.path.exists(processed_path):\n",
    "        if not os.path.exists(file_path):\n",
    "            # Send a GET request to download the zip file\n",
    "            response = requests.get(energy_url)\n",
    "            \n",
    "            # Check if the request was successful\n",
    "            if response.status_code == 200:\n",
    "                # Save the zip file to the local drive\n",
    "                with open(file_path, \"wb\") as file:\n",
    "                    file.write(response.content)\n",
    "                print(\"File downloaded successfully.\")\n",
    "            else:\n",
    "                raise ValueError(\"Failed to download.\")\n",
    "        print(\"Processing file..\")\n",
    "        df = pd.read_csv(file_path, index_col=0)\n",
    "        # convert time index\n",
    "        df.index = pd.DatetimeIndex(pd.to_datetime(df.index, utc=True)).tz_localize(None)\n",
    "        # extract pre-determined period\n",
    "        df = df.loc[(pd.Timestamp(\"2015-01-01\") <= df.index) & (df.index <= pd.Timestamp(\"2022-08-31\"))]\n",
    "        # download and preprocess the weather information\n",
    "        df_weather = download_weather_data()\n",
    "        # add weather data as additional features\n",
    "        df = pd.concat([df, df_weather], axis=1)\n",
    "        # interpolate weather data\n",
    "        df = df.interpolate()\n",
    "        # raining duration is given in minutes -> we divide by 4 from hourly to quarter-hourly records\n",
    "        df[\"RainDur [min]\"] = df[\"RainDur [min]\"] / 4\n",
    "    \n",
    "        # round Electricity cols to 4 decimals, other columns to 2 decimals\n",
    "        cols_precise = [\"Value_NE5\", \"Value_NE7\"]\n",
    "        df = df.round(decimals={col: (4 if col in cols_precise else 2) for col in df.columns})\n",
    "    \n",
    "        # export the dataset\n",
    "        df.index.name = \"Timestamp\"\n",
    "        df.to_csv(processed_path)\n",
    "        os.remove(file_path)\n",
    "        print(\"File successfully processed.\")\n",
    "    else:\n",
    "        print(\"File already downloaded.\")    "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3878a702-32bc-40fd-84ac-39622a922c0b",
   "metadata": {},
   "source": [
    "### Anomaly Detection - ECG"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "38452e99-7ae0-42e8-93cc-06abcde54e0f",
   "metadata": {},
   "outputs": [],
   "source": [
    "def download_ecg():\n",
    "    # URL of the zip file\n",
    "    zip_url = \"https://my.hidrive.com/api/sharelink/download?id=lmCmAjUP\"\n",
    "    \n",
    "    file_path = os.path.join(anom_dir, \"svdb.zip\")\n",
    "    unzipped_path = os.path.join(anom_dir, \"multivariate\")\n",
    "    processed_path = os.path.join(anom_dir, \"842.test.csv\")\n",
    "    if not os.path.exists(processed_path):\n",
    "        if not os.path.exists(file_path):\n",
    "            # Send a GET request to download the zip file\n",
    "            response = requests.get(zip_url)\n",
    "            \n",
    "            # Check if the request was successful\n",
    "            if response.status_code == 200:\n",
    "                # Save the zip file to the local drive\n",
    "                with open(file_path, \"wb\") as file:\n",
    "                    file.write(response.content)\n",
    "                print(\"Zip file downloaded successfully.\")\n",
    "            else:\n",
    "                print(\"Failed to download.\")\n",
    "        else:\n",
    "            print(\"Zip file already downloaded.\")    \n",
    "        \n",
    "        # Extract the zip file\n",
    "        with zipfile.ZipFile(file_path, 'r') as zip_ref:\n",
    "            zip_ref.extractall(anom_dir)\n",
    "        file_path\n",
    "        df = pd.read_csv(os.path.join(anom_dir, \"multivariate\", \"SVDB\", \"842.test.csv\"))\n",
    "        df.to_csv(processed_path, index=False)\n",
    "        shutil.rmtree(unzipped_path, ignore_errors=True)\n",
    "        os.remove(file_path)\n",
    "        print(\"Zip file extracted successfully.\")\n",
    "    else:\n",
    "        print(\"File already downloaded.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "64763f59-979c-4ac0-a4b6-ffcf841a9be7",
   "metadata": {},
   "source": [
    "### Predictive Maintenance - Wind Turbine Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eede08b1-6d32-4192-bd68-e596bb97b946",
   "metadata": {},
   "outputs": [],
   "source": [
    "FREQ = pd.tseries.frequencies.to_offset(\"10min\")\n",
    "\n",
    "def preprocess_data(df):\n",
    "    \"\"\"Preprocess the turbine signals\n",
    "    - drops duplicate time stamps (wrong times from daylight saving, no chance to identify correct times)\n",
    "    - resamples the data to a contiguous 10 Min DatetimeIndex\n",
    "    - flags missing dates with feature \"missin_date\"\n",
    "    - computes a generator/turbine on/off timer for each turbine.\n",
    "    \"\"\"\n",
    "    # drop duplicate time stamps (wrong times from daylight saving, no chance to\n",
    "    # identify correct times)\n",
    "    df = df.drop_duplicates([\"Turbine_ID\", \"Timestamp\"], keep=\"first\").reset_index(drop=True)\n",
    "    # remove time zone information\n",
    "    df[\"Timestamp\"] = pd.DatetimeIndex(df[\"Timestamp\"]).tz_localize(None)\n",
    "    df = df.sort_values(by=[\"Turbine_ID\", \"Timestamp\"])\n",
    "    df = df.set_index(\"Timestamp\")\n",
    "    df = df.groupby(\"Turbine_ID\").apply(compute_on_off_timer).reset_index(\"Turbine_ID\", drop=True)\n",
    "    return df.reset_index()\n",
    "\n",
    "\n",
    "def compute_on_off_timer(df):\n",
    "    \"\"\"Computes the time (number of 10 minute steps) since the generator last crossed the \n",
    "    1200 rotations per minute (RPM) mark.\n",
    "\n",
    "    This is a useful feature to let the model know how long that the turbine has been active/inactive,\n",
    "    and should ultimately improve generator temperature modelling.\n",
    "\n",
    "    - positive counter when the generator went from below 1200 RPM to above\n",
    "    - negative counter when the generator went from above 1200 RPM to below\n",
    "    - upper- and lower-bound by +/- 4 hours (4 * 6 (10 minutes steps) = 24)\n",
    "    \"\"\"\n",
    "    \n",
    "    df = df.ffill()\n",
    "    df = df.asfreq(FREQ)\n",
    "    df[\"missing_date\"] = df.isna().any(axis=1)\n",
    "    df = df.ffill()\n",
    "\n",
    "    # let's use a proxy for generator on if RPM >= 1200\n",
    "    df[\"state_on\"] = df[\"Gen_RPM_Avg\"] >= 1200\n",
    "    df[\"state_off\"] = df[\"Gen_RPM_Avg\"] < 1200\n",
    "\n",
    "    # count how many steps since last time the rotation speed went above threshold (and limit the counter)\n",
    "    b = (~df[\"state_on\"]).cumsum()\n",
    "    df[\"timer_on\"] = df.groupby(b)[\"state_on\"].cumsum()\n",
    "    df.loc[df[\"timer_on\"] > 4 * 6, \"timer_on\"] = 4 * 6\n",
    "\n",
    "    # count how many steps since last time the rotation speed went below threshold (and limit the counter)\n",
    "    b = (~df[\"state_off\"]).cumsum()\n",
    "    df[\"timer_off\"] = -(df.groupby(b)[\"state_off\"]).cumsum()\n",
    "    df.loc[df[\"timer_off\"] < -4 * 6, \"timer_off\"] = -4 * 6\n",
    "\n",
    "    # combine on and off timers\n",
    "    df[\"timer_on_off\"] = np.where(df[\"timer_on\"] > 0., df[\"timer_on\"], df[\"timer_off\"])\n",
    "\n",
    "    # remove the dedicated counters\n",
    "    return df.drop(columns=[\"timer_on\", \"timer_off\", \"state_off\"])\n",
    "\n",
    "\n",
    "def preprocess_anomalies(df):\n",
    "    df[\"Timestamp\"] = pd.DatetimeIndex(df[\"Timestamp\"]).tz_localize(None).round(FREQ.freqstr)\n",
    "    df[\"start\"] = df[\"Timestamp\"]\n",
    "    df[\"end\"] = df[\"start\"] + (6 * 24 - 1) * FREQ\n",
    "    df = df.sort_values(by=[\"Turbine_ID\", \"Timestamp\"]).reset_index(drop=True)\n",
    "    return df\n",
    "\n",
    "\n",
    "def download_wind_turbine():\n",
    "    # URL of the files\n",
    "    url_data = \"https://www.edp.com/sites/default/files/2023-04/Wind-Turbine-SCADA-signals-2016.xlsx\"\n",
    "    url_failures = \"https://www.edp.com/sites/default/files/2023-04/Historical-Failure-Logbook-2016.xlsx\"\n",
    "    \n",
    "    for url, preproc_fn in zip([url_data, url_failures], [preprocess_data, preprocess_anomalies]):\n",
    "        file_path = os.path.join(pdm_dir, url.split(\"/\")[-1])\n",
    "        csv_path = file_path.replace(\"xlsx\", \"csv\")\n",
    "        if not (os.path.exists(file_path) or os.path.exists(csv_path)):\n",
    "            # Send a GET request to download the zip file\n",
    "            response = requests.get(url)\n",
    "    \n",
    "            # Check if the request was successful\n",
    "            if response.status_code == 200:\n",
    "                # Save the zip file to the local drive\n",
    "                with open(file_path, \"wb\") as file:\n",
    "                    file.write(response.content)\n",
    "                print(\"File downloaded successfully.\")\n",
    "            else:\n",
    "                print(\"Failed to download.\")\n",
    "        else:\n",
    "            print(\"File already downloaded.\")\n",
    "    \n",
    "        # Extract the zip file\n",
    "        if os.path.exists(file_path):\n",
    "            print(\"Converting to csv..\")\n",
    "            df = pd.read_excel(file_path)\n",
    "            df = preproc_fn(df)\n",
    "            df.to_csv(csv_path, index=False)\n",
    "            print(\"Successfully converted to csv.\")\n",
    "            os.remove(file_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3b8c204a-f81c-4c3f-ac01-b82a23f045f9",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Downloading Electricity Data..\")\n",
    "download_electricity_data()\n",
    "\n",
    "print(\"\\nDownloading ECG Data..\")\n",
    "download_ecg()\n",
    "\n",
    "print(\"\\nDownloading Wind Turbine Data..\")\n",
    "download_wind_turbine()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {},
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}