CTV_datacollector.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import logging\n",
    "import pandas as pd\n",
    "import lasio as las\n",
    "import numpy as np\n",
    "import os\n",
    "import time\n",
    "from zipfile import ZipFile\n",
    "import regex as re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# required curves/channels\n",
    "\n",
    "required_channels = ['TEMP_DNI', 'VIBX', 'VIBY']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#counter intialization\n",
    "scan_success = 0\n",
    "scan_failed = 0\n",
    "\n",
    "#creating the list for the files counting\n",
    "scanned_files = list()\n",
    "failed_files = list()\n",
    "\n",
    "#logging file name\n",
    "file_name = 'log_file_list.txt'\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "directory_list = [r'****',\n",
    "            r'****'] #just in case if there are several folder where the data can be fetched\n",
    "\n",
    "output_directory = r'****' #the dirctory where the logs and outputs are collected\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# reading the las file\n",
    "\n",
    "def read_las_df(directory:str, run:int, filename:str):\n",
    "    \n",
    "    las_file = las.read(os.path.join(directory,filename))\n",
    "    las_df = pd.DataFrame(\n",
    "                data = las_file.data[1:],\n",
    "                columns = las_file.keys()\n",
    "            )\n",
    "    \n",
    "    las_df['WELL'] = las_file.well['WELL'].value\n",
    "    las_df['RUN'] = run\n",
    "\n",
    "    return las_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def data_processing(data):\n",
    "    \n",
    "    # the function process the data from las files\n",
    "    # also checks whether some curves are within the file\n",
    "    # returns list of predefined variables for further concatination into DataFrame\n",
    "    \n",
    "    #data processing\n",
    "    data['TIME'] = pd.to_datetime(data['TIME'], unit='D') #converting to datatime\n",
    "    data['TIME_DIFF'] = data['TIME'].diff() # computing timestep\n",
    "    well_name = data['WELL'][1]\n",
    "    run = data['RUN'][1]\n",
    "    median_vib = np.nan\n",
    "    max_vib = np.nan\n",
    "    energy = np.nan\n",
    "    n2_rate_av = np.nan\n",
    "    n2_rate_tot = np.nan\n",
    "    n2_rate_max = np.nan\n",
    "    median_vibz = np.nan\n",
    "    max_vibz = np.nan\n",
    "    \n",
    "    #check point if the curves are within the las file\n",
    "    for item in required_channels:\n",
    "        \n",
    "        if not(item in data.columns):\n",
    "            \n",
    "            log.warning(f'The channel {item} has not been found in {well_name} well run {run}!')\n",
    "    \n",
    "    #computing VIB_LAT, max/median(VIB_LAT), energy, time aobve 10g/15g\n",
    "    \n",
    "    if ('VIBX' in data.columns) and ('VIBY' in data.columns):\n",
    "        \n",
    "        data['VIB_LAT'] = np.sqrt(data['VIBX']**2 + data['VIBY']**2)\n",
    "        mask_10g = data['VIB_LAT'] >= 10\n",
    "        mask_15g = data['VIB_LAT'] >= 15\n",
    "        time_10g = data['TIME_DIFF'][mask_10g].sum().seconds\n",
    "        time_15g = data['TIME_DIFF'][mask_15g].sum().seconds\n",
    "        median_vib = data['VIB_LAT'].median()\n",
    "        max_vib = data['VIB_LAT'].max()\n",
    "        energy = (data['VIB_LAT'] * data['TIME_DIFF'].dt.microseconds).sum()/1000000\n",
    "\n",
    "    else:\n",
    "        \n",
    "        time_10g = np.nan\n",
    "        time_15g = np.nan\n",
    "        energy = np.nan\n",
    "    \n",
    "    #TEMP_DNI validation\n",
    "    \n",
    "    if ('TEMP_DNI' in data.columns):\n",
    "        max_temp = data['TEMP_DNI'].max()\n",
    "    \n",
    "    else:\n",
    "        max_temp = np.nan\n",
    "    \n",
    "    # N2 pumping data processing and validation\n",
    "    \n",
    "    if ('N2_RATE' in data.columns):\n",
    "        n2_rate_max = data['N2_RATE'].max()\n",
    "        n2_rate_av = data['N2_RATE'].mean()\n",
    "        n2_rate_tot = ((data['N2_RATE']/60) * data['TIME_DIFF'].dt.microseconds).sum()/1000000\n",
    "    \n",
    "    if ('VIBZ' in data.columns):\n",
    "        median_vibz = data['VIBZ'].median()\n",
    "        max_vibz = data['VIBZ'].max()\n",
    "\n",
    "                \n",
    "    return [well_name, run, time_10g, time_15g, median_vib, max_vib, median_vibz, max_vibz, max_temp, energy, n2_rate_av, n2_rate_tot, n2_rate_max]\n",
    "        \n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_run(text:str):\n",
    "    \n",
    "    #The function gets name of the file as string \n",
    "    #--> returns run number and whether the las file is timebased\n",
    "\n",
    "    text = str.lower(text)\n",
    "    text = text.replace('-', '_')\n",
    "    text = text.replace(' ', '_')\n",
    "    text = text.replace('#', '_')\n",
    "    text = text.replace('.', '_')\n",
    "    \n",
    "    text_list = text.split(sep='_')\n",
    "    Is_time = False #flag whether the file has time format\n",
    "    run = '0'\n",
    "    \n",
    "    for item in text_list:\n",
    "        \n",
    "        if 'run' in item:\n",
    "            if 'run' == item:\n",
    "                run=(item+text_list[text_list.index(item)+1]).upper()\n",
    "            else:\n",
    "                run=item.upper()\n",
    "        \n",
    "        # 'dml' is an indication of timebased las file\n",
    "        \n",
    "        if 'dml' in item:\n",
    "            Is_time = True\n",
    "    \n",
    "    run = re.findall(r'\\d+', run)\n",
    "    \n",
    "    return int(run[0]), Is_time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def wells_to_file(directory:str, well_list:list):\n",
    "    \n",
    "    #the functions saves already scanned wells list to the log file\n",
    "    \n",
    "    well_list = list(set(well_list))\n",
    "    \n",
    "    log_path = os.path.join(directory, file_name)\n",
    "\n",
    "    if os.path.exists(log_path):\n",
    "        log_file = open(log_path, 'a')\n",
    "\n",
    "        for item in well_list:\n",
    "            log_file.write(item + '\\n')\n",
    "\n",
    "        log_file.close()\n",
    "        \n",
    "    else:\n",
    "        log_file = open(log_path, 'w')\n",
    "        \n",
    "        for item in well_list:\n",
    "            log_file.write(item + '\\n')\n",
    "            \n",
    "        log_file.close()\n",
    "        \n",
    "    pass\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_log(directory:str):\n",
    "    \n",
    "    #the function reads log file for already scanned wells \n",
    "    #--> returns list of the wells\n",
    "    \n",
    "    log_path = os.path.join(directory, file_name)\n",
    "    well_list = list()\n",
    "    \n",
    "    if os.path.exists(log_path):\n",
    "        log_file = open(log_path, 'r')\n",
    "        well_list = log_file.read().split('\\n')\n",
    "        log_file.close()\n",
    "\n",
    "    return well_list"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Main Loop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "total_las_files = 0 # Total las file counter for all folders\n",
    "output_list = list() # the list where we are going to collect results for data processing\n",
    "\n",
    "loop_st_time = time.time() #recording the start time of the script\n",
    "\n",
    "#logging activation\n",
    "log = logging.getLogger('my-logger')\n",
    "logging.basicConfig(level=logging.WARNING)\n",
    "logging.basicConfig(format='%(asctime)s - %(message)s')\n",
    "\n",
    "# scanning the las files in the directories\n",
    "\n",
    "for directory in directory_list:\n",
    "\n",
    "    print('Start scanning the folder...', directory)\n",
    "\n",
    "    \n",
    "    las_file_counter = 0 # Las file counter for scanning folder\n",
    "\n",
    "    for root, dirs, files in os.walk(directory):\n",
    "\n",
    "        for file in files:\n",
    "\n",
    "            if file.endswith('.las'):\n",
    "\n",
    "                if not(file in read_log(output_directory)):\n",
    "\n",
    "                    if get_run(file)[1]:\n",
    "\n",
    "                        las_file_counter +=1\n",
    "                        total_las_files +=1\n",
    "            \n",
    "            if file.endswith('.zip'):\n",
    "                \n",
    "                try:\n",
    "                    with ZipFile(os.path.join(root, file), 'r') as zip_file:\n",
    "                    \n",
    "                        files_in_zip = zip_file.namelist()\n",
    "                    \n",
    "                        for element in files_in_zip:\n",
    "                        \n",
    "                            if 'las' in element.split('.'):\n",
    "                                \n",
    "                                if not(element in read_log(output_directory)):\n",
    "                                    \n",
    "                                    if get_run(element)[1]:\n",
    "                                        \n",
    "                                        las_file_counter +=1\n",
    "                                        total_las_files +=1\n",
    "                \n",
    "                except:\n",
    "                    log.warning(f'Error in reading file...{file}')\n",
    "\n",
    "\n",
    "    print('Total Las files: ', las_file_counter)\n",
    "    las_file_counter = 0\n",
    "    \n",
    "#main loop\n",
    "\n",
    "    for root, dirs, files in os.walk(directory):\n",
    "\n",
    "        for file in files:\n",
    "            \n",
    "            #reading of nonarchived las files\n",
    "            if file.endswith('.las'):\n",
    "\n",
    "                if not(file in read_log(output_directory)): #check if the file was scanned before\n",
    "\n",
    "                    if get_run(file)[1]:\n",
    "\n",
    "                        try:\n",
    "                            print('Reading file... {}'.format(file))\n",
    "                            data = read_las_df(root, get_run(file)[0], file)\n",
    "                            print('Processing the file... {}'.format(file))\n",
    "                            output_list.append(data_processing(data))\n",
    "                            scan_success +=1\n",
    "                            print('Current Progress: ', scan_success+scan_failed,'/', total_las_files)\n",
    "                            scanned_files.append(file)\n",
    "                        except:\n",
    "                            log.warning(f'Error in reading file...{file}')\n",
    "                            scan_failed +=1\n",
    "                            print('Current Progress: ', scan_success+scan_failed,'/', total_las_files)\n",
    "                            failed_files.append(file)\n",
    "            \n",
    "            \n",
    "            #reading of archived las files\n",
    "            if file.endswith('.zip'):\n",
    "                \n",
    "                try:\n",
    "                    with ZipFile(os.path.join(root, file), 'r') as zip_file:\n",
    "                    \n",
    "                        files_in_zip = zip_file.namelist()\n",
    "                    \n",
    "                        for element in files_in_zip:\n",
    "                        \n",
    "                            if 'las' in element.split('.'):\n",
    "                                \n",
    "                                if not(element in read_log(output_directory)): #check if the file was scanned before\n",
    "\n",
    "                                    if get_run(element)[1]:\n",
    "\n",
    "                                        try:\n",
    "                                            print('Reading file... {}'.format(element))\n",
    "                                            data = read_las_df(root, get_run(file)[0], element)\n",
    "                                            print('Processing the file... {}'.format(element))\n",
    "                                            output_list.append(data_processing(data))\n",
    "                                            scan_success +=1\n",
    "                                            print('Current Progress: ', scan_success+scan_failed,'/', total_las_files)\n",
    "                                            scanned_files.append(element)\n",
    "                                        except:\n",
    "                                            log.warning(f'Error in reading file...{element}')\n",
    "                                            scan_failed +=1\n",
    "                                            print('Current Progress: ', scan_success+scan_failed,'/', total_las_files)\n",
    "                                            failed_files.append(element)\n",
    "\n",
    "                                \n",
    "                \n",
    "                except:\n",
    "                    log.warning(f'Error in reading file...{file}')\n",
    "                \n",
    "\n",
    "\n",
    "    #end of main loop\n",
    "\n",
    "#calculating the total time\n",
    "\n",
    "loop_end_time = time.time()\n",
    "\n",
    "#displaying results of the scanning\n",
    "\n",
    "print('Total time spent for scanning and processing...', round(loop_end_time-loop_st_time),'sec')\n",
    "print('Total successfully scanned las files: ', scan_success,'/', total_las_files)\n",
    "print('Total failed scans: ', scan_failed)\n",
    "print('Failed files list: ',failed_files)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#reading the previous scan results for updating\n",
    "\n",
    "previous_results = pd.read_excel(os.path.join(output_directory,'CTD_output.xlsx'), index_col=0)\n",
    "\n",
    "if output_list != []:\n",
    "    \n",
    "    output = pd.DataFrame(\n",
    "    output_list,\n",
    "    columns=['Well', 'Run', 'Time_10g', 'Time_15g', 'Median VIB', 'Max VIB', 'Median VIBZ', 'Max VIBZ', 'Max_Temp', 'Energy', 'N2_RATE_av', 'N2_RATE_tot', 'N2_RATE_max']\n",
    "    )\n",
    "    \n",
    "    output_dupl = output.drop_duplicates()\n",
    "    print(output.shape[0] - output_dupl.shape[0], 'duplicates were deleted')\n",
    "    \n",
    "    pd.concat([previous_results, output_dupl], ignore_index=True).to_excel(os.path.join(output_directory,'CTD_output.xlsx'))\n",
    "    \n",
    "    print('Updating the log file...')\n",
    "    wells_to_file(output_directory, scanned_files+failed_files)\n",
    "\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "name": "python",
   "version": "3.8.15"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "c4236cb17b36bf95f1f18732d88bb225cba14ea66e1662d8982e141dcc601d0a"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}