added utility functions and model for handling water_budget files #6

framework/climate/models.py

@@ -257,3 +257,12 @@ class ProvenanceInline(models.Model):
 
     def __str__(self):
         return self.relation_type + " " + self.target.name
+
+
+class NcFile(models.Model):
+    filepath = models.CharField(max_length=2048, unique=True)
+    num_bands = models.IntegerField()
+    band_metadata = models.JSONField()
+
+    def __str__(self):
+        return self.filepath

framework/climate/views.py

@@ -12,6 +12,7 @@
 from datetime import datetime
 
 import requests
+from django.conf import settings
 from django.http import (HttpResponse, HttpResponseBadRequest, JsonResponse, StreamingHttpResponse,)
 from elasticsearch_dsl import Index
 from elasticsearch_dsl.connections import connections
@@ -27,13 +28,21 @@
 from .serializer import ClimateLayerSerializer
 
 
-# URLTXTFILES_DIR = os.path.join(settings.STATICFILES_DIRS[0], 'urltxtfiles')
-URLTXTFILES_DIR = "/opt/rbis/www/tippecc_data/tmp/"
-#TESTCONTENT_DIR = "/opt/rbis/www/tippecc_data/tmp/water_budget"
+URLTXTFILES_DIR = os.path.join(settings.STATICFILES_DIRS[0], 'urltxtfiles')
+CACHED_TIFS_DIR = os.path.join(settings.STATICFILES_DIRS[0], 'cached_tifs')
+WATER_BUDGET_DIR = os.path.join(settings.STATICFILES_DIRS[0], 'test_content')
+WATER_BUDGET_BIAS_DIR = os.path.join(settings.STATICFILES_DIRS[0], 'water_budget_bias')
 
 folder_list = {}
-folder_list["water_budget"] = "/opt/rbis/www/tippecc_data/tmp/water_budget"
-folder_list["water_budget_bias"] = "/opt/rbis/www/tippecc_data/tmp/water_budget/bias"
+folder_list["water_budget"] = WATER_BUDGET_DIR
+folder_list["water_budget_bias"] = WATER_BUDGET_BIAS_DIR
+
+# URLTXTFILES_DIR = "/opt/rbis/www/tippecc_data/tmp/"
+# WATER_BUDGET_DIR = "/opt/rbis/www/tippecc_data/tmp/water_budget"
+
+# folder_list = {}
+# folder_list["water_budget"] = "/opt/rbis/www/tippecc_data/tmp/water_budget"
+# folder_list["water_budget_bias"] = "/opt/rbis/www/tippecc_data/tmp/water_budget/bias"
 
 
 GENERAL_API_URL = "https://leutra.geogr.uni-jena.de/backend_geoportal/"
@@ -80,12 +89,13 @@ def post(self, request):
 
         # for all requested files in requestbody, check if they really exist
         for entry in body:
+            # TODO: change to patchcheck
             if entry not in foldercontent:
                 return HttpResponseBadRequest()
 
         url_content = ""
         for entry in body:
-            url_content += GENERAL_API_URL + "/climate/get_file?name=" + entry + "&type="+ type + "\n"
+            url_content += GENERAL_API_URL + "/climate/get_file?name=" + entry + "&type=" + type + "\n"
 
         unique_filehash = str(uuid.uuid4().hex)
         unique_filename = unique_filehash + ".txt"
@@ -109,18 +119,17 @@ def post(self, request):
         return response
 
 
-# returns all filenames of the specified directory ('TESTCONTENT_DIR' rn)
+# returns all filenames of the specified directory ('WATER_BUDGET_DIR' rn)
 class ContentView(APIView):
     def get(self, request):
-
         folder = request.GET.get("type", default=None)
-        TESTCONTENT_DIR = folder_list[folder]
-        foldercontent = os.listdir(TESTCONTENT_DIR)
-        
+        source_dir = folder_list[folder]
+        foldercontent = os.listdir(source_dir)
+
         dir_content = []
 
         for i, f in enumerate(foldercontent):
-            full_filename = TESTCONTENT_DIR + "/" + f
+            full_filename = source_dir + "/" + f
             file_stats = os.stat(full_filename)
 
             dir_content_element = []
@@ -161,7 +170,6 @@ def get(self, request):
 
         TESTCONTENT_DIR = folder_list[folder]
 
-
         foldercontent = os.listdir(TESTCONTENT_DIR)
         print("FILENAME: ", filename)
         print("FOLDERCONTENT: ", foldercontent)

framework/utils/water_budget_utils.py

@@ -0,0 +1,82 @@
+import json
+import os
+from pathlib import Path
+from subprocess import (PIPE, Popen,)
+
+from django.conf import settings
+from osgeo import gdal
+
+from framework.climate.models import NcFile
+
+
+test_file_name = "GERICS-REMO2015_v1_MOHC-HadGEM2-ES_pr_Afr_day_1970_2099__mm__yearsum__yearsum_mean_1980_2000-2080_2099.nc"
+
+CACHED_TIFS_DIR = os.path.join(settings.STATICFILES_DIRS[0], 'cached_tifs')
+
+
+def extract_file_metadata(filepath: str):
+    if not os.path.isfile(filepath):
+        return False
+
+    process = Popen(["gdalinfo", filepath, "-json", "-mm"], stdout=PIPE, stderr=PIPE)
+    # process = Popen(f"gdalinfo data/tif_data/{input_filename} -json")
+    stdout, stderr = process.communicate()
+    metadata = stdout.decode("utf-8")
+
+    JSON_metadata = json.loads(metadata)
+
+    if 'bands' not in JSON_metadata:
+        return False
+
+    complete_band_metadata = {}
+    num_bands = len(JSON_metadata['bands'])
+
+    try:
+        for i, band_metadata in enumerate(JSON_metadata['bands']):
+            band_collect = {}
+            band_collect['min'] = band_metadata['computedMin']
+            band_collect['max'] = band_metadata['computedMax']
+            band_collect['NETCDF_DIM_time'] = band_metadata['metadata']['']['NETCDF_DIM_time']
+            band_collect['index'] = i+1
+            complete_band_metadata[str(i+1)] = band_collect
+    except Exception:
+        return False
+
+    try:
+        NcFile(filepath=filepath, num_bands=num_bands, band_metadata=complete_band_metadata).save()
+    except Exception:
+        return False
+
+    print(complete_band_metadata)
+    return True
+
+
+def translate_nc_file_to_gtiff(filename_in: str, dir_in: str, dir_out: str):
+    filepath_in = os.path.join(dir_in, filename_in)
+    if not os.path.isfile(filepath_in):
+        return False
+
+    filename_out = Path(filename_in).stem + ".tif"
+    filepath_out = os.path.join(dir_out, filename_out)
+
+    print(f"filename_out: {filename_out}")
+    print(f"filepath_in: {filepath_in}")
+    print(f"filepath_out: {filepath_out}")
+    ds = gdal.Open(filepath_in)
+    gdal.Translate(filepath_out, ds, format='Gtiff')
+
+
+def get_file_metadata(filepath: str):
+    filtered_object: NcFile = NcFile.objects.get(filepath=filepath)
+    print(f"FILTERED OBJECT: {filtered_object}")
+    print(filtered_object.num_bands)
+
+
+def is_tif_file_cached(filename: str):
+    if os.path.isfile(os.path.join(CACHED_TIFS_DIR, filename)):
+        return True
+    else:
+        return False
+
+# extract_file_metadata(os.path.join(WATER_BUDGET_DIR, test_file_name))
+# get_file_metadata(os.path.join(WATER_BUDGET_DIR, test_file_name))