From d0c4245f136b2721146738c3ef2349b1706ad4d9 Mon Sep 17 00:00:00 2001 From: Trygve Aspenes Date: Wed, 16 Oct 2024 08:55:08 +0200 Subject: [PATCH 1/3] add httpserver --- containers/fastapi/Dockerfile | 2 +- mapgen/main.py | 125 +++++++++++++++++++- mapgen/modules/arome_arctic_quicklook.py | 8 +- mapgen/modules/generic_quicklook.py | 8 +- mapgen/modules/get_quicklook.py | 8 +- mapgen/modules/satellite_satpy_quicklook.py | 4 +- 6 files changed, 138 insertions(+), 17 deletions(-) diff --git a/containers/fastapi/Dockerfile b/containers/fastapi/Dockerfile index 0d30f7d..dd27c5d 100644 --- a/containers/fastapi/Dockerfile +++ b/containers/fastapi/Dockerfile @@ -12,7 +12,7 @@ RUN micromamba install -y -n base --file /tmp/environment.yaml USER root RUN apt-get update && apt-get install -y --no-install-recommends \ - git \ + git lsof less\ && rm -rf /var/lib/apt/lists/* USER $MAMBA_USER RUN /opt/conda/bin/pip install "git+${MAPGEN_REPO}@${MAPGEN_VERSION}" xncml diff --git a/mapgen/main.py b/mapgen/main.py index 9f912e3..fef0cc2 100644 --- a/mapgen/main.py +++ b/mapgen/main.py @@ -21,7 +21,8 @@ import time import logging from multiprocessing import Process, Queue -from mapgen.modules.get_quicklook import get_quicklook +from modules.get_quicklook import get_quicklook +from http.server import BaseHTTPRequestHandler, HTTPServer logging_cfg = { 'version': 1, @@ -97,7 +98,7 @@ def app(environ, start_response): end = time.time() logging.debug(f"Complete processing in {end - start:f}seconds") except KeyError as ke: - logging.debug(f"Failed to parse the query.") + logging.debug(f"Failed to parse the query: {str(ke)}") response_code = '404' response = b'Not Found\n' response_headers = [('Content-Type', content_type)] @@ -157,3 +158,123 @@ def app(environ, start_response): start_response(response_code, response_headers) logging.debug(f"Queue length: {q.qsize()}") return [response] + +class wmsServer(BaseHTTPRequestHandler): + def do_GET(self): + content_type = 'text/plain' + start = time.time() + dbg = [self.path, self.client_address, self.requestline, self.request, self.command, self.address_string()] + for d in dbg: + logging.debug(f"{d}") + for h in str(self.headers).split('\n'): + logging.debug(f"Header: {h}") + + try: + q = Queue() + if self.path.startswith('/api/get_quicklook'): + try: + netcdf_path = self.path.replace('/api/get_quicklook','').split('?')[0] + try: + query_string = self.path.split('?')[1] + except IndexError: + query_string = "" + url_scheme = os.environ.get('SCHEME', 'http') # environ.get('HTTP_X_SCHEME', environ['wsgi.url_scheme']) + http_host = os.environ.get('HOST_NAME', self.address_string()) # environ['HTTP_HOST'] + p = Process(target=start_processing, + args=(netcdf_path, + query_string, + http_host, + url_scheme, + q)) + p.start() + end = time.time() + logging.debug(f"Started processing in {end - start:f}seconds") + (response_code, response, content_type) = q.get() + p.join() + logging.debug(f"Returning successfully from query.") + end = time.time() + logging.debug(f"Complete processing in {end - start:f}seconds") + except KeyError as ke: + logging.debug(f"Failed to parse the query: {str(ke)}") + response_code = '404' + response = b'Not Found\n' + else: + """Need this to local images and robots.txt""" + image_path = self.path + logging.debug(f"image path: {image_path}") + logging.debug(f"CWD: {os.getcwd()}") + if image_path in '/robots.txt': + response_code = '404' + response = b"Not Found" + content_type = 'text/plain' + for imgp in [image_path, '.' + image_path]: + try: + logging.debug(f"Try opening {imgp}") + with open(imgp,'rb') as ip: + response = ip.read() + response_code = '200' + break + except FileNotFoundError: + pass + elif image_path in '/favicon.ico': + response_code = '404' + response = b"Not Found" + content_type = 'text/plain' + for imgp in [image_path, '.' + image_path]: + try: + logging.debug(f"Try opening {imgp}") + with open(imgp,'rb') as ip: + response = ip.read() + response_code = '200' + content_type = 'image/vnd.microsoft.icon' + break + except FileNotFoundError: + pass + else: + response_code = '404' + response = b"These aren't the droids you're looking for.\n" + content_type = 'text/plain' + response_headers = [('Content-Type', content_type)] + response_headers.append(('Access-Control-Allow-Origin', '*')) + self.send_response(int(response_code)) + for response_header in response_headers: + self.send_header(response_header[0], response_header[1]) + self.end_headers() + self.wfile.write(response) + except BrokenPipeError: + logging.warning("Lost connection to client.") + pass + + def do_OPTIONS(self): + response_headers = [ ('Access-Control-Allow-Methods', 'GET, OPTIONS'), ('Access-Control-Allow-Headers', '*')] + response_code = '200' + response = b'' + logging.debug(f"OPTIONS respond, {response_headers}") + self.send_response(int(response_code)) + for response_header in response_headers: + self.send_header(response_header[0], response_header[1]) + self.end_headers() + self.wfile.write(response) + +#import ssl +if __name__ == "__main__": + logging.config.dictConfig(logging_cfg) + hostName = "0.0.0.0" + serverPort = 8040 + webServer = HTTPServer((hostName, serverPort), wmsServer) + webServer.timeout = 600 + # sslctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + # sslctx.check_hostname = False # If set to True, only the hostname that matches the certificate will be accepted + # sslctx.load_cert_chain(certfile='cert.pem', keyfile="key.pem") + # webServer.socket = sslctx.wrap_socket(webServer.socket, server_side=True) + + print(webServer.request_queue_size) + print("Server started http://%s:%s" % (hostName, serverPort)) + + try: + webServer.serve_forever() + except KeyboardInterrupt: + pass + + webServer.server_close() + print("Server stopped.") diff --git a/mapgen/modules/arome_arctic_quicklook.py b/mapgen/modules/arome_arctic_quicklook.py index 394344a..5e9bfe9 100644 --- a/mapgen/modules/arome_arctic_quicklook.py +++ b/mapgen/modules/arome_arctic_quicklook.py @@ -40,10 +40,10 @@ import datetime import mapscript import xarray as xr -from mapgen.modules.create_symbol_file import create_symbol_file -from mapgen.modules.helpers import handle_request, _parse_filename, _get_mapfiles_path, _fill_metadata_to_mapfile -from mapgen.modules.helpers import _generate_getcapabilities, _generate_getcapabilities_vector, _generate_layer -from mapgen.modules.helpers import _parse_request, HTTPError +from modules.create_symbol_file import create_symbol_file +from modules.helpers import handle_request, _parse_filename, _get_mapfiles_path, _fill_metadata_to_mapfile +from modules.helpers import _generate_getcapabilities, _generate_getcapabilities_vector, _generate_layer +from modules.helpers import _parse_request, HTTPError grid_mapping_cache = {} wind_rotation_cache = {} diff --git a/mapgen/modules/generic_quicklook.py b/mapgen/modules/generic_quicklook.py index 5fd16d3..82d2f85 100644 --- a/mapgen/modules/generic_quicklook.py +++ b/mapgen/modules/generic_quicklook.py @@ -41,10 +41,10 @@ import mapscript import xarray as xr -from mapgen.modules.create_symbol_file import create_symbol_file -from mapgen.modules.helpers import handle_request, _fill_metadata_to_mapfile, _parse_filename, _get_mapfiles_path -from mapgen.modules.helpers import _generate_getcapabilities, _generate_getcapabilities_vector, _generate_layer -from mapgen.modules.helpers import _parse_request, _read_netcdfs_from_ncml, HTTPError +from modules.create_symbol_file import create_symbol_file +from modules.helpers import handle_request, _fill_metadata_to_mapfile, _parse_filename, _get_mapfiles_path +from modules.helpers import _generate_getcapabilities, _generate_getcapabilities_vector, _generate_layer +from modules.helpers import _parse_request, _read_netcdfs_from_ncml, HTTPError grid_mapping_cache = {} summary_cache = {} diff --git a/mapgen/modules/get_quicklook.py b/mapgen/modules/get_quicklook.py index 144975b..434aa9b 100644 --- a/mapgen/modules/get_quicklook.py +++ b/mapgen/modules/get_quicklook.py @@ -20,11 +20,11 @@ import sys import logging -from mapgen.modules.helpers import find_config_for_this_netcdf, HTTPError +from modules.helpers import find_config_for_this_netcdf, HTTPError -import mapgen.modules.arome_arctic_quicklook -import mapgen.modules.generic_quicklook -import mapgen.modules.satellite_satpy_quicklook +import modules.arome_arctic_quicklook +import modules.generic_quicklook +import modules.satellite_satpy_quicklook logger = logging.getLogger(__name__) diff --git a/mapgen/modules/satellite_satpy_quicklook.py b/mapgen/modules/satellite_satpy_quicklook.py index 72806c8..f659790 100644 --- a/mapgen/modules/satellite_satpy_quicklook.py +++ b/mapgen/modules/satellite_satpy_quicklook.py @@ -31,8 +31,8 @@ from datetime import datetime from urllib.parse import parse_qs -from mapgen.modules.helpers import handle_request -from mapgen.modules.helpers import HTTPError +from modules.helpers import handle_request +from modules.helpers import HTTPError boto3.set_stream_logger('botocore', logging.CRITICAL) boto3.set_stream_logger('boto3', logging.CRITICAL) From 83128bdf019f5288080c10cd4ee474034a1480c1 Mon Sep 17 00:00:00 2001 From: Trygve Aspenes Date: Wed, 16 Oct 2024 10:05:10 +0200 Subject: [PATCH 2/3] force regen --- mapgen/modules/satellite_satpy_quicklook.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mapgen/modules/satellite_satpy_quicklook.py b/mapgen/modules/satellite_satpy_quicklook.py index f659790..a74a343 100644 --- a/mapgen/modules/satellite_satpy_quicklook.py +++ b/mapgen/modules/satellite_satpy_quicklook.py @@ -246,6 +246,7 @@ def _exists_on_ceph(satpy_product, start_time): finally: del s3 s3 = None + exists = False return exists def _generate_satpy_geotiff(netcdf_paths, satpy_products_to_generate, start_time, product_config, resolution): From 41e222771531496166f91898e86ac855bea7aa7c Mon Sep 17 00:00:00 2001 From: Trygve Aspenes Date: Wed, 16 Oct 2024 10:45:10 +0200 Subject: [PATCH 3/3] revert force. debug --- mapgen/modules/satellite_satpy_quicklook.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mapgen/modules/satellite_satpy_quicklook.py b/mapgen/modules/satellite_satpy_quicklook.py index a74a343..122bccd 100644 --- a/mapgen/modules/satellite_satpy_quicklook.py +++ b/mapgen/modules/satellite_satpy_quicklook.py @@ -246,7 +246,6 @@ def _exists_on_ceph(satpy_product, start_time): finally: del s3 s3 = None - exists = False return exists def _generate_satpy_geotiff(netcdf_paths, satpy_products_to_generate, start_time, product_config, resolution): @@ -259,7 +258,7 @@ def _generate_satpy_geotiff(netcdf_paths, satpy_products_to_generate, start_time if not satpy_products: logger.debug(f"No products needs to be generated.") return True - logger.debug(f"Need to generate: {satpy_products}") + logger.debug(f"Need to generate: {satpy_products} from {netcdf_paths}") logger.debug(f"Before Scene") swath_scene = Scene(filenames=netcdf_paths, reader='satpy_cf_nc') logger.debug(f"Before load, resolution: {resolution}")