diff --git a/containers/fastapi/Dockerfile b/containers/fastapi/Dockerfile index dd27c5d..df35b0c 100644 --- a/containers/fastapi/Dockerfile +++ b/containers/fastapi/Dockerfile @@ -1,34 +1,86 @@ -FROM docker.io/mambaorg/micromamba:1.5.1-jammy +FROM docker.io/mambaorg/micromamba:2.0.2-ubuntu24.04 +#FROM docker.io/mambaorg/micromamba:1.5.1-jammy LABEL maintainer="trygveas@met.no" ENV MAPGEN_REPO=https://github.com/metno/mapgen-fastapi.git \ MAPGEN_VERSION=main -COPY ./app /app # Install dependencies: -COPY --chown=$MAMBA_USER:$MAMBA_USER environment.yaml /tmp/environment.yaml +COPY --chown=$MAMBA_USER:$MAMBA_USER containers/fastapi/environment.yaml /tmp/environment.yaml RUN micromamba install -y -n base --file /tmp/environment.yaml USER root -RUN apt-get update && apt-get install -y --no-install-recommends \ - git lsof less\ - && rm -rf /var/lib/apt/lists/* +RUN set -ex \ + && ln -s /opt/conda/bin/curl /usr/bin/curl \ + && savedAptMark="$(apt-mark showmanual)" \ + && apt-get update \ + && apt-get install --no-install-recommends --no-install-suggests -y ca-certificates git lsof less build-essential libssl-dev libpcre2-dev pkg-config procps \ + && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ + && mkdir -p /usr/src/unit \ + && cd /usr/src/unit \ + && git clone --depth 1 -b 1.33.0-1 https://github.com/nginx/unit \ + && cd unit \ + && NCPU="$(getconf _NPROCESSORS_ONLN)" \ + && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ + && CC_OPT="$(DEB_BUILD_MAINT_OPTIONS="hardening=+all,-pie" DEB_CFLAGS_MAINT_APPEND="-Wp,-D_FORTIFY_SOURCE=2 -fPIC $(dpkg-buildflags --get CFLAGS)")" \ + && LD_OPT="$(DEB_BUILD_MAINT_OPTIONS="hardening=+all,-pie" DEB_LDFLAGS_MAINT_APPEND="-Wl,--as-needed -pie" dpkg-buildflags --get LDFLAGS)" \ + && CONFIGURE_ARGS_MODULES="--prefix=/usr \ + --statedir=/var/lib/unit \ + --control=unix:/var/run/unit/control.unit.sock \ + --runstatedir=/var/run/unit \ + --pid=/var/run/unit/unit.pid \ + --logdir=/var/log/unit \ + --log=/var/log/unit/unit.log \ + --tmpdir=/var/tmp \ + --user=$MAMBA_USER \ + --group=$MAMBA_USER \ + --openssl \ + --libdir=/usr/lib/$DEB_HOST_MULTIARCH" \ + && CONFIGURE_ARGS="$CONFIGURE_ARGS_MODULES \ + --njs" \ + && make -j $NCPU -C pkg/contrib .njs \ + && export PKG_CONFIG_PATH=$(pwd)/pkg/contrib/njs/build \ + && ./configure $CONFIGURE_ARGS --cc-opt="$CC_OPT" --ld-opt="$LD_OPT" --modulesdir=/usr/lib/unit/debug-modules --debug \ + && make -j $NCPU unitd \ + && install -pm755 build/sbin/unitd /usr/sbin/unitd-debug \ + && make clean \ + && ./configure $CONFIGURE_ARGS --cc-opt="$CC_OPT" --ld-opt="$LD_OPT" --modulesdir=/usr/lib/unit/modules \ + && make -j $NCPU unitd \ + && install -pm755 build/sbin/unitd /usr/sbin/unitd \ + && make clean \ + && /bin/true \ + && ./configure $CONFIGURE_ARGS_MODULES --cc-opt="$CC_OPT" --modulesdir=/usr/lib/unit/debug-modules --debug \ + && ./configure python --config=/opt/conda/bin/python3-config \ + && make -j $NCPU python3-install \ + && make clean \ + && ./configure $CONFIGURE_ARGS_MODULES --cc-opt="$CC_OPT" --modulesdir=/usr/lib/unit/modules \ + && ./configure python --config=/opt/conda/bin/python3-config \ + && make -j $NCPU python3-install \ + && cd \ + && rm -rf /usr/src/unit \ + && for f in /usr/sbin/unitd /usr/lib/unit/modules/*.unit.so; do \ + ldd $f | awk '/=>/{print $(NF-1)}' | while read n; do dpkg-query -S `basename $n`; done | sed 's/^\([^:]\+\):.*$/\1/' | sort | uniq >> /requirements.apt; \ + done \ + && apt-mark showmanual | xargs apt-mark auto > /dev/null \ + && { [ -z "$savedAptMark" ] || apt-mark manual $savedAptMark; } \ + && /bin/true \ + && mkdir -p /var/lib/unit/ \ + && mkdir -p /docker-entrypoint.d/ \ + && apt-get update \ + && apt-get --no-install-recommends --no-install-suggests -y install $(cat /requirements.apt) \ + && apt-get purge -y --auto-remove build-essential \ + && rm -rf /var/lib/apt/lists/* \ + && rm -f /requirements.apt + USER $MAMBA_USER RUN /opt/conda/bin/pip install "git+${MAPGEN_REPO}@${MAPGEN_VERSION}" xncml - -COPY --chown=$MAMBA_USER:$MAMBA_USER ./start.sh /start.sh - -COPY --chown=$MAMBA_USER:$MAMBA_USER ./gunicorn_conf.py /gunicorn_conf.py - -COPY --chown=$MAMBA_USER:$MAMBA_USER ./start-reload.sh /start-reload.sh -RUN chmod +x /start.sh \ - && chmod +x /start-reload.sh - +USER root WORKDIR /app -EXPOSE 80 - -# Run the start script, it will check for an /app/prestart.sh script (e.g. for migrations) -# And then will start Gunicorn with Uvicorn -CMD ["/start.sh"] \ No newline at end of file +ENV LD_LIBRARY_PATH=/usr/local/lib:/opt/conda/lib:/usr/local/lib/x86_64-linux-gnu:/lib/x86_64-linux-gnu:/usr/lib/x86_64-linux-gnu +COPY ./docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh +RUN mkdir -pv /usr/share/unit/welcome && chown -R $MAMBA_USER:$MAMBA_USER /usr/share/unit +COPY --chown=$MAMBA_USER:$MAMBA_USER welcome.* /usr/share/unit/welcome/ +ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"] +CMD ["unitd", "--no-daemon", "--control", "unix:/var/run/unit/control.unit.sock", "--statedir", "/var/lib/unit", "--pid", "/var/run/unit/unit.pid", "--log", "/proc/1/fd/1"] diff --git a/containers/fastapi/Dockerfile-http b/containers/fastapi/Dockerfile-http new file mode 100644 index 0000000..dcd1b5f --- /dev/null +++ b/containers/fastapi/Dockerfile-http @@ -0,0 +1,35 @@ +FROM docker.io/mambaorg/micromamba:1.5.1-jammy + +LABEL maintainer="trygveas@met.no" + +ENV MAPGEN_REPO=https://github.com/metno/mapgen-fastapi.git \ + MAPGEN_VERSION=httpserver + +COPY ./app /app +# Install dependencies: +COPY --chown=$MAMBA_USER:$MAMBA_USER environment.yaml /tmp/environment.yaml +RUN micromamba install -y -n base --file /tmp/environment.yaml + +USER root +RUN apt-get update && apt-get install -y --no-install-recommends \ + git lsof less\ + && rm -rf /var/lib/apt/lists/* +USER $MAMBA_USER +RUN /opt/conda/bin/pip install "git+${MAPGEN_REPO}@${MAPGEN_VERSION}" xncml + +#COPY --chown=$MAMBA_USER:$MAMBA_USER ./start.sh /start.sh + +#COPY --chown=$MAMBA_USER:$MAMBA_USER ./gunicorn_conf.py /gunicorn_conf.py + +#COPY --chown=$MAMBA_USER:$MAMBA_USER ./start-reload.sh /start-reload.sh +#RUN chmod +x /start.sh \ +# && chmod +x /start-reload.sh + +WORKDIR /app + +#EXPOSE 80 + +# Run the start script, it will check for an /app/prestart.sh script (e.g. for migrations) +# And then will start Gunicorn with Uvicorn +#CMD ["/start.sh"] +CMD ["/opt/conda/bin/python", "/opt/conda/lib/python3.12/site-packages/mapgen/main.py"] diff --git a/containers/fastapi/environment.yaml b/containers/fastapi/environment.yaml index afcab54..4437bc6 100644 --- a/containers/fastapi/environment.yaml +++ b/containers/fastapi/environment.yaml @@ -2,24 +2,26 @@ name: base channels: - conda-forge dependencies: -- python=3.11 +- python=3.12 - pip - wheel - Jinja2 -- gunicorn +#- gunicorn - pyyaml -- xarray=2024.6.0 +- xarray=2024.9.0 #=2024.6.0 - boto3 - netcdf4 - jinja2 - pytest-mock - rasterio - pandas -- satpy=0.49.0 -- mapserver=8.0.2 -- gdal=3.9.0 +- satpy=0.52.1 +- mapserver=8.2.2 +- gdal=3.9.2 #3.9.0 +- libgdal=3.9.2 - cartopy - metpy - lxml - cmocean - pyspectral +- curl diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh new file mode 100755 index 0000000..91adccb --- /dev/null +++ b/docker-entrypoint.sh @@ -0,0 +1,104 @@ +#!/bin/sh + +set -e + +WAITLOOPS=5 +SLEEPSEC=1 + +curl_put() +{ + RET=$(/usr/bin/curl -s -w '%{http_code}' -X PUT --data-binary @$1 --unix-socket /var/run/unit/control.unit.sock http://localhost/$2) + RET_BODY=$(echo $RET | /bin/sed '$ s/...$//') + RET_STATUS=$(echo $RET | /usr/bin/tail -c 4) + if [ "$RET_STATUS" -ne "200" ]; then + echo "$0: Error: HTTP response status code is '$RET_STATUS'" + echo "$RET_BODY" + return 1 + else + echo "$0: OK: HTTP response status code is '$RET_STATUS'" + echo "$RET_BODY" + fi + return 0 +} + +if [ "$1" = "unitd" ] || [ "$1" = "unitd-debug" ]; then + mkdir -pv /var/lib/unit + mkdir -pv /var/run/unit + mkdir -pv /var/log/unit + if /usr/bin/find "/var/lib/unit/" -mindepth 1 -print -quit 2>/dev/null | /bin/grep -q .; then + echo "$0: /var/lib/unit/ is not empty, skipping initial configuration..." + else + echo "$0: Launching Unit daemon to perform initial configuration..." + /usr/sbin/$1 --control unix:/var/run/unit/control.unit.sock --log /proc/1/fd/1 --statedir /var/lib/unit --pid /var/run/unit/unit.pid + + for i in $(/usr/bin/seq $WAITLOOPS); do + if [ ! -S /var/run/unit/control.unit.sock ]; then + echo "$0: Waiting for control socket to be created..." + /bin/sleep $SLEEPSEC + else + break + fi + done + # even when the control socket exists, it does not mean unit has finished initialisation + # this curl call will get a reply once unit is fully launched + /usr/bin/curl -s -X GET --unix-socket /var/run/unit/control.unit.sock http://localhost/ + + if /usr/bin/find "/docker-entrypoint.d/" -mindepth 1 -print -quit 2>/dev/null | /bin/grep -q .; then + echo "$0: /docker-entrypoint.d/ is not empty, applying initial configuration..." + + echo "$0: Looking for certificate bundles in /docker-entrypoint.d/..." + for f in $(/usr/bin/find /docker-entrypoint.d/ -type f -name "*.pem"); do + echo "$0: Uploading certificates bundle: $f" + curl_put $f "certificates/$(basename $f .pem)" + done + + echo "$0: Looking for JavaScript modules in /docker-entrypoint.d/..." + for f in $(/usr/bin/find /docker-entrypoint.d/ -type f -name "*.js"); do + echo "$0: Uploading JavaScript module: $f" + curl_put $f "js_modules/$(basename $f .js)" + done + + echo "$0: Looking for configuration snippets in /docker-entrypoint.d/..." + for f in $(/usr/bin/find /docker-entrypoint.d/ -type f -name "*.json"); do + echo "$0: Applying configuration $f"; + curl_put $f "config" + done + + echo "$0: Looking for shell scripts in /docker-entrypoint.d/..." + for f in $(/usr/bin/find /docker-entrypoint.d/ -type f -name "*.sh"); do + echo "$0: Launching $f"; + "$f" + done + + # warn on filetypes we don't know what to do with + for f in $(/usr/bin/find /docker-entrypoint.d/ -type f -not -name "*.sh" -not -name "*.json" -not -name "*.pem" -not -name "*.js"); do + echo "$0: Ignoring $f"; + done + else + echo "$0: /docker-entrypoint.d/ is empty, creating 'welcome' configuration..." + curl_put /usr/share/unit/welcome/welcome.json "config" + fi + + echo "$0: Stopping Unit daemon after initial configuration..." + kill -TERM $(/bin/cat /var/run/unit/unit.pid) + + for i in $(/usr/bin/seq $WAITLOOPS); do + if [ -S /var/run/unit/control.unit.sock ]; then + echo "$0: Waiting for control socket to be removed..." + /bin/sleep $SLEEPSEC + else + break + fi + done + if [ -S /var/run/unit/control.unit.sock ]; then + kill -KILL $(/bin/cat /var/run/unit/unit.pid) + rm -f /var/run/unit/control.unit.sock + fi + + echo + echo "$0: Unit initial configuration complete; ready for start up..." + echo + fi +fi + +exec "$@" diff --git a/mapgen/main.py b/mapgen/main.py index 60240a2..3d4f513 100644 --- a/mapgen/main.py +++ b/mapgen/main.py @@ -20,6 +20,8 @@ import os import time import logging +import threading +from random import randrange from multiprocessing import Process, Queue from modules.get_quicklook import get_quicklook from http.server import BaseHTTPRequestHandler, HTTPServer @@ -111,7 +113,7 @@ def app(environ, start_response): # response_code = '404' # response = "These aren't the droids you're looking for." # content_type = 'text/plain' - if image_path in '/robots.txt': + if image_path == '/robots.txt': response_code = '404' response = b"Not Found" content_type = 'text/plain' @@ -124,7 +126,7 @@ def app(environ, start_response): break except FileNotFoundError: pass - elif image_path in '/favicon.ico': + elif image_path == '/favicon.ico': response_code = '404' response = b"Not Found" content_type = 'text/plain' @@ -159,8 +161,34 @@ def app(environ, start_response): logging.debug(f"Queue length: {q.qsize()}") return [response] +def terminate_process(obj): + """Terminate process""" + logging.debug(f"{obj}") + if obj.returncode is None: + child_pid = obj.pid + logging.error("This child pid is %s.", str(child_pid)) + obj.kill() + logging.error("Process timed out and pre-maturely terminated.") + else: + logging.info("Process finished before time out.") + return + class wmsServer(BaseHTTPRequestHandler): + def send_response(self, code, message=None): + """Add the response header to the headers buffer and log the + response code. + + Also send two standard headers with the server software + version and the current date. + + """ + self.log_request(code) + self.send_response_only(code, message) + self.send_header('Server', 'ogc-wms-from-netcdf') + self.send_header('Date', self.date_time_string()) + def do_GET(self): + global number_of_successfull_requests content_type = 'text/plain' start = time.time() dbg = [self.path, self.client_address, self.requestline, self.request, self.command, self.address_string()] @@ -200,9 +228,17 @@ def do_GET(self): p.start() end = time.time() logging.debug(f"Started processing in {end - start:f}seconds") - (response_code, response, content_type) = q.get() - p.join() - logging.debug(f"Returning successfully from query.") + p.join(300) # Timeout + logging.debug(f"Processing exitcode: {p.exitcode}") + if p.exitcode is None: + logging.debug(f"Processing took too long. Stopping this process. Sorry.") + p.terminate() + response_code = '500' + response = b'Processing took too long. Stopping this process. Sorry.\n' + else: + logging.debug(f"Returning successfully from query: {p.exitcode}") + (response_code, response, content_type) = q.get() + number_of_successfull_requests += 1 end = time.time() logging.debug(f"Complete processing in {end - start:f}seconds") except KeyError as ke: @@ -267,25 +303,72 @@ def do_OPTIONS(self): self.end_headers() self.wfile.write(response) -#import ssl + +def request_counter(web_server, request_limit): + while 1: + try: + if number_of_successfull_requests > request_limit: + logging.info("Request limit reach, shutting down...") + web_server.shutdown() + logging.info("Shutting down complete.") + break + logging.info(f"Number of requests {number_of_successfull_requests} of {request_limit}") + time.sleep(1) + except KeyboardInterrupt: + break + +class request_limit_shutdown(threading.Thread): + + """""" + + def __init__(self, web_server, request_limit): + threading.Thread.__init__(self) + self.loop = True + self.web_server = web_server + self.request_limit = request_limit + + def stop(self): + """Stops the request_limit loop""" + self.loop = False + + def run(self): + try: + self.loop = True + while self.loop: + if number_of_successfull_requests > self.request_limit: + logging.info("Request limit reach, shutting down...") + self.web_server.shutdown() + logging.info("Shutting down complete.") + self.loop = False + logging.info(f"Number of requests {number_of_successfull_requests} of {self.request_limit}") + time.sleep(1) + except KeyboardInterrupt: + logging.info("Shutdown request_limit") + +class CustomHTTPServer(HTTPServer): + request_queue_size = 1 + if __name__ == "__main__": logging.config.dictConfig(logging_cfg) hostName = "0.0.0.0" serverPort = 8040 - webServer = HTTPServer((hostName, serverPort), wmsServer) + webServer = CustomHTTPServer((hostName, serverPort), wmsServer) webServer.timeout = 600 - # sslctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) - # sslctx.check_hostname = False # If set to True, only the hostname that matches the certificate will be accepted - # sslctx.load_cert_chain(certfile='cert.pem', keyfile="key.pem") - # webServer.socket = sslctx.wrap_socket(webServer.socket, server_side=True) + number_of_successfull_requests = 0 - print(webServer.request_queue_size) - print("Server started http://%s:%s" % (hostName, serverPort)) + logging.info(f"request queue size: {webServer.request_queue_size}") + logging.info(f"Server started http://{hostName}:{serverPort}") + request_limit = randrange(50,100) + logging.debug(f"This server request_limit: {request_limit}") + request_counter_thread = request_limit_shutdown(webServer, request_limit) + request_counter_thread.start() try: webServer.serve_forever() except KeyboardInterrupt: + request_counter_thread.stop() pass + request_counter_thread.join() webServer.server_close() print("Server stopped.") diff --git a/mapgen/modules/helpers.py b/mapgen/modules/helpers.py index b65e0cc..ae03749 100644 --- a/mapgen/modules/helpers.py +++ b/mapgen/modules/helpers.py @@ -475,11 +475,14 @@ def _generate_getcapabilities(layer, ds, variable, grid_mapping_cache, netcdf_fi layer.metadata.set("wms_extent", f"{ll_x} {ll_y} {ur_x} {ur_y}") dims_list = [] if 'time' not in ds[variable].dims: - try: - valid_time = datetime.datetime.fromisoformat(ds.time_coverage_start).strftime('%Y-%m-%dT%H:%M:%SZ') - layer.metadata.set("wms_timeextent", f'{valid_time}') - except Exception: - logger.debug("Could not use time_coverange_start global attribute. wms_timeextent is not added") + logger.debug(f"variable {variable} do not contain time variable. wms_timeextent as dimension is not added.") + # It makes no sense to add time dimension to a variable without timedimension. It can never be found. + # Removed from code 2024-10-23 + # try: + # valid_time = datetime.datetime.fromisoformat(ds.time_coverage_start).strftime('%Y-%m-%dT%H:%M:%SZ') + # layer.metadata.set("wms_timeextent", f'{valid_time}') + # except Exception: + # logger.debug("Could not use time_coverange_start global attribute. wms_timeextent is not added") for dim_name in ds[variable].dims: if dim_name in ['x', 'X', 'Xc', 'y', 'Y', 'Yc', 'longitude', 'latitude']: @@ -750,12 +753,13 @@ def _find_dimensions(ds, actual_variable, variable, qp, netcdf_file, last_ds): _ds['ds_size'] = ds[dim_name].data.size _ds['selected_band_number'] = 0 dimension_search.append(_ds) - logger.debug(f"{dimension_search}") + logger.debug(f"Dimension Search: {dimension_search}") return dimension_search def _calc_band_number_from_dimensions(dimension_search): band_number = 0 first = True + logger.debug(f"Calculate band number from dimension: {dimension_search}") #dimension_search.reverse() for _ds in dimension_search[::-1]: if first: @@ -765,6 +769,9 @@ def _calc_band_number_from_dimensions(dimension_search): first = False prev_ds = _ds + if band_number == 0 and len(dimension_search) == 0: + logger.warning("Could not calculate band number from empty dimension search. Use 1.") + band_number = 1 logger.debug(f"selected band number {band_number}") return band_number @@ -1215,7 +1222,11 @@ def _generate_layer(layer, ds, grid_mapping_cache, netcdf_file, qp, map_obj, pro else: logger.debug(f"Dimmension search len {len(dimension_search)}") - if len(dimension_search) == 1: + if len(dimension_search) == 0: + logger.debug("Len 0") + min_val = np.nanmin(ds[actual_variable][:,:].data) + max_val = np.nanmax(ds[actual_variable][:,:].data) + elif len(dimension_search) == 1: logger.debug("Len 1") min_val = np.nanmin(ds[actual_variable][dimension_search[0]['selected_band_number'],:,:].data) max_val = np.nanmax(ds[actual_variable][dimension_search[0]['selected_band_number'],:,:].data) @@ -1260,7 +1271,11 @@ def _generate_layer(layer, ds, grid_mapping_cache, netcdf_file, qp, map_obj, pro logger.debug("Dimension search empty. Possible calculated field.") else: logger.error(f"Could not estimate or read min and/or max val of dataset: {actual_variable}") - logger.debug(f"MIN:MAX {min_val} {max_val}") + try: + logger.debug(f"MIN:MAX {min_val} {max_val}") + except UnboundLocalError as le: + logger.error(f"status_code=500, Failed with: {str(le)}.") + raise HTTPError(response_code='500', response=f"Unspecified internal server error.") #Grayscale if style in 'contour': #variable.endswith('_contour'): logger.debug("Style in contour for style setup.") diff --git a/mapgen/modules/satellite_satpy_quicklook.py b/mapgen/modules/satellite_satpy_quicklook.py index 122bccd..fd1fd67 100644 --- a/mapgen/modules/satellite_satpy_quicklook.py +++ b/mapgen/modules/satellite_satpy_quicklook.py @@ -260,7 +260,11 @@ def _generate_satpy_geotiff(netcdf_paths, satpy_products_to_generate, start_time return True logger.debug(f"Need to generate: {satpy_products} from {netcdf_paths}") logger.debug(f"Before Scene") - swath_scene = Scene(filenames=netcdf_paths, reader='satpy_cf_nc') + try: + swath_scene = Scene(filenames=netcdf_paths, reader='satpy_cf_nc') + except ValueError as ve: + logger.error(f"Scene creation failed with: {str(ve)}") + return False logger.debug(f"Before load, resolution: {resolution}") swath_scene.load(satpy_products, resolution=resolution) logger.debug(f"Available composites names: {swath_scene.available_composite_names()}") diff --git a/welcome.html b/welcome.html new file mode 100644 index 0000000..7167ddb --- /dev/null +++ b/welcome.html @@ -0,0 +1,45 @@ + + +
+Congratulations! NGINX Unit is installed and running.
+Unit's control API is currently listening for configuration changes
+ on the Unix socket at
+ /var/run/control.unit.sock inside the container.
+ To see the current configuration run:
docker exec -ti <containerID> curl --unix-socket /var/run/control.unit.sock http://localhost/config+
NGINX Unit — the universal web app server
+ NGINX, Inc. © 2024