From 3ae9efd82b48717dcb339eacd2a47d7b4914ab4f Mon Sep 17 00:00:00 2001 From: Trygve Aspenes Date: Wed, 16 Oct 2024 14:49:15 +0200 Subject: [PATCH 01/15] add try except for robustness --- mapgen/modules/satellite_satpy_quicklook.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/mapgen/modules/satellite_satpy_quicklook.py b/mapgen/modules/satellite_satpy_quicklook.py index 122bccd..fd1fd67 100644 --- a/mapgen/modules/satellite_satpy_quicklook.py +++ b/mapgen/modules/satellite_satpy_quicklook.py @@ -260,7 +260,11 @@ def _generate_satpy_geotiff(netcdf_paths, satpy_products_to_generate, start_time return True logger.debug(f"Need to generate: {satpy_products} from {netcdf_paths}") logger.debug(f"Before Scene") - swath_scene = Scene(filenames=netcdf_paths, reader='satpy_cf_nc') + try: + swath_scene = Scene(filenames=netcdf_paths, reader='satpy_cf_nc') + except ValueError as ve: + logger.error(f"Scene creation failed with: {str(ve)}") + return False logger.debug(f"Before load, resolution: {resolution}") swath_scene.load(satpy_products, resolution=resolution) logger.debug(f"Available composites names: {swath_scene.available_composite_names()}") From 78ab9311115de751f94ffb593cc3dba255dd0d10 Mon Sep 17 00:00:00 2001 From: Trygve Aspenes Date: Thu, 17 Oct 2024 11:01:48 +0200 Subject: [PATCH 02/15] test count thread --- mapgen/main.py | 69 ++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 64 insertions(+), 5 deletions(-) diff --git a/mapgen/main.py b/mapgen/main.py index 60240a2..6c515fe 100644 --- a/mapgen/main.py +++ b/mapgen/main.py @@ -20,6 +20,7 @@ import os import time import logging +import threading from multiprocessing import Process, Queue from modules.get_quicklook import get_quicklook from http.server import BaseHTTPRequestHandler, HTTPServer @@ -160,7 +161,21 @@ def app(environ, start_response): return [response] class wmsServer(BaseHTTPRequestHandler): + def send_response(self, code, message=None): + """Add the response header to the headers buffer and log the + response code. + + Also send two standard headers with the server software + version and the current date. + + """ + self.log_request(code) + self.send_response_only(code, message) + self.send_header('Server', 'ogc-wms-from-netcdf') + self.send_header('Date', self.date_time_string()) + def do_GET(self): + global number_of_successfull_requests content_type = 'text/plain' start = time.time() dbg = [self.path, self.client_address, self.requestline, self.request, self.command, self.address_string()] @@ -205,6 +220,7 @@ def do_GET(self): logging.debug(f"Returning successfully from query.") end = time.time() logging.debug(f"Complete processing in {end - start:f}seconds") + number_of_successfull_requests += 1 except KeyError as ke: logging.debug(f"Failed to parse the query: {str(ke)}") response_code = '404' @@ -267,25 +283,68 @@ def do_OPTIONS(self): self.end_headers() self.wfile.write(response) -#import ssl + +def request_counter(web_server, request_limit): + while 1: + try: + if number_of_successfull_requests > request_limit: + logging.info("Request limit reach, shutting down...") + web_server.shutdown() + logging.info("Shutting down complete.") + break + logging.info(f"Number of requests {number_of_successfull_requests} of {request_limit}") + time.sleep(1) + except KeyboardInterrupt: + break + +class request_limit_shutdown(threading.Thread): + + """""" + + def __init__(self, web_server, request_limit): + threading.Thread.__init__(self) + self.loop = True + self.web_server = web_server + self.request_limit = request_limit + + def stop(self): + """Stops the request_limit loop""" + self.loop = False + + def run(self): + try: + self.loop = True + while self.loop: + if number_of_successfull_requests > self.request_limit: + logging.info("Request limit reach, shutting down...") + self.web_server.shutdown() + logging.info("Shutting down complete.") + self.loop = False + logging.info(f"Number of requests {number_of_successfull_requests} of {self.request_limit}") + time.sleep(1) + except KeyboardInterrupt: + logging.info("Shutdown request_limit") + if __name__ == "__main__": logging.config.dictConfig(logging_cfg) hostName = "0.0.0.0" serverPort = 8040 webServer = HTTPServer((hostName, serverPort), wmsServer) webServer.timeout = 600 - # sslctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) - # sslctx.check_hostname = False # If set to True, only the hostname that matches the certificate will be accepted - # sslctx.load_cert_chain(certfile='cert.pem', keyfile="key.pem") - # webServer.socket = sslctx.wrap_socket(webServer.socket, server_side=True) + number_of_successfull_requests = 0 print(webServer.request_queue_size) print("Server started http://%s:%s" % (hostName, serverPort)) + request_counter_thread = request_limit_shutdown(webServer, 10) + request_counter_thread.start() + try: webServer.serve_forever() except KeyboardInterrupt: + request_counter_thread.stop() pass + request_counter_thread.join() webServer.server_close() print("Server stopped.") From 4d512494adebc54f95d4eda8f777391850aff06d Mon Sep 17 00:00:00 2001 From: Trygve Aspenes Date: Fri, 18 Oct 2024 11:02:29 +0200 Subject: [PATCH 03/15] add more debug --- containers/fastapi/Dockerfile | 17 ++++++------ containers/fastapi/environment.yaml | 13 ++++----- mapgen/main.py | 42 ++++++++++++++++++++++------- 3 files changed, 49 insertions(+), 23 deletions(-) diff --git a/containers/fastapi/Dockerfile b/containers/fastapi/Dockerfile index dd27c5d..dcd1b5f 100644 --- a/containers/fastapi/Dockerfile +++ b/containers/fastapi/Dockerfile @@ -3,7 +3,7 @@ FROM docker.io/mambaorg/micromamba:1.5.1-jammy LABEL maintainer="trygveas@met.no" ENV MAPGEN_REPO=https://github.com/metno/mapgen-fastapi.git \ - MAPGEN_VERSION=main + MAPGEN_VERSION=httpserver COPY ./app /app # Install dependencies: @@ -17,18 +17,19 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ USER $MAMBA_USER RUN /opt/conda/bin/pip install "git+${MAPGEN_REPO}@${MAPGEN_VERSION}" xncml -COPY --chown=$MAMBA_USER:$MAMBA_USER ./start.sh /start.sh +#COPY --chown=$MAMBA_USER:$MAMBA_USER ./start.sh /start.sh -COPY --chown=$MAMBA_USER:$MAMBA_USER ./gunicorn_conf.py /gunicorn_conf.py +#COPY --chown=$MAMBA_USER:$MAMBA_USER ./gunicorn_conf.py /gunicorn_conf.py -COPY --chown=$MAMBA_USER:$MAMBA_USER ./start-reload.sh /start-reload.sh -RUN chmod +x /start.sh \ - && chmod +x /start-reload.sh +#COPY --chown=$MAMBA_USER:$MAMBA_USER ./start-reload.sh /start-reload.sh +#RUN chmod +x /start.sh \ +# && chmod +x /start-reload.sh WORKDIR /app -EXPOSE 80 +#EXPOSE 80 # Run the start script, it will check for an /app/prestart.sh script (e.g. for migrations) # And then will start Gunicorn with Uvicorn -CMD ["/start.sh"] \ No newline at end of file +#CMD ["/start.sh"] +CMD ["/opt/conda/bin/python", "/opt/conda/lib/python3.12/site-packages/mapgen/main.py"] diff --git a/containers/fastapi/environment.yaml b/containers/fastapi/environment.yaml index afcab54..b06795a 100644 --- a/containers/fastapi/environment.yaml +++ b/containers/fastapi/environment.yaml @@ -2,22 +2,23 @@ name: base channels: - conda-forge dependencies: -- python=3.11 +- python=3.12 - pip - wheel - Jinja2 -- gunicorn +#- gunicorn - pyyaml -- xarray=2024.6.0 +- xarray=2024.9.0 #=2024.6.0 - boto3 - netcdf4 - jinja2 - pytest-mock - rasterio - pandas -- satpy=0.49.0 -- mapserver=8.0.2 -- gdal=3.9.0 +- satpy=0.51.0 +- mapserver=8.2.2 +- gdal=3.9.2 #3.9.0 +- libgdal=3.9.2 - cartopy - metpy - lxml diff --git a/mapgen/main.py b/mapgen/main.py index 6c515fe..25b89c2 100644 --- a/mapgen/main.py +++ b/mapgen/main.py @@ -21,6 +21,7 @@ import time import logging import threading +from random import randrange from multiprocessing import Process, Queue from modules.get_quicklook import get_quicklook from http.server import BaseHTTPRequestHandler, HTTPServer @@ -160,6 +161,18 @@ def app(environ, start_response): logging.debug(f"Queue length: {q.qsize()}") return [response] +def terminate_process(obj): + """Terminate process""" + logging.debug(f"{obj}") + if obj.returncode is None: + child_pid = obj.pid + logging.error("This child pid is %s.", str(child_pid)) + obj.kill() + logging.error("Process timed out and pre-maturely terminated.") + else: + logging.info("Process finished before time out.") + return + class wmsServer(BaseHTTPRequestHandler): def send_response(self, code, message=None): """Add the response header to the headers buffer and log the @@ -215,12 +228,19 @@ def do_GET(self): p.start() end = time.time() logging.debug(f"Started processing in {end - start:f}seconds") - (response_code, response, content_type) = q.get() - p.join() - logging.debug(f"Returning successfully from query.") + p.join(300) # Timeout + logging.debug(f"Processing exitcode: {p.exitcode}") + if p.exitcode is None: + logging.debug(f"Processing took too long. Stopping this process. Sorry.") + p.terminate() + response_code = '500' + response = b'Processing took too long. Stopping this process. Sorry.\n' + else: + logging.debug(f"Returning successfully from query: {p.exitcode}") + (response_code, response, content_type) = q.get() + number_of_successfull_requests += 1 end = time.time() logging.debug(f"Complete processing in {end - start:f}seconds") - number_of_successfull_requests += 1 except KeyError as ke: logging.debug(f"Failed to parse the query: {str(ke)}") response_code = '404' @@ -325,20 +345,24 @@ def run(self): except KeyboardInterrupt: logging.info("Shutdown request_limit") +class CustomHTTPServer(HTTPServer): + request_queue_size = 1 + if __name__ == "__main__": logging.config.dictConfig(logging_cfg) hostName = "0.0.0.0" serverPort = 8040 - webServer = HTTPServer((hostName, serverPort), wmsServer) + webServer = CustomHTTPServer((hostName, serverPort), wmsServer) webServer.timeout = 600 number_of_successfull_requests = 0 - print(webServer.request_queue_size) - print("Server started http://%s:%s" % (hostName, serverPort)) + logging.info(f"request queue size: {webServer.request_queue_size}") + logging.info(f"Server started http://{hostName}:{serverPort}") - request_counter_thread = request_limit_shutdown(webServer, 10) + request_limit = randrange(50,100) + logging.debug(f"This server request_limit: {request_limit}") + request_counter_thread = request_limit_shutdown(webServer, request_limit) request_counter_thread.start() - try: webServer.serve_forever() except KeyboardInterrupt: From 3b522c0bc36bbed01209434a393fb529e9050a77 Mon Sep 17 00:00:00 2001 From: Trygve Aspenes Date: Wed, 23 Oct 2024 15:26:29 +0200 Subject: [PATCH 04/15] handle dataset without any time dimension --- mapgen/modules/helpers.py | 31 +++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/mapgen/modules/helpers.py b/mapgen/modules/helpers.py index b65e0cc..ae03749 100644 --- a/mapgen/modules/helpers.py +++ b/mapgen/modules/helpers.py @@ -475,11 +475,14 @@ def _generate_getcapabilities(layer, ds, variable, grid_mapping_cache, netcdf_fi layer.metadata.set("wms_extent", f"{ll_x} {ll_y} {ur_x} {ur_y}") dims_list = [] if 'time' not in ds[variable].dims: - try: - valid_time = datetime.datetime.fromisoformat(ds.time_coverage_start).strftime('%Y-%m-%dT%H:%M:%SZ') - layer.metadata.set("wms_timeextent", f'{valid_time}') - except Exception: - logger.debug("Could not use time_coverange_start global attribute. wms_timeextent is not added") + logger.debug(f"variable {variable} do not contain time variable. wms_timeextent as dimension is not added.") + # It makes no sense to add time dimension to a variable without timedimension. It can never be found. + # Removed from code 2024-10-23 + # try: + # valid_time = datetime.datetime.fromisoformat(ds.time_coverage_start).strftime('%Y-%m-%dT%H:%M:%SZ') + # layer.metadata.set("wms_timeextent", f'{valid_time}') + # except Exception: + # logger.debug("Could not use time_coverange_start global attribute. wms_timeextent is not added") for dim_name in ds[variable].dims: if dim_name in ['x', 'X', 'Xc', 'y', 'Y', 'Yc', 'longitude', 'latitude']: @@ -750,12 +753,13 @@ def _find_dimensions(ds, actual_variable, variable, qp, netcdf_file, last_ds): _ds['ds_size'] = ds[dim_name].data.size _ds['selected_band_number'] = 0 dimension_search.append(_ds) - logger.debug(f"{dimension_search}") + logger.debug(f"Dimension Search: {dimension_search}") return dimension_search def _calc_band_number_from_dimensions(dimension_search): band_number = 0 first = True + logger.debug(f"Calculate band number from dimension: {dimension_search}") #dimension_search.reverse() for _ds in dimension_search[::-1]: if first: @@ -765,6 +769,9 @@ def _calc_band_number_from_dimensions(dimension_search): first = False prev_ds = _ds + if band_number == 0 and len(dimension_search) == 0: + logger.warning("Could not calculate band number from empty dimension search. Use 1.") + band_number = 1 logger.debug(f"selected band number {band_number}") return band_number @@ -1215,7 +1222,11 @@ def _generate_layer(layer, ds, grid_mapping_cache, netcdf_file, qp, map_obj, pro else: logger.debug(f"Dimmension search len {len(dimension_search)}") - if len(dimension_search) == 1: + if len(dimension_search) == 0: + logger.debug("Len 0") + min_val = np.nanmin(ds[actual_variable][:,:].data) + max_val = np.nanmax(ds[actual_variable][:,:].data) + elif len(dimension_search) == 1: logger.debug("Len 1") min_val = np.nanmin(ds[actual_variable][dimension_search[0]['selected_band_number'],:,:].data) max_val = np.nanmax(ds[actual_variable][dimension_search[0]['selected_band_number'],:,:].data) @@ -1260,7 +1271,11 @@ def _generate_layer(layer, ds, grid_mapping_cache, netcdf_file, qp, map_obj, pro logger.debug("Dimension search empty. Possible calculated field.") else: logger.error(f"Could not estimate or read min and/or max val of dataset: {actual_variable}") - logger.debug(f"MIN:MAX {min_val} {max_val}") + try: + logger.debug(f"MIN:MAX {min_val} {max_val}") + except UnboundLocalError as le: + logger.error(f"status_code=500, Failed with: {str(le)}.") + raise HTTPError(response_code='500', response=f"Unspecified internal server error.") #Grayscale if style in 'contour': #variable.endswith('_contour'): logger.debug("Style in contour for style setup.") From af00ff05dc389673b8f09ef0fa1cc3b92dfb555b Mon Sep 17 00:00:00 2001 From: Trygve Aspenes Date: Fri, 25 Oct 2024 15:16:01 +0200 Subject: [PATCH 05/15] Need exact path --- mapgen/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mapgen/main.py b/mapgen/main.py index 25b89c2..3d4f513 100644 --- a/mapgen/main.py +++ b/mapgen/main.py @@ -113,7 +113,7 @@ def app(environ, start_response): # response_code = '404' # response = "These aren't the droids you're looking for." # content_type = 'text/plain' - if image_path in '/robots.txt': + if image_path == '/robots.txt': response_code = '404' response = b"Not Found" content_type = 'text/plain' @@ -126,7 +126,7 @@ def app(environ, start_response): break except FileNotFoundError: pass - elif image_path in '/favicon.ico': + elif image_path == '/favicon.ico': response_code = '404' response = b"Not Found" content_type = 'text/plain' From f07704e75276bd1b37dcecb7bcf293f66d9c6111 Mon Sep 17 00:00:00 2001 From: Trygve Aspenes Date: Fri, 25 Oct 2024 15:44:21 +0200 Subject: [PATCH 06/15] test new build --- containers/fastapi/Dockerfile | 128 ++++++++++++++++++++++++++--- containers/fastapi/Dockerfile-http | 35 ++++++++ 2 files changed, 152 insertions(+), 11 deletions(-) create mode 100644 containers/fastapi/Dockerfile-http diff --git a/containers/fastapi/Dockerfile b/containers/fastapi/Dockerfile index dcd1b5f..44487bb 100644 --- a/containers/fastapi/Dockerfile +++ b/containers/fastapi/Dockerfile @@ -1,21 +1,25 @@ -FROM docker.io/mambaorg/micromamba:1.5.1-jammy +FROM docker.io/mambaorg/micromamba:2.0.2-ubuntu24.04 +#FROM docker.io/mambaorg/micromamba:1.5.1-jammy LABEL maintainer="trygveas@met.no" ENV MAPGEN_REPO=https://github.com/metno/mapgen-fastapi.git \ MAPGEN_VERSION=httpserver -COPY ./app /app +#COPY ./app /app # Install dependencies: -COPY --chown=$MAMBA_USER:$MAMBA_USER environment.yaml /tmp/environment.yaml +COPY --chown=$MAMBA_USER:$MAMBA_USER containers/fastapi/environment.yaml /tmp/environment.yaml RUN micromamba install -y -n base --file /tmp/environment.yaml -USER root -RUN apt-get update && apt-get install -y --no-install-recommends \ - git lsof less\ - && rm -rf /var/lib/apt/lists/* -USER $MAMBA_USER -RUN /opt/conda/bin/pip install "git+${MAPGEN_REPO}@${MAPGEN_VERSION}" xncml +# USER root +# RUN apt-get update && apt-get install -y --no-install-recommends \ +# git lsof less curl \ +# && rm -rf /var/lib/apt/lists/* +#RUN curl --output /usr/share/keyrings/nginx-keyring.gpg https://unit.nginx.org/keys/nginx-keyring.gpg \ +# && echo "deb [signed-by=/usr/share/keyrings/nginx-keyring.gpg] https://packages.nginx.org/unit/ubuntu/ noble unit\ndeb-src [signed-by=/usr/share/keyrings/nginx-keyring.gpg] https://packages.nginx.org/unit/ubuntu/ noble unit" > /etc/apt/sources.list.d/unit.list \ +# && apt-get update \ +# && apt-cache search unit \ +# && apt-get install -y unit-python3.12 #COPY --chown=$MAMBA_USER:$MAMBA_USER ./start.sh /start.sh @@ -25,11 +29,113 @@ RUN /opt/conda/bin/pip install "git+${MAPGEN_REPO}@${MAPGEN_VERSION}" xncml #RUN chmod +x /start.sh \ # && chmod +x /start-reload.sh +# USER root +# RUN apt-get update && apt-get install -y --no-install-recommends \ +# git lsof less curl \ +# && rm -rf /var/lib/apt/lists/* + +USER root +RUN set -ex \ + && savedAptMark="$(apt-mark showmanual)" \ + && apt-get update \ + && apt-get install --no-install-recommends --no-install-suggests -y ca-certificates git lsof less build-essential libssl-dev libpcre2-dev curl pkg-config procps \ + && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ + && mkdir -p /usr/src/unit \ + && cd /usr/src/unit \ + && git clone --depth 1 -b 1.33.0-1 https://github.com/nginx/unit \ + && cd unit \ + && NCPU="$(getconf _NPROCESSORS_ONLN)" \ + && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ + && CC_OPT="$(DEB_BUILD_MAINT_OPTIONS="hardening=+all,-pie" DEB_CFLAGS_MAINT_APPEND="-Wp,-D_FORTIFY_SOURCE=2 -fPIC" dpkg-buildflags --get CFLAGS)" \ + && LD_OPT="$(DEB_BUILD_MAINT_OPTIONS="hardening=+all,-pie" DEB_LDFLAGS_MAINT_APPEND="-Wl,--as-needed -pie" dpkg-buildflags --get LDFLAGS)" \ + && CONFIGURE_ARGS_MODULES="--prefix=/usr \ + --statedir=/var/lib/unit \ + --control=unix:/var/run/control.unit.sock \ + --runstatedir=/var/run \ + --pid=/var/run/unit.pid \ + --logdir=/var/log \ + --log=/var/log/unit.log \ + --tmpdir=/var/tmp \ + --user=$MAMBA_USER \ + --group=$MAMBA_USER \ + --openssl \ + --libdir=/usr/lib/$DEB_HOST_MULTIARCH" \ + && CONFIGURE_ARGS="$CONFIGURE_ARGS_MODULES \ + --njs" \ + && make -j $NCPU -C pkg/contrib .njs \ + && export PKG_CONFIG_PATH=$(pwd)/pkg/contrib/njs/build +RUN cd /usr/src/unit/unit \ + && ./configure $CONFIGURE_ARGS --cc-opt="$CC_OPT" --ld-opt="$LD_OPT" --modulesdir=/usr/lib/unit/debug-modules --debug \ + && make -j $NCPU unitd \ + && install -pm755 build/sbin/unitd /usr/sbin/unitd-debug \ + && make clean \ + && ./configure $CONFIGURE_ARGS --cc-opt="$CC_OPT" --ld-opt="$LD_OPT" --modulesdir=/usr/lib/unit/modules \ + && make -j $NCPU unitd \ + && install -pm755 build/sbin/unitd /usr/sbin/unitd \ + && make clean \ + && /bin/true \ + && ./configure $CONFIGURE_ARGS_MODULES --cc-opt="$CC_OPT" --modulesdir=/usr/lib/unit/debug-modules --debug \ + && ./configure python --config=/opt/conda/bin/python3-config \ + && make -j $NCPU python3-install \ + && make clean \ + && ./configure $CONFIGURE_ARGS_MODULES --cc-opt="$CC_OPT" --modulesdir=/usr/lib/unit/modules \ + && ./configure python --config=/opt/conda/bin/python3-config \ + && make -j $NCPU python3-install \ + && cd \ + && rm -rf /usr/src/unit \ + && for f in /usr/sbin/unitd /usr/lib/unit/modules/*.unit.so; do \ + ldd $f | awk '/=>/{print $(NF-1)}' | while read n; do dpkg-query -S $n; done | sed 's/^\([^:]\+\):.*$/\1/' | sort | uniq >> /requirements.apt; \ + done \ + && apt-mark showmanual | xargs apt-mark auto > /dev/null \ + && { [ -z "$savedAptMark" ] || apt-mark manual $savedAptMark; } \ + && /bin/true \ + && mkdir -p /var/lib/unit/ \ + && mkdir -p /docker-entrypoint.d/ \ + && apt-get update \ + && apt-get --no-install-recommends --no-install-suggests -y install curl $(cat /requirements.apt) \ + && apt-get purge -y --auto-remove build-essential \ + && rm -rf /var/lib/apt/lists/* \ + && rm -f /requirements.apt \ + && ln -sf /dev/stderr /var/log/unit.log + + # && groupadd --gid 999 unit \ + # && useradd \ + # --uid 999 \ + # --gid unit \ + # --no-create-home \ + # --home /nonexistent \ + # --comment "unit user" \ + # --shell /bin/false \ + # unit \ + +USER $MAMBA_USER +ARG FORCE_FROM_HERE=unknown +RUN /opt/conda/bin/pip install "git+${MAPGEN_REPO}@${MAPGEN_VERSION}" xncml +USER root +# RUN cd / +# RUN mkdir -pv /usr/local/var/lib/unit/ && chown $MAMBA_USER:$MAMBA_USER /usr/local/var/lib/unit/ +# RUN ls -al /usr/sbin/unitd-debug && /usr/sbin/unitd-debug --help WORKDIR /app #EXPOSE 80 - +ENV LD_LIBRARY_PATH=/usr/local/lib:/usr/local/lib/x86_64-linux-gnu:/lib/x86_64-linux-gnu:/usr/lib/x86_64-linux-gnu:/opt/conda/lib # Run the start script, it will check for an /app/prestart.sh script (e.g. for migrations) # And then will start Gunicorn with Uvicorn #CMD ["/start.sh"] -CMD ["/opt/conda/bin/python", "/opt/conda/lib/python3.12/site-packages/mapgen/main.py"] +#CMD ["/opt/conda/bin/python", "/opt/conda/lib/python3.12/site-packages/mapgen/main.py"] +COPY ./docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh +#USER root +#RUN chown "$MAMBA_USER":"$MAMBA_USER" /var/lib/unit +RUN mkdir -pv /usr/share/unit/welcome && chown -R $MAMBA_USER:$MAMBA_USER /usr/share/unit +COPY --chown=$MAMBA_USER:$MAMBA_USER welcome.* /usr/share/unit/welcome/ +#USER $MAMBA_USER +#RUN chown unit:unit /var/lib/unit +#RUN mkdir -pv /usr/share/unit/welcome && chown -R unit:unit /usr/share/unit +#COPY --chown=unit:unit welcome.* /usr/share/unit/welcome/ +#USER $MAMBA_USER +ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"] +#EXPOSE 80 +#RUN echo "After" +#USER unit +CMD ["unitd", "--no-daemon", "--control", "unix:/var/run/control.unit.sock", "--statedir", "/var/lib/unit", "--pid", "/var/run/unit.pid", "--log", "/var/log/unit.log"] +#USER $MAMBA_USER diff --git a/containers/fastapi/Dockerfile-http b/containers/fastapi/Dockerfile-http new file mode 100644 index 0000000..dcd1b5f --- /dev/null +++ b/containers/fastapi/Dockerfile-http @@ -0,0 +1,35 @@ +FROM docker.io/mambaorg/micromamba:1.5.1-jammy + +LABEL maintainer="trygveas@met.no" + +ENV MAPGEN_REPO=https://github.com/metno/mapgen-fastapi.git \ + MAPGEN_VERSION=httpserver + +COPY ./app /app +# Install dependencies: +COPY --chown=$MAMBA_USER:$MAMBA_USER environment.yaml /tmp/environment.yaml +RUN micromamba install -y -n base --file /tmp/environment.yaml + +USER root +RUN apt-get update && apt-get install -y --no-install-recommends \ + git lsof less\ + && rm -rf /var/lib/apt/lists/* +USER $MAMBA_USER +RUN /opt/conda/bin/pip install "git+${MAPGEN_REPO}@${MAPGEN_VERSION}" xncml + +#COPY --chown=$MAMBA_USER:$MAMBA_USER ./start.sh /start.sh + +#COPY --chown=$MAMBA_USER:$MAMBA_USER ./gunicorn_conf.py /gunicorn_conf.py + +#COPY --chown=$MAMBA_USER:$MAMBA_USER ./start-reload.sh /start-reload.sh +#RUN chmod +x /start.sh \ +# && chmod +x /start-reload.sh + +WORKDIR /app + +#EXPOSE 80 + +# Run the start script, it will check for an /app/prestart.sh script (e.g. for migrations) +# And then will start Gunicorn with Uvicorn +#CMD ["/start.sh"] +CMD ["/opt/conda/bin/python", "/opt/conda/lib/python3.12/site-packages/mapgen/main.py"] From a854d0d179f4638244b824d43ae40315c5e0bbf2 Mon Sep 17 00:00:00 2001 From: Trygve Aspenes Date: Fri, 25 Oct 2024 16:23:51 +0200 Subject: [PATCH 07/15] update Dockerfile --- containers/fastapi/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/containers/fastapi/Dockerfile b/containers/fastapi/Dockerfile index 44487bb..05e8cdd 100644 --- a/containers/fastapi/Dockerfile +++ b/containers/fastapi/Dockerfile @@ -84,7 +84,7 @@ RUN cd /usr/src/unit/unit \ && cd \ && rm -rf /usr/src/unit \ && for f in /usr/sbin/unitd /usr/lib/unit/modules/*.unit.so; do \ - ldd $f | awk '/=>/{print $(NF-1)}' | while read n; do dpkg-query -S $n; done | sed 's/^\([^:]\+\):.*$/\1/' | sort | uniq >> /requirements.apt; \ + ldd $f | awk '/=>/{print $(NF-1)}' | while read n; do dpkg-query -S `basename $n`; done | sed 's/^\([^:]\+\):.*$/\1/' | sort | uniq >> /requirements.apt; \ done \ && apt-mark showmanual | xargs apt-mark auto > /dev/null \ && { [ -z "$savedAptMark" ] || apt-mark manual $savedAptMark; } \ From 424d5bd039364fb15f13d2815982929296d44b32 Mon Sep 17 00:00:00 2001 From: Trygve Aspenes Date: Sat, 26 Oct 2024 09:45:21 +0200 Subject: [PATCH 08/15] adjust Dockefile --- containers/fastapi/Dockerfile | 13 ++++++------- containers/fastapi/environment.yaml | 3 ++- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/containers/fastapi/Dockerfile b/containers/fastapi/Dockerfile index 05e8cdd..12adc18 100644 --- a/containers/fastapi/Dockerfile +++ b/containers/fastapi/Dockerfile @@ -36,9 +36,10 @@ RUN micromamba install -y -n base --file /tmp/environment.yaml USER root RUN set -ex \ + && ln -s /opt/conda/bin/curl /usr/bin/curl \ && savedAptMark="$(apt-mark showmanual)" \ && apt-get update \ - && apt-get install --no-install-recommends --no-install-suggests -y ca-certificates git lsof less build-essential libssl-dev libpcre2-dev curl pkg-config procps \ + && apt-get install --no-install-recommends --no-install-suggests -y ca-certificates git lsof less build-essential libssl-dev libpcre2-dev pkg-config procps \ && mkdir -p /usr/lib/unit/modules /usr/lib/unit/debug-modules \ && mkdir -p /usr/src/unit \ && cd /usr/src/unit \ @@ -46,7 +47,7 @@ RUN set -ex \ && cd unit \ && NCPU="$(getconf _NPROCESSORS_ONLN)" \ && DEB_HOST_MULTIARCH="$(dpkg-architecture -q DEB_HOST_MULTIARCH)" \ - && CC_OPT="$(DEB_BUILD_MAINT_OPTIONS="hardening=+all,-pie" DEB_CFLAGS_MAINT_APPEND="-Wp,-D_FORTIFY_SOURCE=2 -fPIC" dpkg-buildflags --get CFLAGS)" \ + && CC_OPT="$(DEB_BUILD_MAINT_OPTIONS="hardening=+all,-pie" DEB_CFLAGS_MAINT_APPEND="-Wp,-D_FORTIFY_SOURCE=2 -fPIC $(dpkg-buildflags --get CFLAGS)")" \ && LD_OPT="$(DEB_BUILD_MAINT_OPTIONS="hardening=+all,-pie" DEB_LDFLAGS_MAINT_APPEND="-Wl,--as-needed -pie" dpkg-buildflags --get LDFLAGS)" \ && CONFIGURE_ARGS_MODULES="--prefix=/usr \ --statedir=/var/lib/unit \ @@ -63,8 +64,7 @@ RUN set -ex \ && CONFIGURE_ARGS="$CONFIGURE_ARGS_MODULES \ --njs" \ && make -j $NCPU -C pkg/contrib .njs \ - && export PKG_CONFIG_PATH=$(pwd)/pkg/contrib/njs/build -RUN cd /usr/src/unit/unit \ + && export PKG_CONFIG_PATH=$(pwd)/pkg/contrib/njs/build \ && ./configure $CONFIGURE_ARGS --cc-opt="$CC_OPT" --ld-opt="$LD_OPT" --modulesdir=/usr/lib/unit/debug-modules --debug \ && make -j $NCPU unitd \ && install -pm755 build/sbin/unitd /usr/sbin/unitd-debug \ @@ -92,7 +92,7 @@ RUN cd /usr/src/unit/unit \ && mkdir -p /var/lib/unit/ \ && mkdir -p /docker-entrypoint.d/ \ && apt-get update \ - && apt-get --no-install-recommends --no-install-suggests -y install curl $(cat /requirements.apt) \ + && apt-get --no-install-recommends --no-install-suggests -y install $(cat /requirements.apt) \ && apt-get purge -y --auto-remove build-essential \ && rm -rf /var/lib/apt/lists/* \ && rm -f /requirements.apt \ @@ -109,7 +109,6 @@ RUN cd /usr/src/unit/unit \ # unit \ USER $MAMBA_USER -ARG FORCE_FROM_HERE=unknown RUN /opt/conda/bin/pip install "git+${MAPGEN_REPO}@${MAPGEN_VERSION}" xncml USER root # RUN cd / @@ -118,7 +117,7 @@ USER root WORKDIR /app #EXPOSE 80 -ENV LD_LIBRARY_PATH=/usr/local/lib:/usr/local/lib/x86_64-linux-gnu:/lib/x86_64-linux-gnu:/usr/lib/x86_64-linux-gnu:/opt/conda/lib +ENV LD_LIBRARY_PATH=/usr/local/lib:/opt/conda/lib:/usr/local/lib/x86_64-linux-gnu:/lib/x86_64-linux-gnu:/usr/lib/x86_64-linux-gnu # Run the start script, it will check for an /app/prestart.sh script (e.g. for migrations) # And then will start Gunicorn with Uvicorn #CMD ["/start.sh"] diff --git a/containers/fastapi/environment.yaml b/containers/fastapi/environment.yaml index b06795a..4437bc6 100644 --- a/containers/fastapi/environment.yaml +++ b/containers/fastapi/environment.yaml @@ -15,7 +15,7 @@ dependencies: - pytest-mock - rasterio - pandas -- satpy=0.51.0 +- satpy=0.52.1 - mapserver=8.2.2 - gdal=3.9.2 #3.9.0 - libgdal=3.9.2 @@ -24,3 +24,4 @@ dependencies: - lxml - cmocean - pyspectral +- curl From 69c82b03cc9e2f6158757169aff925a797efb4b9 Mon Sep 17 00:00:00 2001 From: Trygve Aspenes Date: Sat, 26 Oct 2024 09:56:33 +0200 Subject: [PATCH 09/15] need endtrypoint --- docker-entrypoint.sh | 101 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100755 docker-entrypoint.sh diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh new file mode 100755 index 0000000..7254791 --- /dev/null +++ b/docker-entrypoint.sh @@ -0,0 +1,101 @@ +#!/bin/sh + +set -e + +WAITLOOPS=5 +SLEEPSEC=1 + +curl_put() +{ + RET=$(/usr/bin/curl -s -w '%{http_code}' -X PUT --data-binary @$1 --unix-socket /var/run/control.unit.sock http://localhost/$2) + RET_BODY=$(echo $RET | /bin/sed '$ s/...$//') + RET_STATUS=$(echo $RET | /usr/bin/tail -c 4) + if [ "$RET_STATUS" -ne "200" ]; then + echo "$0: Error: HTTP response status code is '$RET_STATUS'" + echo "$RET_BODY" + return 1 + else + echo "$0: OK: HTTP response status code is '$RET_STATUS'" + echo "$RET_BODY" + fi + return 0 +} + +if [ "$1" = "unitd" ] || [ "$1" = "unitd-debug" ]; then + if /usr/bin/find "/var/lib/unit/" -mindepth 1 -print -quit 2>/dev/null | /bin/grep -q .; then + echo "$0: /var/lib/unit/ is not empty, skipping initial configuration..." + else + echo "$0: Launching Unit daemon to perform initial configuration..." + /usr/sbin/$1 --control unix:/var/run/control.unit.sock --log /var/log/unit.log --statedir /var/lib/unit --pid /var/run/unit.pid + + for i in $(/usr/bin/seq $WAITLOOPS); do + if [ ! -S /var/run/control.unit.sock ]; then + echo "$0: Waiting for control socket to be created..." + /bin/sleep $SLEEPSEC + else + break + fi + done + # even when the control socket exists, it does not mean unit has finished initialisation + # this curl call will get a reply once unit is fully launched + /usr/bin/curl -s -X GET --unix-socket /var/run/control.unit.sock http://localhost/ + + if /usr/bin/find "/docker-entrypoint.d/" -mindepth 1 -print -quit 2>/dev/null | /bin/grep -q .; then + echo "$0: /docker-entrypoint.d/ is not empty, applying initial configuration..." + + echo "$0: Looking for certificate bundles in /docker-entrypoint.d/..." + for f in $(/usr/bin/find /docker-entrypoint.d/ -type f -name "*.pem"); do + echo "$0: Uploading certificates bundle: $f" + curl_put $f "certificates/$(basename $f .pem)" + done + + echo "$0: Looking for JavaScript modules in /docker-entrypoint.d/..." + for f in $(/usr/bin/find /docker-entrypoint.d/ -type f -name "*.js"); do + echo "$0: Uploading JavaScript module: $f" + curl_put $f "js_modules/$(basename $f .js)" + done + + echo "$0: Looking for configuration snippets in /docker-entrypoint.d/..." + for f in $(/usr/bin/find /docker-entrypoint.d/ -type f -name "*.json"); do + echo "$0: Applying configuration $f"; + curl_put $f "config" + done + + echo "$0: Looking for shell scripts in /docker-entrypoint.d/..." + for f in $(/usr/bin/find /docker-entrypoint.d/ -type f -name "*.sh"); do + echo "$0: Launching $f"; + "$f" + done + + # warn on filetypes we don't know what to do with + for f in $(/usr/bin/find /docker-entrypoint.d/ -type f -not -name "*.sh" -not -name "*.json" -not -name "*.pem" -not -name "*.js"); do + echo "$0: Ignoring $f"; + done + else + echo "$0: /docker-entrypoint.d/ is empty, creating 'welcome' configuration..." + curl_put /usr/share/unit/welcome/welcome.json "config" + fi + + echo "$0: Stopping Unit daemon after initial configuration..." + kill -TERM $(/bin/cat /var/run/unit.pid) + + for i in $(/usr/bin/seq $WAITLOOPS); do + if [ -S /var/run/control.unit.sock ]; then + echo "$0: Waiting for control socket to be removed..." + /bin/sleep $SLEEPSEC + else + break + fi + done + if [ -S /var/run/control.unit.sock ]; then + kill -KILL $(/bin/cat /var/run/unit.pid) + rm -f /var/run/control.unit.sock + fi + + echo + echo "$0: Unit initial configuration complete; ready for start up..." + echo + fi +fi + +exec "$@" From cbb046ed59ca80c015bf7801e03209041bb47502 Mon Sep 17 00:00:00 2001 From: Trygve Aspenes Date: Sat, 26 Oct 2024 10:20:10 +0200 Subject: [PATCH 10/15] need welcome? --- welcome.html | 45 +++++++++++++++++++++++++++++++++++++++++++++ welcome.json | 25 +++++++++++++++++++++++++ welcome.md | 29 +++++++++++++++++++++++++++++ 3 files changed, 99 insertions(+) create mode 100644 welcome.html create mode 100644 welcome.json create mode 100644 welcome.md diff --git a/welcome.html b/welcome.html new file mode 100644 index 0000000..7167ddb --- /dev/null +++ b/welcome.html @@ -0,0 +1,45 @@ + + + + Welcome to NGINX Unit + + + +

Welcome to NGINX Unit

+

Congratulations! NGINX Unit is installed and running.

+

Useful Links

+ + +

Next steps

+ +

Check Current Configuration

+
+

Unit's control API is currently listening for configuration changes + on the Unix socket at + /var/run/control.unit.sock inside the container.
+ To see the current configuration run:

+ 
docker exec -ti <containerID> curl --unix-socket /var/run/control.unit.sock http://localhost/config
+
+ +
+

NGINX Unit — the universal web app server
+ NGINX, Inc. © 2024

+ + diff --git a/welcome.json b/welcome.json new file mode 100644 index 0000000..2a148da --- /dev/null +++ b/welcome.json @@ -0,0 +1,25 @@ +{ + "listeners": { + "*:80": { + "pass": "routes" + } + }, + + "routes": [ + { + "match": { + "headers": { + "accept": "*text/html*" + } + }, + "action": { + "share": "/usr/share/unit/welcome/welcome.html" + } + }, + { + "action": { + "share": "/usr/share/unit/welcome/welcome.md" + } + } + ] +} diff --git a/welcome.md b/welcome.md new file mode 100644 index 0000000..fef3d15 --- /dev/null +++ b/welcome.md @@ -0,0 +1,29 @@ +Welcome to NGINX Unit +===================== + +Congratulations! NGINX Unit is installed and running. + +Useful Links +------------ + + * https://unit.nginx.org/ + - Get started with the 'Configuration' docs, starting with the 'Quick Start' guide. + + * https://unit.nginx.org/howto/docker/ + - Guidance for running Unit in a container and tips for containerized applications. + + * https://github.com/nginx/unit + - See our GitHub repo to browse the code, contribute, or seek help from the community. + +Current Configuration +--------------------- +Unit's control API is currently listening for configuration changes on the Unix socket at +`/var/run/control.unit.sock` inside the container. + +Read the current configuration with +``` +docker exec -ti curl --unix-socket /var/run/control.unit.sock http://localhost/config +``` + +--- +NGINX Unit - the universal web app server From ddbc3f6e7fb8102ec51c89fa6ac118455ccd21bc Mon Sep 17 00:00:00 2001 From: Trygve Aspenes Date: Sun, 27 Oct 2024 09:05:53 +0100 Subject: [PATCH 11/15] need specific dirs for k8s --- containers/fastapi/Dockerfile | 8 ++++---- docker-entrypoint.sh | 18 +++++++++--------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/containers/fastapi/Dockerfile b/containers/fastapi/Dockerfile index 12adc18..249f891 100644 --- a/containers/fastapi/Dockerfile +++ b/containers/fastapi/Dockerfile @@ -51,9 +51,9 @@ RUN set -ex \ && LD_OPT="$(DEB_BUILD_MAINT_OPTIONS="hardening=+all,-pie" DEB_LDFLAGS_MAINT_APPEND="-Wl,--as-needed -pie" dpkg-buildflags --get LDFLAGS)" \ && CONFIGURE_ARGS_MODULES="--prefix=/usr \ --statedir=/var/lib/unit \ - --control=unix:/var/run/control.unit.sock \ - --runstatedir=/var/run \ - --pid=/var/run/unit.pid \ + --control=unix:/var/run/unit/control.unit.sock \ + --runstatedir=/var/run/unit \ + --pid=/var/run/unit/unit.pid \ --logdir=/var/log \ --log=/var/log/unit.log \ --tmpdir=/var/tmp \ @@ -136,5 +136,5 @@ ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"] #EXPOSE 80 #RUN echo "After" #USER unit -CMD ["unitd", "--no-daemon", "--control", "unix:/var/run/control.unit.sock", "--statedir", "/var/lib/unit", "--pid", "/var/run/unit.pid", "--log", "/var/log/unit.log"] +CMD ["unitd", "--no-daemon", "--control", "unix:/var/run/unit/control.unit.sock", "--statedir", "/var/lib/unit", "--pid", "/var/run/unit/unit.pid", "--log", "/var/log/unit.log"] #USER $MAMBA_USER diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index 7254791..ab2e335 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -7,7 +7,7 @@ SLEEPSEC=1 curl_put() { - RET=$(/usr/bin/curl -s -w '%{http_code}' -X PUT --data-binary @$1 --unix-socket /var/run/control.unit.sock http://localhost/$2) + RET=$(/usr/bin/curl -s -w '%{http_code}' -X PUT --data-binary @$1 --unix-socket /var/run/unit/control.unit.sock http://localhost/$2) RET_BODY=$(echo $RET | /bin/sed '$ s/...$//') RET_STATUS=$(echo $RET | /usr/bin/tail -c 4) if [ "$RET_STATUS" -ne "200" ]; then @@ -26,10 +26,10 @@ if [ "$1" = "unitd" ] || [ "$1" = "unitd-debug" ]; then echo "$0: /var/lib/unit/ is not empty, skipping initial configuration..." else echo "$0: Launching Unit daemon to perform initial configuration..." - /usr/sbin/$1 --control unix:/var/run/control.unit.sock --log /var/log/unit.log --statedir /var/lib/unit --pid /var/run/unit.pid + /usr/sbin/$1 --control unix:/var/run/unit/control.unit.sock --log /var/log/unit.log --statedir /var/lib/unit --pid /var/run/unit/unit.pid for i in $(/usr/bin/seq $WAITLOOPS); do - if [ ! -S /var/run/control.unit.sock ]; then + if [ ! -S /var/run/unit/control.unit.sock ]; then echo "$0: Waiting for control socket to be created..." /bin/sleep $SLEEPSEC else @@ -38,7 +38,7 @@ if [ "$1" = "unitd" ] || [ "$1" = "unitd-debug" ]; then done # even when the control socket exists, it does not mean unit has finished initialisation # this curl call will get a reply once unit is fully launched - /usr/bin/curl -s -X GET --unix-socket /var/run/control.unit.sock http://localhost/ + /usr/bin/curl -s -X GET --unix-socket /var/run/unit/control.unit.sock http://localhost/ if /usr/bin/find "/docker-entrypoint.d/" -mindepth 1 -print -quit 2>/dev/null | /bin/grep -q .; then echo "$0: /docker-entrypoint.d/ is not empty, applying initial configuration..." @@ -77,19 +77,19 @@ if [ "$1" = "unitd" ] || [ "$1" = "unitd-debug" ]; then fi echo "$0: Stopping Unit daemon after initial configuration..." - kill -TERM $(/bin/cat /var/run/unit.pid) + kill -TERM $(/bin/cat /var/run/unit/unit.pid) for i in $(/usr/bin/seq $WAITLOOPS); do - if [ -S /var/run/control.unit.sock ]; then + if [ -S /var/run/unit/control.unit.sock ]; then echo "$0: Waiting for control socket to be removed..." /bin/sleep $SLEEPSEC else break fi done - if [ -S /var/run/control.unit.sock ]; then - kill -KILL $(/bin/cat /var/run/unit.pid) - rm -f /var/run/control.unit.sock + if [ -S /var/run/unit/control.unit.sock ]; then + kill -KILL $(/bin/cat /var/run/unit/unit.pid) + rm -f /var/run/unit/control.unit.sock fi echo From 51b818bbde3176b27afae518f320fe8eb9b2e372 Mon Sep 17 00:00:00 2001 From: Trygve Aspenes Date: Sun, 27 Oct 2024 10:00:47 +0100 Subject: [PATCH 12/15] more debug --- docker-entrypoint.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index ab2e335..e54662b 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -22,9 +22,13 @@ curl_put() } if [ "$1" = "unitd" ] || [ "$1" = "unitd-debug" ]; then + echo "$0: $(ls -al /var/lib/unit/)" + /usr/bin/find "/var/lib/unit/" -mindepth 1 -print -quit 2>/dev/null + echo "After $?" if /usr/bin/find "/var/lib/unit/" -mindepth 1 -print -quit 2>/dev/null | /bin/grep -q .; then echo "$0: /var/lib/unit/ is not empty, skipping initial configuration..." - else + fi + if [ 1 ]; then echo "$0: Launching Unit daemon to perform initial configuration..." /usr/sbin/$1 --control unix:/var/run/unit/control.unit.sock --log /var/log/unit.log --statedir /var/lib/unit --pid /var/run/unit/unit.pid From a05ef3eab29988c9bc06db33e80117e9da5c69ed Mon Sep 17 00:00:00 2001 From: Trygve Aspenes Date: Mon, 28 Oct 2024 09:06:47 +0100 Subject: [PATCH 13/15] log as separate dir --- containers/fastapi/Dockerfile | 2 +- docker-entrypoint.sh | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/containers/fastapi/Dockerfile b/containers/fastapi/Dockerfile index 249f891..b935039 100644 --- a/containers/fastapi/Dockerfile +++ b/containers/fastapi/Dockerfile @@ -136,5 +136,5 @@ ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"] #EXPOSE 80 #RUN echo "After" #USER unit -CMD ["unitd", "--no-daemon", "--control", "unix:/var/run/unit/control.unit.sock", "--statedir", "/var/lib/unit", "--pid", "/var/run/unit/unit.pid", "--log", "/var/log/unit.log"] +CMD ["unitd", "--no-daemon", "--control", "unix:/var/run/unit/control.unit.sock", "--statedir", "/var/lib/unit", "--pid", "/var/run/unit/unit.pid", "--log", "/var/log/unit/unit.log"] #USER $MAMBA_USER diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index e54662b..5f14d1c 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -27,10 +27,9 @@ if [ "$1" = "unitd" ] || [ "$1" = "unitd-debug" ]; then echo "After $?" if /usr/bin/find "/var/lib/unit/" -mindepth 1 -print -quit 2>/dev/null | /bin/grep -q .; then echo "$0: /var/lib/unit/ is not empty, skipping initial configuration..." - fi - if [ 1 ]; then + else echo "$0: Launching Unit daemon to perform initial configuration..." - /usr/sbin/$1 --control unix:/var/run/unit/control.unit.sock --log /var/log/unit.log --statedir /var/lib/unit --pid /var/run/unit/unit.pid + /usr/sbin/$1 --control unix:/var/run/unit/control.unit.sock --log /var/log/unit/unit.log --statedir /var/lib/unit --pid /var/run/unit/unit.pid for i in $(/usr/bin/seq $WAITLOOPS); do if [ ! -S /var/run/unit/control.unit.sock ]; then From be352129ee271b1b2a9f753f506a707d3efe1ff8 Mon Sep 17 00:00:00 2001 From: Trygve Aspenes Date: Mon, 28 Oct 2024 09:08:04 +0100 Subject: [PATCH 14/15] log as separate dir --- containers/fastapi/Dockerfile | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/containers/fastapi/Dockerfile b/containers/fastapi/Dockerfile index b935039..3aff24f 100644 --- a/containers/fastapi/Dockerfile +++ b/containers/fastapi/Dockerfile @@ -54,8 +54,8 @@ RUN set -ex \ --control=unix:/var/run/unit/control.unit.sock \ --runstatedir=/var/run/unit \ --pid=/var/run/unit/unit.pid \ - --logdir=/var/log \ - --log=/var/log/unit.log \ + --logdir=/var/log/unit \ + --log=/var/log/unit/unit.log \ --tmpdir=/var/tmp \ --user=$MAMBA_USER \ --group=$MAMBA_USER \ @@ -95,8 +95,7 @@ RUN set -ex \ && apt-get --no-install-recommends --no-install-suggests -y install $(cat /requirements.apt) \ && apt-get purge -y --auto-remove build-essential \ && rm -rf /var/lib/apt/lists/* \ - && rm -f /requirements.apt \ - && ln -sf /dev/stderr /var/log/unit.log + && rm -f /requirements.apt # && groupadd --gid 999 unit \ # && useradd \ From fa8dda542885d34210f060578ad776ad7295a0e2 Mon Sep 17 00:00:00 2001 From: Trygve Aspenes Date: Mon, 28 Oct 2024 12:28:25 +0100 Subject: [PATCH 15/15] clean up, log to stdout --- containers/fastapi/Dockerfile | 57 ++--------------------------------- docker-entrypoint.sh | 8 ++--- 2 files changed, 6 insertions(+), 59 deletions(-) diff --git a/containers/fastapi/Dockerfile b/containers/fastapi/Dockerfile index 3aff24f..df35b0c 100644 --- a/containers/fastapi/Dockerfile +++ b/containers/fastapi/Dockerfile @@ -4,36 +4,12 @@ FROM docker.io/mambaorg/micromamba:2.0.2-ubuntu24.04 LABEL maintainer="trygveas@met.no" ENV MAPGEN_REPO=https://github.com/metno/mapgen-fastapi.git \ - MAPGEN_VERSION=httpserver + MAPGEN_VERSION=main -#COPY ./app /app # Install dependencies: COPY --chown=$MAMBA_USER:$MAMBA_USER containers/fastapi/environment.yaml /tmp/environment.yaml RUN micromamba install -y -n base --file /tmp/environment.yaml -# USER root -# RUN apt-get update && apt-get install -y --no-install-recommends \ -# git lsof less curl \ -# && rm -rf /var/lib/apt/lists/* -#RUN curl --output /usr/share/keyrings/nginx-keyring.gpg https://unit.nginx.org/keys/nginx-keyring.gpg \ -# && echo "deb [signed-by=/usr/share/keyrings/nginx-keyring.gpg] https://packages.nginx.org/unit/ubuntu/ noble unit\ndeb-src [signed-by=/usr/share/keyrings/nginx-keyring.gpg] https://packages.nginx.org/unit/ubuntu/ noble unit" > /etc/apt/sources.list.d/unit.list \ -# && apt-get update \ -# && apt-cache search unit \ -# && apt-get install -y unit-python3.12 - -#COPY --chown=$MAMBA_USER:$MAMBA_USER ./start.sh /start.sh - -#COPY --chown=$MAMBA_USER:$MAMBA_USER ./gunicorn_conf.py /gunicorn_conf.py - -#COPY --chown=$MAMBA_USER:$MAMBA_USER ./start-reload.sh /start-reload.sh -#RUN chmod +x /start.sh \ -# && chmod +x /start-reload.sh - -# USER root -# RUN apt-get update && apt-get install -y --no-install-recommends \ -# git lsof less curl \ -# && rm -rf /var/lib/apt/lists/* - USER root RUN set -ex \ && ln -s /opt/conda/bin/curl /usr/bin/curl \ @@ -97,43 +73,14 @@ RUN set -ex \ && rm -rf /var/lib/apt/lists/* \ && rm -f /requirements.apt - # && groupadd --gid 999 unit \ - # && useradd \ - # --uid 999 \ - # --gid unit \ - # --no-create-home \ - # --home /nonexistent \ - # --comment "unit user" \ - # --shell /bin/false \ - # unit \ - USER $MAMBA_USER RUN /opt/conda/bin/pip install "git+${MAPGEN_REPO}@${MAPGEN_VERSION}" xncml USER root -# RUN cd / -# RUN mkdir -pv /usr/local/var/lib/unit/ && chown $MAMBA_USER:$MAMBA_USER /usr/local/var/lib/unit/ -# RUN ls -al /usr/sbin/unitd-debug && /usr/sbin/unitd-debug --help WORKDIR /app -#EXPOSE 80 ENV LD_LIBRARY_PATH=/usr/local/lib:/opt/conda/lib:/usr/local/lib/x86_64-linux-gnu:/lib/x86_64-linux-gnu:/usr/lib/x86_64-linux-gnu -# Run the start script, it will check for an /app/prestart.sh script (e.g. for migrations) -# And then will start Gunicorn with Uvicorn -#CMD ["/start.sh"] -#CMD ["/opt/conda/bin/python", "/opt/conda/lib/python3.12/site-packages/mapgen/main.py"] COPY ./docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh -#USER root -#RUN chown "$MAMBA_USER":"$MAMBA_USER" /var/lib/unit RUN mkdir -pv /usr/share/unit/welcome && chown -R $MAMBA_USER:$MAMBA_USER /usr/share/unit COPY --chown=$MAMBA_USER:$MAMBA_USER welcome.* /usr/share/unit/welcome/ -#USER $MAMBA_USER -#RUN chown unit:unit /var/lib/unit -#RUN mkdir -pv /usr/share/unit/welcome && chown -R unit:unit /usr/share/unit -#COPY --chown=unit:unit welcome.* /usr/share/unit/welcome/ -#USER $MAMBA_USER ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"] -#EXPOSE 80 -#RUN echo "After" -#USER unit -CMD ["unitd", "--no-daemon", "--control", "unix:/var/run/unit/control.unit.sock", "--statedir", "/var/lib/unit", "--pid", "/var/run/unit/unit.pid", "--log", "/var/log/unit/unit.log"] -#USER $MAMBA_USER +CMD ["unitd", "--no-daemon", "--control", "unix:/var/run/unit/control.unit.sock", "--statedir", "/var/lib/unit", "--pid", "/var/run/unit/unit.pid", "--log", "/proc/1/fd/1"] diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index 5f14d1c..91adccb 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -22,14 +22,14 @@ curl_put() } if [ "$1" = "unitd" ] || [ "$1" = "unitd-debug" ]; then - echo "$0: $(ls -al /var/lib/unit/)" - /usr/bin/find "/var/lib/unit/" -mindepth 1 -print -quit 2>/dev/null - echo "After $?" + mkdir -pv /var/lib/unit + mkdir -pv /var/run/unit + mkdir -pv /var/log/unit if /usr/bin/find "/var/lib/unit/" -mindepth 1 -print -quit 2>/dev/null | /bin/grep -q .; then echo "$0: /var/lib/unit/ is not empty, skipping initial configuration..." else echo "$0: Launching Unit daemon to perform initial configuration..." - /usr/sbin/$1 --control unix:/var/run/unit/control.unit.sock --log /var/log/unit/unit.log --statedir /var/lib/unit --pid /var/run/unit/unit.pid + /usr/sbin/$1 --control unix:/var/run/unit/control.unit.sock --log /proc/1/fd/1 --statedir /var/lib/unit --pid /var/run/unit/unit.pid for i in $(/usr/bin/seq $WAITLOOPS); do if [ ! -S /var/run/unit/control.unit.sock ]; then