From 2b606635161f685403962ee7b1a904ceffcb1866 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Wed, 11 Sep 2024 00:17:15 -0700 Subject: [PATCH] Functional with Django 4.0 --- setup.cfg | 2 - src/servestatic/asgi.py | 8 +-- src/servestatic/middleware.py | 24 +++---- src/servestatic/responders.py | 26 +++----- src/servestatic/utils.py | 111 +++++++++++++++++++++++++++++--- tests/test_asgi.py | 8 +-- tests/test_django_whitenoise.py | 7 +- 7 files changed, 133 insertions(+), 53 deletions(-) diff --git a/setup.cfg b/setup.cfg index fac6dc3..6cde17e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -26,8 +26,6 @@ project_urls = [options] packages = find: -install_requires = - aiofiles>=22.1.0 python_requires = >=3.9 include_package_data = True package_dir = diff --git a/src/servestatic/asgi.py b/src/servestatic/asgi.py index daa0157..0550bf2 100644 --- a/src/servestatic/asgi.py +++ b/src/servestatic/asgi.py @@ -5,10 +5,7 @@ from asgiref.compatibility import guarantee_single_callable from servestatic.base import BaseServeStatic -from servestatic.utils import decode_path_info - -# This is the same size as wsgiref.FileWrapper -BLOCK_SIZE = 8192 +from servestatic.utils import decode_path_info, get_block_size class ServeStaticASGI(BaseServeStatic): @@ -42,6 +39,7 @@ class FileServerASGI: def __init__(self, static_file): self.static_file = static_file + self.block_size = get_block_size() async def __call__(self, scope, receive, send): # Convert ASGI headers into WSGI headers. Allows us to reuse all of our WSGI @@ -75,7 +73,7 @@ async def __call__(self, scope, receive, send): # Stream the file response body async with response.file as async_file: while True: - chunk = await async_file.read(BLOCK_SIZE) + chunk = await async_file.read(self.block_size) more_body = bool(chunk) await send( { diff --git a/src/servestatic/middleware.py b/src/servestatic/middleware.py index 4a5ba09..946fa6b 100644 --- a/src/servestatic/middleware.py +++ b/src/servestatic/middleware.py @@ -7,7 +7,6 @@ from urllib.request import url2pathname import django -from aiofiles.base import AiofilesContextManager from asgiref.sync import iscoroutinefunction, markcoroutinefunction from django.conf import settings as django_settings from django.contrib.staticfiles import finders @@ -15,22 +14,23 @@ ManifestStaticFilesStorage, staticfiles_storage, ) -from django.http import FileResponse +from django.http import FileResponse, HttpRequest -from servestatic.responders import MissingFileError +from servestatic.responders import MissingFileError, StaticFile from servestatic.utils import ( + AsyncFile, AsyncFileIterator, AsyncToSyncIterator, EmptyAsyncIterator, ensure_leading_trailing_slash, stat_files, ) -from servestatic.wsgi import ServeStatic +from servestatic.wsgi import BaseServeStatic __all__ = ["ServeStaticMiddleware"] -class ServeStaticMiddleware(ServeStatic): +class ServeStaticMiddleware(BaseServeStatic): """ Wrap ServeStatic to allow it to function as Django middleware, rather than ASGI/WSGI middleware. @@ -133,7 +133,7 @@ async def __call__(self, request): return await self.get_response(request) @staticmethod - async def aserve(static_file, request): + async def aserve(static_file: StaticFile, request: HttpRequest): response = await static_file.aget_response(request.method, request.META) status = int(response.status) http_response = AsyncServeStaticFileResponse( @@ -263,12 +263,12 @@ def set_headers(self, *args, **kwargs): pass def _set_streaming_content(self, value): - if isinstance(value, AiofilesContextManager): - value = AsyncFileIterator(value) - - # Django < 4.2 doesn't support async file responses, so we convert to sync - if django.VERSION < (4, 2) and hasattr(value, "__aiter__"): - value = AsyncToSyncIterator(value) + if isinstance(value, AsyncFile): + # Django < 4.2 doesn't support async file responses, so we use a sync file handle + if django.VERSION < (4, 2): + value = value.open_raw() + else: + value = AsyncFileIterator(value) super()._set_streaming_content(value) diff --git a/src/servestatic/responders.py b/src/servestatic/responders.py index 9fe3c8e..39fbdf9 100644 --- a/src/servestatic/responders.py +++ b/src/servestatic/responders.py @@ -12,9 +12,7 @@ from urllib.parse import quote from wsgiref.headers import Headers -import aiofiles -from aiofiles.base import AiofilesContextManager -from aiofiles.threadpool.binary import AsyncBufferedIOBase +from servestatic.utils import AsyncFile class Response: @@ -74,22 +72,16 @@ def close(self): class AsyncSlicedFile: """ - Variant of `SlicedFile` that works as an async context manager for `aiofiles`. - - This class does not need a `close` or `__await__` method, since we always open - async file handle via context managers (`async with`). + Variant of `SlicedFile` that works on async files. """ - def __init__(self, context_manager: AiofilesContextManager, start: int, end: int): - self.fileobj: AsyncBufferedIOBase # This is populated during `__aenter__` + def __init__(self, fileobj: AsyncFile, start: int, end: int): + self.fileobj = fileobj self.seeked = False self.start = start self.remaining = end - start + 1 - self.context_manager = context_manager async def read(self, size=-1): - if not self.fileobj: # pragma: no cover - raise RuntimeError("Async file objects need to be open via `async with`.") if not self.seeked: await self.fileobj.seek(self.start) self.seeked = True @@ -100,12 +92,14 @@ async def read(self, size=-1): self.remaining -= len(data) return data + async def close(self): + await self.fileobj.close() + async def __aenter__(self): - self.fileobj = await self.context_manager.__aenter__() return self - async def __aexit__(self, exc_type, exc, tb): - return await self.context_manager.__aexit__(exc_type, exc, tb) + async def __aexit__(self, exc_type, exc_val, exc_tb): + await self.close() class StaticFile: @@ -143,7 +137,7 @@ async def aget_response(self, method, request_headers): path, headers = self.get_path_and_headers(request_headers) # We do not await this async file handle to allow us the option of opening # it in a thread later - file_handle = aiofiles.open(path, "rb") if method != "HEAD" else None + file_handle = AsyncFile(path, "rb") if method != "HEAD" else None range_header = request_headers.get("HTTP_RANGE") if range_header: # If we can't interpret the Range request for any reason then diff --git a/src/servestatic/utils.py b/src/servestatic/utils.py index 103591a..a5eff36 100644 --- a/src/servestatic/utils.py +++ b/src/servestatic/utils.py @@ -3,10 +3,19 @@ import asyncio import concurrent.futures import contextlib +import functools import os -from typing import AsyncIterable +import threading +from concurrent.futures import ThreadPoolExecutor +from io import IOBase +from typing import AsyncIterable, Callable -from aiofiles.base import AiofilesContextManager +# This is the same size as wsgiref.FileWrapper +ASGI_BLOCK_SIZE = 8192 + + +def get_block_size(): + return ASGI_BLOCK_SIZE # Follow Django in treating URLs as UTF-8 encoded (which requires undoing the @@ -72,6 +81,90 @@ def __iter__(self): thread_executor.shutdown(wait=False) +class AsyncFile: + """A class that wraps a file object and provides async methods for reading and writing. + This currently only covers the file operations needed by ServeStatic, but could be expanded + in the future.""" + + def __init__( + self, + file_path, + mode: str = "r", + buffering: int = -1, + encoding: str | None = None, + errors: str | None = None, + newline: str | None = None, + closefd: bool = True, + opener: Callable[[str, int], int] | None = None, + ): + self.open_args = ( + file_path, + mode, + buffering, + encoding, + errors, + newline, + closefd, + opener, + ) + self.loop: asyncio.AbstractEventLoop | None = None + self.executor = ThreadPoolExecutor( + max_workers=1, thread_name_prefix="ServeStatic-AsyncFile" + ) + self.lock = threading.Lock() + self.file_obj: None | IOBase = None + self.closed = False + + async def _executor(self, func, *args): + """Run a function in a dedicated thread, specific to this instance.""" + if self.loop is None: + self.loop = asyncio.get_event_loop() + with self.lock: + return await self.loop.run_in_executor(self.executor, func, *args) + + @staticmethod + def open_lazy(f): + """Decorator that ensures the file is open before calling a function.""" + + @functools.wraps(f) + async def wrapper(self: "AsyncFile", *args, **kwargs): + if self.closed: + raise ValueError("I/O operation on closed file.") + if self.file_obj is None: + self.file_obj = await self._executor(open, *self.open_args) + return await f(self, *args, **kwargs) + + return wrapper + + def open_raw(self): + """Open the file without using the executor.""" + self.executor.shutdown(wait=True) + return open(*self.open_args) # pylint: disable=unspecified-encoding + + async def close(self): + self.closed = True + if self.file_obj: + await self._executor(self.file_obj.close) + + @open_lazy + async def read(self, size=-1): + return await self._executor(self.file_obj.read, size) + + @open_lazy + async def seek(self, offset, whence=0): + return await self._executor(self.file_obj.seek, offset, whence) + + @open_lazy + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb): + await self.close() + + def __del__(self): + self.executor.shutdown(wait=True) + + class EmptyAsyncIterator: """Placeholder async iterator for responses that have no content.""" @@ -83,17 +176,15 @@ async def __anext__(self): class AsyncFileIterator: - def __init__(self, file_context: AiofilesContextManager): - self.file_context = file_context + """Async iterator that yields chunks of data from the provided async file.""" - async def __aiter__(self): - """Async iterator compatible with Django Middleware. Yields chunks of data from - the provided async file context manager.""" - from servestatic.asgi import BLOCK_SIZE + def __init__(self, async_file: AsyncFile): + self.async_file = async_file - async with self.file_context as async_file: + async def __aiter__(self): + async with self.async_file as async_file: while True: - chunk = await async_file.read(BLOCK_SIZE) + chunk = await async_file.read(get_block_size()) if not chunk: break yield chunk diff --git a/tests/test_asgi.py b/tests/test_asgi.py index 860bffd..6d10972 100644 --- a/tests/test_asgi.py +++ b/tests/test_asgi.py @@ -82,13 +82,13 @@ def test_small_block_size(application, test_files): scope = AsgiScopeEmulator({"path": "/static/app.js"}) receive = AsgiReceiveEmulator() send = AsgiSendEmulator() - from servestatic import asgi + from servestatic import utils - DEFAULT_BLOCK_SIZE = asgi.BLOCK_SIZE - asgi.BLOCK_SIZE = 10 + default_block_size = utils.ASGI_BLOCK_SIZE + utils.ASGI_BLOCK_SIZE = 10 asyncio.run(application(scope, receive, send)) assert send[1]["body"] == test_files.js_content[:10] - asgi.BLOCK_SIZE = DEFAULT_BLOCK_SIZE + utils.ASGI_BLOCK_SIZE = default_block_size def test_request_range_response(application, test_files): diff --git a/tests/test_django_whitenoise.py b/tests/test_django_whitenoise.py index 428e9b7..6d81b81 100644 --- a/tests/test_django_whitenoise.py +++ b/tests/test_django_whitenoise.py @@ -8,7 +8,6 @@ from pathlib import Path from urllib.parse import urljoin, urlparse -import aiofiles import brotli import django import pytest @@ -22,6 +21,7 @@ from django.utils.functional import empty from servestatic.middleware import AsyncServeStaticFileResponse, ServeStaticMiddleware +from servestatic.utils import AsyncFile from .utils import ( AppServer, @@ -279,11 +279,10 @@ def test_directory_path_without_trailing_slash_redirected( def test_servestatic_file_response_has_only_one_header(): - response = AsyncServeStaticFileResponse(aiofiles.open(__file__, "rb")) + response = AsyncServeStaticFileResponse(AsyncFile(__file__, "rb")) response.close() headers = {key.lower() for key, value in response.items()} - # This subclass should have none of the default headers that FileReponse - # sets + # This subclass should have none of the default headers that FileReponse sets assert headers == {"content-type"}