diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 77590eb1..9721262a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -16,7 +16,13 @@ jobs: cache: pip cache-dependency-path: setup.py - run: pip install -e .[test] - - run: pytest -W error -W ignore::ResourceWarning tests --cov scrapyd + # Python 3.12 deprecates pkg_resources (also used by py-html-checker). + - run: | + pytest -W error -W ignore::ResourceWarning \ + -W ignore::DeprecationWarning:scrapyd.runner \ + -W ignore::DeprecationWarning:html_checker \ + -W ignore::DeprecationWarning:pkg_resources \ + tests --cov scrapyd # Occasional "ConnectionRefusedError: [Errno 111] Connection refused". - name: Run integration tests run: | diff --git a/scrapyd/launcher.py b/scrapyd/launcher.py index b85b5580..d619e25b 100644 --- a/scrapyd/launcher.py +++ b/scrapyd/launcher.py @@ -55,11 +55,11 @@ def _get_message(self, slot): def _spawn_process(self, message, slot): project = message["_project"] - environ = self.app.getComponent(IEnvironment) + environment = self.app.getComponent(IEnvironment) message.setdefault("settings", {}) - message["settings"].update(environ.get_settings(message)) + message["settings"].update(environment.get_settings(message)) - env = environ.get_environment(message, slot) + env = environment.get_environment(message, slot) args = [sys.executable, "-m", self.runner, "crawl", *get_crawl_args(message)] process = ScrapyProcessProtocol(project, message["_spider"], message["_job"], env, args) diff --git a/scrapyd/sqlite.py b/scrapyd/sqlite.py index 5e060d0b..bedb7795 100644 --- a/scrapyd/sqlite.py +++ b/scrapyd/sqlite.py @@ -1,7 +1,7 @@ +import datetime import json import os import sqlite3 -from datetime import datetime # The database argument is "jobs" (in SqliteJobStorage), or a project (in SqliteSpiderQueue) from get_spider_queues(), @@ -19,6 +19,19 @@ def initialize(cls, config, database, table): return cls(connection_string, table) +# https://docs.python.org/3/library/sqlite3.html#sqlite3-adapter-converter-recipes +def adapt_datetime(val): + return val.strftime("%Y-%m-%d %H:%M:%S.%f") + + +def convert_datetime(val): + return datetime.datetime.strptime(val.decode(), "%Y-%m-%d %H:%M:%S.%f") + + +sqlite3.register_adapter(datetime.datetime, adapt_datetime) +sqlite3.register_converter("datetime", convert_datetime) + + class SqliteMixin: def __init__(self, database, table): self.database = database or ":memory:" @@ -139,8 +152,8 @@ def __iter__(self): project, spider, job, - datetime.strptime(start_time, "%Y-%m-%d %H:%M:%S.%f"), - datetime.strptime(end_time, "%Y-%m-%d %H:%M:%S.%f"), + datetime.datetime.strptime(start_time, "%Y-%m-%d %H:%M:%S.%f"), + datetime.datetime.strptime(end_time, "%Y-%m-%d %H:%M:%S.%f"), ) for project, spider, job, start_time, end_time in self.conn.execute( f"SELECT project, spider, job, start_time, end_time FROM {self.table} ORDER BY end_time DESC"