From d13e2733b125d7b0633939521a6960bbf5769345 Mon Sep 17 00:00:00 2001 From: Tom Parker-Shemilt Date: Sat, 11 May 2024 13:04:00 +0100 Subject: [PATCH] Fix various linter issues --- billiard-stubs/common.pyi | 9 +-- billiard-stubs/compat.pyi | 19 ++++-- billiard-stubs/connection.pyi | 52 +++------------ billiard-stubs/context.pyi | 15 +++-- billiard-stubs/dummy/__init__.pyi | 24 +++++-- billiard-stubs/pool.pyi | 94 +++++++-------------------- billiard-stubs/process.pyi | 10 +-- billiard-stubs/util.pyi | 14 +++- celery-stubs/app/base.pyi | 2 +- celery-stubs/beat.pyi | 50 ++++---------- celery-stubs/concurrency/asynpool.pyi | 25 ++----- celery-stubs/concurrency/prefork.pyi | 4 +- celery-stubs/worker/components.pyi | 2 +- celery-stubs/worker/worker.pyi | 2 +- 14 files changed, 116 insertions(+), 206 deletions(-) diff --git a/billiard-stubs/common.pyi b/billiard-stubs/common.pyi index d4b6204..4bd64e1 100644 --- a/billiard-stubs/common.pyi +++ b/billiard-stubs/common.pyi @@ -1,6 +1,7 @@ import pickle -from .exceptions import RestartFreqExceeded as _RestartFreqExceeded -from signal import _SIGNUM, _HANDLER, Signals +from signal import _HANDLER, _SIGNUM, Signals + +from billiard.exceptions import RestartFreqExceeded as _RestartFreqExceeded pickle_load = pickle.load pickle_loads = pickle.loads @@ -17,11 +18,11 @@ TERMSIGS_FULL: set[str] def human_status(status: int) -> str: ... def maybe_setsignal(signum: _SIGNUM, handler: _HANDLER) -> None: ... def signum(sig: int) -> _SIGNUM: ... -def reset_signals(handler:_HANDLER=..., full: bool = False) -> None: ... +def reset_signals(handler: _HANDLER = ..., full: bool = ...) -> None: ... class restart_state: RestartFreqExceeded = _RestartFreqExceeded def __init__(self, maxR: int, maxT: int) -> None: ... R: int T: float | None - def step(self, now: float | None = None) -> None: ... + def step(self, now: float | None = ...) -> None: ... diff --git a/billiard-stubs/compat.pyi b/billiard-stubs/compat.pyi index 631f6ae..66169da 100644 --- a/billiard-stubs/compat.pyi +++ b/billiard-stubs/compat.pyi @@ -1,28 +1,35 @@ import io import numbers import os -from _typeshed import ReadableBuffer, StrOrBytesPath, FileDescriptorLike -from typing import Iterator, Sequence, TypeVar, SupportsInt +from collections.abc import Iterator, Sequence +from typing import SupportsInt, TypeVar + +from _typeshed import FileDescriptorLike, ReadableBuffer, StrOrBytesPath FILENO_ERRORS: tuple[Exception] __write__ = os.write def send_offset(fd: int, buf: ReadableBuffer, offset: int) -> int: ... + fsencode = os.fsencode fsdecode = os.fsdecode -MaybeFileNo = numbers.Integral| io.IOBase +MaybeFileNo = numbers.Integral | io.IOBase + def maybe_fileno(f: MaybeFileNo) -> numbers.Integral: ... -def get_fdmax(default: int | None = None) -> int|None: ... +def get_fdmax(default: int | None = ...) -> int | None: ... T = TypeVar("T") + def uniq(it: Sequence[T]) -> Iterator[T]: ... closerange = os.closerange -def close_open_fds(keep: Sequence[MaybeFileNo] | None = None) -> None: ... +def close_open_fds(keep: Sequence[MaybeFileNo] | None = ...) -> None: ... def get_errno(exc: Exception | None) -> int: ... -def spawnv_passfds(path: StrOrBytesPath, args: os._ExecVArgs, passfds: Sequence[MaybeFileNo]) -> int: ... +def spawnv_passfds( + path: StrOrBytesPath, args: os._ExecVArgs, passfds: Sequence[MaybeFileNo] +) -> int: ... def isblocking(handle: FileDescriptorLike) -> bool: ... def setblocking(handle: FileDescriptorLike, blocking: bool) -> None: ... diff --git a/billiard-stubs/connection.pyi b/billiard-stubs/connection.pyi index 312ff1b..9c9641b 100644 --- a/billiard-stubs/connection.pyi +++ b/billiard-stubs/connection.pyi @@ -1,59 +1,23 @@ -from _typeshed import Incomplete +__all__ = ["Listener"] -__all__ = ['Client', 'Listener', 'Pipe', 'wait'] - -class _SocketContainer: - sock: Incomplete - def __init__(self, sock) -> None: ... +class _SocketContainer: ... class _ConnectionBase: - def __init__(self, handle, readable: bool = True, writable: bool = True) -> None: ... def __del__(self) -> None: ... - @property - def closed(self): ... - @property - def readable(self): ... - @property - def writable(self): ... - def fileno(self): ... def close(self) -> None: ... - def send_bytes(self, buf, offset: int = 0, size: Incomplete | None = None) -> None: ... - def send(self, obj) -> None: ... - def recv_bytes(self, maxlength: Incomplete | None = None): ... - def recv_bytes_into(self, buf, offset: int = 0): ... - def recv(self): ... - def poll(self, timeout: float = 0.0): ... - def __enter__(self): ... - def __exit__(self, exc_type: type[BaseException] | None, exc_value: BaseException | None, exc_tb: types.TracebackType | None) -> None: ... - def send_offset(self, buf, offset): ... - def setblocking(self, blocking) -> None: ... class PipeConnection(_ConnectionBase): ... class Connection(_ConnectionBase): ... class Listener: - def __init__(self, address: Incomplete | None = None, family: Incomplete | None = None, backlog: int = 1, authkey: Incomplete | None = None) -> None: ... - def accept(self): ... + def __init__( + self, + backlog: int = ..., + ) -> None: ... def close(self) -> None: ... - address: Incomplete - last_accepted: Incomplete - def __enter__(self): ... - def __exit__(self, exc_type: type[BaseException] | None, exc_value: BaseException | None, exc_tb: types.TracebackType | None) -> None: ... - -def Client(address, family: Incomplete | None = None, authkey: Incomplete | None = None): ... -def Pipe(duplex: bool = True, rnonblock: bool = False, wnonblock: bool = False): ... class SocketListener: - def __init__(self, address, family, backlog: int = 1) -> None: ... - def accept(self): ... def close(self) -> None: ... -class ConnectionWrapper: - def __init__(self, conn, dumps, loads) -> None: ... - def send(self, obj) -> None: ... - def recv(self): ... - -class XmlListener(Listener): - def accept(self): ... - -def wait(object_list, timeout: Incomplete | None = None): ... +class ConnectionWrapper: ... +class XmlListener(Listener): ... diff --git a/billiard-stubs/context.pyi b/billiard-stubs/context.pyi index 43b4bdd..434f3e3 100644 --- a/billiard-stubs/context.pyi +++ b/billiard-stubs/context.pyi @@ -1,18 +1,19 @@ -from . import process as process -from .exceptions import AuthenticationError as AuthenticationError, BufferTooShort as BufferTooShort, ProcessError as ProcessError, SoftTimeLimitExceeded as SoftTimeLimitExceeded, TimeLimitExceeded as TimeLimitExceeded, TimeoutError as TimeoutError, WorkerLostError as WorkerLostError -from _typeshed import Incomplete +from billiard import process as process +from billiard.exceptions import AuthenticationError as AuthenticationError +from billiard.exceptions import BufferTooShort as BufferTooShort +from billiard.exceptions import ProcessError as ProcessError +from billiard.exceptions import SoftTimeLimitExceeded as SoftTimeLimitExceeded +from billiard.exceptions import TimeLimitExceeded as TimeLimitExceeded +from billiard.exceptions import TimeoutError as TimeoutError +from billiard.exceptions import WorkerLostError as WorkerLostError W_NO_EXECV: str class BaseContext: - current_process: Incomplete - active_children: Incomplete def freeze_support(self) -> None: ... def allow_connection_pickling(self) -> None: ... - def set_start_method(self, method: Incomplete | None = None) -> None: ... class Process(process.BaseProcess): ... - class DefaultContext(BaseContext): ... class ForkProcess(process.BaseProcess): ... class SpawnProcess(process.BaseProcess): ... diff --git a/billiard-stubs/dummy/__init__.pyi b/billiard-stubs/dummy/__init__.pyi index ad1346a..bdfad92 100644 --- a/billiard-stubs/dummy/__init__.pyi +++ b/billiard-stubs/dummy/__init__.pyi @@ -1,12 +1,28 @@ import threading -from _typeshed import Incomplete from queue import Queue as Queue -from threading import BoundedSemaphore as BoundedSemaphore, Event as Event, Lock as Lock, RLock as RLock, Semaphore as Semaphore +from threading import BoundedSemaphore as BoundedSemaphore +from threading import Event as Event +from threading import Lock as Lock +from threading import RLock as RLock +from threading import Semaphore as Semaphore -__all__ = ['Process', 'current_process', 'freeze_support', 'Lock', 'RLock', 'Semaphore', 'BoundedSemaphore', 'Event', 'Queue', 'JoinableQueue'] +__all__ = [ + "Process", + "current_process", + "freeze_support", + "Lock", + "RLock", + "Semaphore", + "BoundedSemaphore", + "Event", + "Queue", + "JoinableQueue", +] class DummyProcess(threading.Thread): - def __init__(self, group: Incomplete | None = None, target: Incomplete | None = None, name: Incomplete | None = None) -> None: ... + def __init__( + self, + ) -> None: ... def start(self) -> None: ... Process = DummyProcess diff --git a/billiard-stubs/pool.pyi b/billiard-stubs/pool.pyi index dd41747..2863080 100644 --- a/billiard-stubs/pool.pyi +++ b/billiard-stubs/pool.pyi @@ -1,17 +1,27 @@ import threading -from .common import TERM_SIGNAL as TERM_SIGNAL, human_status as human_status, pickle_loads as pickle_loads, reset_signals as reset_signals, restart_state as restart_state -from .compat import get_errno as get_errno, mem_rss as mem_rss, send_offset as send_offset -from .dummy import Process as DummyProcess -from .einfo import ExceptionInfo as ExceptionInfo -from .exceptions import CoroStop as CoroStop, RestartFreqExceeded as RestartFreqExceeded, SoftTimeLimitExceeded as SoftTimeLimitExceeded, Terminated as Terminated, TimeLimitExceeded as TimeLimitExceeded, TimeoutError as TimeoutError, WorkerLostError as WorkerLostError -from .util import Finalize as Finalize, debug as debug, warning as warning -from _typeshed import Incomplete -from collections.abc import Generator + +from billiard.common import TERM_SIGNAL as TERM_SIGNAL +from billiard.common import human_status as human_status +from billiard.common import pickle_loads as pickle_loads +from billiard.common import reset_signals as reset_signals +from billiard.common import restart_state as restart_state +from billiard.compat import get_errno as get_errno +from billiard.compat import mem_rss as mem_rss +from billiard.compat import send_offset as send_offset +from billiard.dummy import Process as DummyProcess +from billiard.einfo import ExceptionInfo as ExceptionInfo +from billiard.exceptions import CoroStop as CoroStop +from billiard.exceptions import RestartFreqExceeded as RestartFreqExceeded +from billiard.exceptions import SoftTimeLimitExceeded as SoftTimeLimitExceeded +from billiard.exceptions import Terminated as Terminated +from billiard.exceptions import TimeLimitExceeded as TimeLimitExceeded +from billiard.exceptions import TimeoutError as TimeoutError +from billiard.exceptions import WorkerLostError as WorkerLostError +from billiard.util import debug as debug +from billiard.util import warning as warning MAXMEM_USED_FMT: str -PY3: Incomplete SIGKILL = TERM_SIGNAL -TIMEOUT_MAX: Incomplete RUN: int CLOSE: int TERMINATE: int @@ -23,102 +33,50 @@ DEATH: int EX_OK: int EX_FAILURE: int EX_RECYCLE: int -SIG_SOFT_TIMEOUT: Incomplete LOST_WORKER_TIMEOUT: float GUARANTEE_MESSAGE_CONSUMPTION_RETRY_LIMIT: int GUARANTEE_MESSAGE_CONSUMPTION_RETRY_INTERVAL: float -job_counter: Incomplete Lock = threading.Lock class LaxBoundedSemaphore(threading.Semaphore): def shrink(self) -> None: ... - def __init__(self, value: int = 1, verbose: Incomplete | None = None) -> None: ... def grow(self) -> None: ... def release(self) -> None: ... def clear(self) -> None: ... class MaybeEncodingError(Exception): - exc: Incomplete - value: Incomplete def __init__(self, exc: object, value: object) -> None: ... class WorkersJoined(Exception): ... class Worker: - initializer: Incomplete - initargs: Incomplete - maxtasks: Incomplete - max_memory_per_child: Incomplete - on_exit: Incomplete - sigprotection: Incomplete - wrap_exception: Incomplete - on_ready_counter: Incomplete def after_fork(self) -> None: ... class PoolThread(DummyProcess): daemon: bool def on_stop_not_started(self) -> None: ... - def stop(self, timeout: Incomplete | None = None) -> None: ... def terminate(self) -> None: ... def close(self) -> None: ... class Supervisor(PoolThread): - pool: Incomplete def body(self) -> None: ... class TaskHandler(PoolThread): - taskqueue: Incomplete - put: Incomplete - outqueue: Incomplete - pool: Incomplete - cache: Incomplete def body(self) -> None: ... def tell_others(self) -> None: ... def on_stop_not_started(self) -> None: ... class TimeoutHandler(PoolThread): - processes: Incomplete - cache: Incomplete - t_soft: Incomplete - t_hard: Incomplete - def handle_timeouts(self) -> Generator[None, None, Incomplete]: ... def body(self) -> None: ... class ResultHandler(PoolThread): - outqueue: Incomplete - get: Incomplete - cache: Incomplete - poll: Incomplete - join_exited_workers: Incomplete - putlock: Incomplete - restart_state: Incomplete - check_timeouts: Incomplete - on_job_ready: Incomplete - on_ready_counters: Incomplete def on_stop_not_started(self) -> None: ... - def handle_event(self, fileno: Incomplete | None = None, events: Incomplete | None = None) -> None: ... def body(self) -> None: ... - def finish_at_shutdown(self, handle_timeouts: bool = False) -> None: ... + def finish_at_shutdown(self, handle_timeouts: bool = ...) -> None: ... class Pool: - synack: Incomplete - timeout: Incomplete - soft_timeout: Incomplete - lost_worker_timeout: Incomplete - on_process_up: Incomplete - on_process_down: Incomplete - on_timeout_set: Incomplete - on_timeout_cancel: Incomplete - threads: Incomplete - readers: Incomplete - allow_restart: Incomplete - enable_timeouts: Incomplete - max_restarts: Incomplete - restart_state: Incomplete - putlocks: Incomplete - check_timeouts: Incomplete - def shrink(self, n: int = 1) -> None: ... - def grow(self, n: int = 1) -> None: ... + def shrink(self, n: int = ...) -> None: ... + def grow(self, n: int = ...) -> None: ... def maintain_pool(self) -> None: ... def __reduce__(self) -> None: ... def close(self) -> None: ... @@ -127,15 +85,11 @@ class Pool: def restart(self) -> None: ... class ApplyResult: - correlation_id: Incomplete def discard(self) -> None: ... - def wait(self, timeout: Incomplete | None = None) -> None: ... - def handle_timeout(self, soft: bool = False) -> None: ... + def handle_timeout(self, soft: bool = ...) -> None: ... class MapResult(ApplyResult): ... - class IMapIterator: ... - class IMapUnorderedIterator(IMapIterator): ... class ThreadPool(Pool): diff --git a/billiard-stubs/process.pyi b/billiard-stubs/process.pyi index 5d612bb..4301c02 100644 --- a/billiard-stubs/process.pyi +++ b/billiard-stubs/process.pyi @@ -1,18 +1,18 @@ -from _typeshed import Incomplete - -__all__ = ['BaseProcess', 'Process'] +__all__ = ["BaseProcess", "Process"] class BaseProcess: - def __init__(self, group: Incomplete | None = None, target: Incomplete | None = None, name: Incomplete | None = None, daemon: Incomplete | None = None) -> None: ... + def __init__( + self, + ) -> None: ... def run(self) -> None: ... def start(self) -> None: ... def close(self) -> None: ... def terminate(self) -> None: ... def terminate_controlled(self) -> None: ... - def join(self, timeout: Incomplete | None = None) -> None: ... class AuthenticationString(bytes): ... class _MainProcess(BaseProcess): def __init__(self) -> None: ... + Process = BaseProcess diff --git a/billiard-stubs/util.pyi b/billiard-stubs/util.pyi index c58b1c1..f997a40 100644 --- a/billiard-stubs/util.pyi +++ b/billiard-stubs/util.pyi @@ -1,7 +1,15 @@ -from multiprocessing.util import Finalize as Finalize, ForkAwareLocal as ForkAwareLocal, ForkAwareThreadLock as ForkAwareThreadLock, get_temp_dir as get_temp_dir, is_exiting as is_exiting, register_after_fork as register_after_fork from logging import Logger, _Level -__all__ = ['sub_debug', 'debug', 'info', 'sub_warning', 'get_logger', 'log_to_stderr', 'get_temp_dir', 'register_after_fork', 'is_exiting', 'Finalize', 'ForkAwareThreadLock', 'ForkAwareLocal', 'SUBDEBUG', 'SUBWARNING'] +__all__ = [ + "sub_debug", + "debug", + "info", + "sub_warning", + "get_logger", + "log_to_stderr", + "SUBDEBUG", + "SUBWARNING", +] SUBDEBUG: int SUBWARNING: int @@ -12,4 +20,4 @@ def info(msg: str, *args: object, **kwargs: object) -> None: ... def sub_warning(msg: str, *args: object, **kwargs: object) -> None: ... def warning(msg: str, *args: object, **kwargs: object) -> None: ... def get_logger() -> Logger: ... -def log_to_stderr(level: _Level | None = None) -> Logger: ... +def log_to_stderr(level: _Level | None = ...) -> Logger: ... diff --git a/celery-stubs/app/base.pyi b/celery-stubs/app/base.pyi index e198031..1b8f2e6 100644 --- a/celery-stubs/app/base.pyi +++ b/celery-stubs/app/base.pyi @@ -145,7 +145,7 @@ class Celery: def start(self, argv: list[str] | None = ...) -> NoReturn: ... def worker_main(self, argv: list[str] | None = ...) -> NoReturn: ... @overload - def task(self, fun: Callable[_P, _R]) -> CeleryTask[_P, _R]: ... # type: ignore [misc] + def task(self, fun: Callable[_P, _R]) -> CeleryTask[_P, _R]: ... @overload def task( self, diff --git a/celery-stubs/beat.pyi b/celery-stubs/beat.pyi index 9938070..0f85777 100644 --- a/celery-stubs/beat.pyi +++ b/celery-stubs/beat.pyi @@ -1,75 +1,49 @@ import shelve -from _typeshed import Incomplete -from billiard.context import Process from threading import Thread from typing import NamedTuple -__all__ = ['SchedulingError', 'ScheduleEntry', 'Scheduler', 'PersistentScheduler', 'Service'] +from billiard.context import Process -class event_t(NamedTuple): - time: Incomplete - priority: Incomplete - entry: Incomplete +__all__ = [ + "SchedulingError", + "ScheduleEntry", + "Scheduler", + "PersistentScheduler", + "Service", +] +class event_t(NamedTuple): ... class SchedulingError(Exception): ... - class BeatLazyFunc: ... class ScheduleEntry: - name: Incomplete - schedule: Incomplete - args: Incomplete - kwargs: Incomplete - options: Incomplete - last_run_at: Incomplete total_run_count: int - app: Incomplete - task: Incomplete - __next__: Incomplete - next: Incomplete class Scheduler: Entry = ScheduleEntry - schedule: Incomplete - sync_every: Incomplete - sync_every_tasks: Incomplete - app: Incomplete - data: Incomplete - Producer: Incomplete - old_schedulers: Incomplete def setup_schedule(self) -> None: ... def sync(self) -> None: ... def close(self) -> None: ... class PersistentScheduler(Scheduler): persistence = shelve - known_suffixes: Incomplete - schedule_filename: Incomplete def setup_schedule(self) -> None: ... - schedule: Incomplete def sync(self) -> None: ... def close(self) -> None: ... class Service: scheduler_cls = PersistentScheduler - app: Incomplete - max_interval: Incomplete - schedule_filename: Incomplete - def start(self, embedded_process: bool = False) -> None: ... + def start(self, embedded_process: bool = ...) -> None: ... def sync(self) -> None: ... - def stop(self, wait: bool = False) -> None: ... + def stop(self, wait: bool = ...) -> None: ... class _Threaded(Thread): - app: Incomplete - service: Incomplete daemon: bool name: str def run(self) -> None: ... def stop(self) -> None: ... class _Process(Process): - app: Incomplete - service: Incomplete name: str def run(self) -> None: ... - def stop(self) -> None: ... \ No newline at end of file + def stop(self) -> None: ... diff --git a/celery-stubs/concurrency/asynpool.pyi b/celery-stubs/concurrency/asynpool.pyi index d47c945..0349dcb 100644 --- a/celery-stubs/concurrency/asynpool.pyi +++ b/celery-stubs/concurrency/asynpool.pyi @@ -1,29 +1,14 @@ -from _typeshed import Incomplete -from billiard import pool as _pool from typing import NamedTuple -__all__ = ['AsynPool'] +from billiard import pool as _pool -class Ack(NamedTuple): - id: Incomplete - fd: Incomplete - payload: Incomplete +__all__ = ["AsynPool"] -class Worker(_pool.Worker): - pass +class Ack(NamedTuple): ... +class Worker(_pool.Worker): ... class ResultHandler(_pool.ResultHandler): - fileno_to_outq: Incomplete - on_process_alive: Incomplete def on_stop_not_started(self) -> None: ... class AsynPool(_pool.Pool): - sched_strategy: Incomplete - synack: Incomplete - outbound_buffer: Incomplete - write_stats: Incomplete - on_soft_timeout: Incomplete - on_hard_timeout: Incomplete - def __init__(self, processes: Incomplete | None = None, synack: bool = False, sched_strategy: Incomplete | None = None, proc_alive_timeout: Incomplete | None = None) -> None: ... - handle_result_event: Incomplete - def flush(self) -> None: ... \ No newline at end of file + def flush(self) -> None: ... diff --git a/celery-stubs/concurrency/prefork.pyi b/celery-stubs/concurrency/prefork.pyi index 885de34..8742217 100644 --- a/celery-stubs/concurrency/prefork.pyi +++ b/celery-stubs/concurrency/prefork.pyi @@ -1,7 +1,7 @@ -from .asynpool import AsynPool +from celery.concurrency.asynpool import AsynPool from celery.concurrency.base import BasePool -__all__ = ['TaskPool'] +__all__ = ["TaskPool"] class TaskPool(BasePool): Pool = AsynPool diff --git a/celery-stubs/worker/components.pyi b/celery-stubs/worker/components.pyi index db3dc72..35aff10 100644 --- a/celery-stubs/worker/components.pyi +++ b/celery-stubs/worker/components.pyi @@ -5,4 +5,4 @@ class Hub(bootsteps.StartStopStep): ... class Pool(bootsteps.StartStopStep): ... class Beat(bootsteps.StartStopStep): ... class StateDB(bootsteps.Step): ... -class Consumer(bootsteps.StartStopStep): ... \ No newline at end of file +class Consumer(bootsteps.StartStopStep): ... diff --git a/celery-stubs/worker/worker.pyi b/celery-stubs/worker/worker.pyi index f39ec1f..2c8e497 100644 --- a/celery-stubs/worker/worker.pyi +++ b/celery-stubs/worker/worker.pyi @@ -1,4 +1,4 @@ -from .components import Timer, Hub, Pool +from celery.worker.components import Hub, Pool, Timer class WorkController: hub: Hub | None