make tools, io modules pluggable

.github/workflows/ci.yml

@@ -73,8 +73,8 @@ jobs:
         aexpy -vvv report ./cache/diff.json.gz ./cache/report.json.gz
     - name: Test Image Runner
       run: |
-        aexpy runimage ./cache -t stardustdl/aexpy:latest -- --version
-        aexpy runimage ./cache -t -- view report.json.gz
+        aexpy runimage -t stardustdl/aexpy:latest -- --version
+        aexpy runimage -v ./cache -t stardustdl/aexpy:latest -- view report.json.gz
     - name: Test Service Provider
       env:
         AEXPY_SERVICE: "./scripts/services.py"

README.md

@@ -352,7 +352,7 @@ aexpy view ./stats.json
 
 AexPy has four loosely-coupled stages in its pipeline. The adjacent stages transfer data by JSON, defined in [models](https://github.com/StardustDL/aexpy/blob/main/src/aexpy/models/) directory. You can easily write your own implementation for every stage, and combine your implementation into the pipeline.
 
-To write your own services, copy from [aexpy/services](https://github.com/StardustDL/aexpy/blob/main/src/aexpy/services/__init__.py) and write your subclass of `ServiceProvider` and modify the `getService` function to return your service instance.
+To write your own services, copy from [aexpy/services.py](https://github.com/StardustDL/aexpy/blob/main/src/aexpy/services.py) and write your subclass of `ServiceProvider` and modify the `getService` function to return your service instance.
 
 ```python
 from aexpy.services import ServiceProvider
@@ -371,10 +371,11 @@ aexpy -s services.py -vvv view --help
 AEXPY_SERVICE=services.py aexpy -vvv view --help
 ```
 
-We have implemented an image service provider, which replaces the default extractor, differ, and reporter by the container worker. See [aexpy/services/workers.py](https://github.com/StardustDL/aexpy/blob/main/src/aexpy/services/workers.py) for its implementation. Here is the demo service file to use the image service provider.
+We have implemented an image service provider, which replaces the default extractor, differ, and reporter by the container worker. See [aexpy/tools/workers/services.py](https://github.com/StardustDL/aexpy/blob/main/src/aexpy/tools/workers/services.py) for its implementation. Here is the demo service file to use the image service provider.
 
 ```python
-from aexpy.services.workers import DockerWorkerServiceProvider
+from aexpy.tools.workers.services import DockerWorkerServiceProvider
+
 
 def getService():
     return DockerWorkerServiceProvider(tag="stardustdl/aexpy:latest")

scripts/services.py

@@ -1,4 +1,4 @@
-from aexpy.services.workers import DockerWorkerServiceProvider
+from aexpy.tools.workers.services import DockerWorkerServiceProvider
 
 
 def getService():

src/aexpy/__main__.py

@@ -1,7 +1,11 @@
 from .cli import main
-from .tools.cli import tool
 
-main.add_command(tool)
+try:
+    from .tools.cli import tool
+
+    main.add_command(tool)
+except Exception:
+    pass
 
 if __name__ == "__main__":
     main()

src/aexpy/cli.py

@@ -11,13 +11,24 @@
 
 import click
 
-from . import (__version__, getBuildDate, getShortCommitId, initializeLogging,
-               runInContainer)
-from .models import (ApiDescription, ApiDifference, Distribution, ProduceState,
-                     Product, Release, Report)
+from . import (
+    __version__,
+    getBuildDate,
+    getShortCommitId,
+    initializeLogging,
+    runInContainer,
+)
+from .models import (
+    ApiDescription,
+    ApiDifference,
+    Distribution,
+    ProduceState,
+    Product,
+    Release,
+    Report,
+)
 from .producers import ProduceContext, produce
 from .services import ServiceProvider, getService, loadServiceFromCode
-from .tools.models import StatSummary
 
 DEFAULT_SERVICE = getService()
 
@@ -279,8 +290,10 @@ def extractCore(
         elif temp:
             envBuilder = service.environmentBuilder(logger=context.logger)
         else:
-            from .environments import (CurrentEnvironment,
-                                       SingleExecutionEnvironmentBuilder)
+            from .environments import (
+                CurrentEnvironment,
+                SingleExecutionEnvironmentBuilder,
+            )
 
             envBuilder = SingleExecutionEnvironmentBuilder(
                 CurrentEnvironment(context.logger), context.logger
@@ -639,7 +652,14 @@ def view(ctx: click.Context, file: IO[bytes]):
 
     from .io import load
 
-    result = load(cache.raw(), lambda data: StatSummary.model_validate(data))
+    try:
+        from .tools.models import StatSummary
+
+        fallback = lambda data: StatSummary.model_validate(data)
+    except Exception:
+        fallback = None
+
+    result = load(cache.raw(), fallback)
 
     print(result.overview())
     if isinstance(result, Report):

src/aexpy/io/__init__.py

@@ -4,8 +4,14 @@
 from pathlib import Path
 from typing import IO, Callable, Literal, overload, override
 
-from ..models import (ApiDescription, ApiDifference, CoreProduct, Distribution,
-                      Product, Report)
+from ..models import (
+    ApiDescription,
+    ApiDifference,
+    CoreProduct,
+    Distribution,
+    Product,
+    Report,
+)
 from ..utils import ensureDirectory
 
 
@@ -91,6 +97,10 @@ def save(self, /, product, log):
 type LoadSourceType = Path | IOBase | bytes | str | dict
 
 
+@overload
+def load(data: LoadSourceType, fallback: None) -> CoreProduct: ...
+
+
 @overload
 def load[T: Product](data: LoadSourceType, fallback: type[T]) -> T: ...
 

src/aexpy/services/__init__.py → src/aexpy/services.py

@@ -1,16 +1,15 @@
 from contextlib import contextmanager
 from logging import Logger
 
-from .. import __version__, getShortCommitId
-from ..diffing import Differ
-from ..environments import ExecutionEnvironment, ExecutionEnvironmentBuilder
-from ..extracting import Extractor
-from ..models import (ApiDescription, ApiDifference, Distribution, Product,
+from . import __version__, getShortCommitId
+from .diffing import Differ
+from .environments import ExecutionEnvironment, ExecutionEnvironmentBuilder
+from .extracting import Extractor
+from .models import (ApiDescription, ApiDifference, Distribution, Product,
                       Report)
-from ..preprocessing import Preprocessor
-from ..producers import ProduceContext, produce
-from ..reporting import Reporter
-from ..tools.stats import StatisticianWorker
+from .preprocessing import Preprocessor
+from .producers import ProduceContext, produce
+from .reporting import Reporter
 
 
 class ServiceProvider:
@@ -20,35 +19,32 @@ def __init__(self, name: str | None = None) -> None:
     def environmentBuilder(
         self, /, logger: Logger | None = None
     ) -> ExecutionEnvironmentBuilder:
-        from ..extracting.environment import getExtractorEnvironmentBuilder
+        from .extracting.environment import getExtractorEnvironmentBuilder
 
         return getExtractorEnvironmentBuilder(logger=logger)
 
     def preprocessor(self, /, logger: Logger | None = None) -> Preprocessor:
-        from ..preprocessing.counter import FileCounterPreprocessor
+        from .preprocessing.counter import FileCounterPreprocessor
 
         return FileCounterPreprocessor(logger=logger)
 
     def extractor(
         self, /, logger: Logger | None = None, env: ExecutionEnvironment | None = None
     ) -> Extractor:
-        from ..extracting.default import DefaultExtractor
+        from .extracting.default import DefaultExtractor
 
         return DefaultExtractor(logger=logger, env=env)
 
     def differ(self, /, logger: Logger | None = None) -> Differ:
-        from ..diffing.default import DefaultDiffer
+        from .diffing.default import DefaultDiffer
 
         return DefaultDiffer(logger=logger)
 
     def reporter(self, /, logger: Logger | None = None) -> Reporter:
-        from ..reporting.text import TextReporter
+        from .reporting.text import TextReporter
 
         return TextReporter(logger=logger)
 
-    def statistician(self, /, logger: Logger | None = None) -> StatisticianWorker:
-        return StatisticianWorker(logger=logger)
-
     @contextmanager
     def produce[
         T: Product

src/aexpy/tools/cli.py

@@ -8,6 +8,7 @@
 from ..cli import AliasedGroup, CliContext, StreamProductSaver, exitWithContext
 from ..producers import produce
 from .models import StatSummary
+from .stats import StatisticianWorker
 
 
 @click.group(cls=AliasedGroup)
@@ -45,9 +46,7 @@ def stat(ctx: click.Context, files: tuple[Path], output: IO[bytes]):
     clictx = ctx.ensure_object(CliContext)
 
     with produce(StatSummary(), service=clictx.service.name) as context:
-        with context.using(
-            clictx.service.statistician(logger=context.logger)
-        ) as worker:
+        with context.using(StatisticianWorker(logger=context.logger)) as worker:
             worker.count(files, context.product)
 
     result = context.product
@@ -81,7 +80,9 @@ def stat(ctx: click.Context, files: tuple[Path], output: IO[bytes]):
     "args",
     nargs=-1,
 )
-def runimage(ctx: click.Context, args: tuple[str], volume: Path=Path("."), tag: str = ""):
+def runimage(
+    ctx: click.Context, args: tuple[str], volume: Path = Path("."), tag: str = ""
+):
     """Quick runner to execute commands using AexPy images.
 
     VOLUME describes the mount directory for containers (to /data), default using current working directory.

src/aexpy/tools/workers.py → src/aexpy/tools/workers/__init__.py

@@ -9,14 +9,14 @@
 from tempfile import TemporaryDirectory
 from typing import Callable, override
 
-from .. import __version__, getEnvironmentManager
-from ..diffing import Differ
-from ..extracting import Extractor
-from ..io import StreamProductSaver
-from ..models import (ApiDescription, ApiDifference, Distribution, Product,
+from ... import __version__, getEnvironmentManager
+from ...diffing import Differ
+from ...extracting import Extractor
+from ...io import StreamProductSaver
+from ...models import (ApiDescription, ApiDifference, Distribution, Product,
                       Report)
-from ..producers import Producer
-from ..reporting import Reporter
+from ...producers import Producer
+from ...reporting import Reporter
 
 
 @dataclass

src/aexpy/services/workers.py → src/aexpy/tools/workers/services.py

@@ -3,13 +3,13 @@
 from pathlib import Path
 from typing import Callable, override
 
-from ..environments import (CurrentEnvironment, ExecutionEnvironmentBuilder,
+from ...environments import (CurrentEnvironment, ExecutionEnvironmentBuilder,
                             SingleExecutionEnvironmentBuilder)
-from ..models import ApiDescription, Distribution
-from ..producers import ProduceContext
-from ..tools.workers import (AexPyDockerWorker, AexPyWorker, WorkerDiffer,
+from ...models import ApiDescription, Distribution
+from ...producers import ProduceContext
+from . import (AexPyDockerWorker, AexPyWorker, WorkerDiffer,
                              WorkerExtractor, WorkerReporter)
-from . import ServiceProvider
+from ...services import ServiceProvider
 
 
 class WorkerServiceProvider(ServiceProvider):