From 010a960ad65ef297fa3a3216110dcf0a32514426 Mon Sep 17 00:00:00 2001 From: Vaclav Lunak Date: Sun, 1 Oct 2023 09:17:54 +0200 Subject: [PATCH] refactor bugfixes --- .gitignore | 1 + poetry.lock | 114 ++++++++++++++++-- pyproject.toml | 6 +- src/admin/router.py | 12 +- src/admin/service.py | 10 +- .../cli_scripts/create_cr_and_diff.py | 20 +-- .../cli_scripts/create_mtr_and_diff.py | 10 +- src/cli_scripts/run.py | 35 ++++++ src/cr/keyword_def.py | 2 +- src/cr/models.py | 2 +- src/cr/router.py | 16 +-- src/cr/schemas.py | 6 +- src/cr/service.py | 10 +- src/cr/utils.py | 6 +- src/diffs/models.py | 2 +- src/diffs/router.py | 10 +- src/diffs/schemas.py | 20 +-- src/diffs/service.py | 6 +- src/difftool/diffmaker.py | 6 +- src/difftool/matcher.py | 2 +- src/extractor/cr/extract_cr.py | 6 +- src/extractor/cr/refresh_cr.py | 22 ++-- src/extractor/download_doc.py | 2 +- src/extractor/ipg/refresh_ipg.py | 6 +- src/extractor/mtr/extract_mtr.py | 2 +- src/extractor/mtr/refresh_mtr.py | 14 +-- src/ipg/models.py | 2 +- src/ipg/router.py | 8 +- src/ipg/schemas.py | 2 +- src/ipg/service.py | 2 +- src/link/models.py | 2 +- src/link/router.py | 10 +- src/link/service.py | 2 +- src/main.py | 23 ++-- src/mtr/models.py | 2 +- src/mtr/router.py | 8 +- src/mtr/schemas.py | 2 +- src/mtr/service.py | 2 +- src/openapi/openapi_decorators.py | 4 +- src/resources/cache.py | 2 +- src/resources/seeder.py | 5 +- src/scraper/cr_scraper.py | 8 +- src/scraper/docs_scraper.py | 10 +- src/utils/backup.py | 6 +- src/utils/notifier.py | 2 - src/utils/scheduler.py | 9 +- 46 files changed, 292 insertions(+), 167 deletions(-) rename create_cr_and_diff.py => src/cli_scripts/create_cr_and_diff.py (88%) rename create_mtr_and_diff.py => src/cli_scripts/create_mtr_and_diff.py (91%) create mode 100644 src/cli_scripts/run.py diff --git a/.gitignore b/.gitignore index 57a95b8..8139987 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ src/static/raw_docs .env .idea/ venv/ +dist/ diff --git a/poetry.lock b/poetry.lock index 8661915..ac8a189 100644 --- a/poetry.lock +++ b/poetry.lock @@ -789,16 +789,6 @@ files = [ [package.dependencies] rapidfuzz = ">=2.3.0,<4.0.0" -[[package]] -name = "logging" -version = "0.4.9.6" -description = "A logging module for Python" -optional = false -python-versions = "*" -files = [ - { file = "logging-0.4.9.6.tar.gz", hash = "sha256:26f6b50773f085042d301085bd1bf5d9f3735704db9f37c1ce6d8b85c38f2417" }, -] - [[package]] name = "mako" version = "1.2.4" @@ -818,6 +808,30 @@ babel = ["Babel"] lingua = ["lingua"] testing = ["pytest"] +[[package]] +name = "markdown-it-py" +version = "3.0.0" +description = "Python port of markdown-it. Markdown parsing, done right!" +optional = false +python-versions = ">=3.8" +files = [ + { file = "markdown-it-py-3.0.0.tar.gz", hash = "sha256:e3f60a94fa066dc52ec76661e37c851cb232d92f9886b15cb560aaada2df8feb" }, + { file = "markdown_it_py-3.0.0-py3-none-any.whl", hash = "sha256:355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1" }, +] + +[package.dependencies] +mdurl = ">=0.1,<1.0" + +[package.extras] +benchmarking = ["psutil", "pytest", "pytest-benchmark"] +code-style = ["pre-commit (>=3.0,<4.0)"] +compare = ["commonmark (>=0.9,<1.0)", "markdown (>=3.4,<4.0)", "mistletoe (>=1.0,<2.0)", "mistune (>=2.0,<3.0)", "panflute (>=2.3,<3.0)"] +linkify = ["linkify-it-py (>=1,<3)"] +plugins = ["mdit-py-plugins"] +profiling = ["gprof2dot"] +rtd = ["jupyter_sphinx", "mdit-py-plugins", "myst-parser", "pyyaml", "sphinx", "sphinx-copybutton", "sphinx-design", "sphinx_book_theme"] +testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions"] + [[package]] name = "markupsafe" version = "2.1.3" @@ -898,6 +912,17 @@ files = [ { file = "mccabe-0.7.0.tar.gz", hash = "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325" }, ] +[[package]] +name = "mdurl" +version = "0.1.2" +description = "Markdown URL utilities" +optional = false +python-versions = ">=3.7" +files = [ + { file = "mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8" }, + { file = "mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba" }, +] + [[package]] name = "mypy-extensions" version = "1.0.0" @@ -1264,6 +1289,20 @@ files = [ { file = "pyflakes-3.1.0.tar.gz", hash = "sha256:a0aae034c444db0071aa077972ba4768d40c830d9539fd45bf4cd3f8f6992efc" }, ] +[[package]] +name = "pygments" +version = "2.16.1" +description = "Pygments is a syntax highlighting package written in Python." +optional = false +python-versions = ">=3.7" +files = [ + { file = "Pygments-2.16.1-py3-none-any.whl", hash = "sha256:13fc09fa63bc8d8671a6d247e1eb303c4b343eaee81d861f3404db2935653692" }, + { file = "Pygments-2.16.1.tar.gz", hash = "sha256:1daff0494820c69bc8941e407aa20f577374ee88364ee10a98fdbe0aece96e29" }, +] + +[package.extras] +plugins = ["importlib-metadata"] + [[package]] name = "python-dateutil" version = "2.8.2" @@ -1531,6 +1570,24 @@ urllib3 = ">=1.21.1,<3" socks = ["PySocks (>=1.5.6,!=1.5.7)"] use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] +[[package]] +name = "rich" +version = "13.6.0" +description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" +optional = false +python-versions = ">=3.7.0" +files = [ + { file = "rich-13.6.0-py3-none-any.whl", hash = "sha256:2b38e2fe9ca72c9a00170a1a2d20c63c790d0e10ef1fe35eba76e1e7b1d7d245" }, + { file = "rich-13.6.0.tar.gz", hash = "sha256:5c14d22737e6d5084ef4771b62d5d4363165b403455a30a1c8ca39dc7b644bef" }, +] + +[package.dependencies] +markdown-it-py = ">=2.2.0" +pygments = ">=2.13.0,<3.0.0" + +[package.extras] +jupyter = ["ipywidgets (>=7.5.1,<9)"] + [[package]] name = "s3transfer" version = "0.7.0" @@ -1564,6 +1621,17 @@ docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.1)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] +[[package]] +name = "shellingham" +version = "1.5.3" +description = "Tool to Detect Surrounding Shell" +optional = false +python-versions = ">=3.7" +files = [ + { file = "shellingham-1.5.3-py2.py3-none-any.whl", hash = "sha256:419c6a164770c9c7cfcaeddfacb3d31ac7a8db0b0f3e9c1287679359734107e9" }, + { file = "shellingham-1.5.3.tar.gz", hash = "sha256:cb4a6fec583535bc6da17b647dd2330cf7ef30239e05d547d99ae3705fd0f7f8" }, +] + [[package]] name = "six" version = "1.16.0" @@ -1731,6 +1799,30 @@ files = [ { file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f" }, ] +[[package]] +name = "typer" +version = "0.9.0" +description = "Typer, build great CLIs. Easy to code. Based on Python type hints." +optional = false +python-versions = ">=3.6" +files = [ + { file = "typer-0.9.0-py3-none-any.whl", hash = "sha256:5d96d986a21493606a358cae4461bd8cdf83cbf33a5aa950ae629ca3b51467ee" }, + { file = "typer-0.9.0.tar.gz", hash = "sha256:50922fd79aea2f4751a8e0408ff10d2662bd0c8bbfa84755a699f3bada2978b2" }, +] + +[package.dependencies] +click = ">=7.1.1,<9.0.0" +colorama = { version = ">=0.4.3,<0.5.0", optional = true, markers = "extra == \"all\"" } +rich = { version = ">=10.11.0,<14.0.0", optional = true, markers = "extra == \"all\"" } +shellingham = { version = ">=1.3.0,<2.0.0", optional = true, markers = "extra == \"all\"" } +typing-extensions = ">=3.7.4.3" + +[package.extras] +all = ["colorama (>=0.4.3,<0.5.0)", "rich (>=10.11.0,<14.0.0)", "shellingham (>=1.3.0,<2.0.0)"] +dev = ["autoflake (>=1.3.1,<2.0.0)", "flake8 (>=3.8.3,<4.0.0)", "pre-commit (>=2.17.0,<3.0.0)"] +doc = ["cairosvg (>=2.5.2,<3.0.0)", "mdx-include (>=1.4.1,<2.0.0)", "mkdocs (>=1.1.2,<2.0.0)", "mkdocs-material (>=8.1.4,<9.0.0)", "pillow (>=9.3.0,<10.0.0)"] +test = ["black (>=22.3.0,<23.0.0)", "coverage (>=6.2,<7.0)", "isort (>=5.0.6,<6.0.0)", "mypy (==0.910)", "pytest (>=4.4.0,<8.0.0)", "pytest-cov (>=2.10.0,<5.0.0)", "pytest-sugar (>=0.9.4,<0.10.0)", "pytest-xdist (>=1.32.0,<4.0.0)", "rich (>=10.11.0,<14.0.0)", "shellingham (>=1.3.0,<2.0.0)"] + [[package]] name = "typing-extensions" version = "4.8.0" @@ -2042,4 +2134,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "13d698d6399dce71af26a89457a01e6de5dc436bbf1cb6fb0c9fa16760056579" +content-hash = "ab6efaeb55e8fc477d211a7e0564bcf96c01f6e24bcc636f01fb3918cf5a45c7" diff --git a/pyproject.toml b/pyproject.toml index 43eecfd..1aa2299 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,13 +10,15 @@ skip_gitignore = true name = "academyruins-api" version = "0.5.0" description = "" -authors = ["Your Name "] +authors = ["Václav Luňák < >"] readme = "README.md" +[tool.poetry.scripts] +academyruins = "src.cli_scripts.run:app" + [tool.poetry.dependencies] python = "^3.10" fastapi = { extras = ["all"], version = "^0.103.2" } -logging = "^0.4.9.6" APScheduler = "^3.9.1" boto3 = "^1.24.84" pydantic = "^2.4.2" diff --git a/src/admin/router.py b/src/admin/router.py index d8d2cd8..f766a3a 100644 --- a/src/admin/router.py +++ b/src/admin/router.py @@ -3,12 +3,12 @@ from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, Response from sqlalchemy.orm import Session -from admin import service -from db import get_db -from extractor.cr.refresh_cr import refresh_cr -from extractor.ipg.refresh_ipg import refresh_ipg -from extractor.mtr.refresh_mtr import refresh_mtr -from schemas import ResponseModel +from src.admin import service +from src.db import get_db +from src.extractor.cr.refresh_cr import refresh_cr +from src.extractor.ipg.refresh_ipg import refresh_ipg +from src.extractor.mtr.refresh_mtr import refresh_mtr +from src.schemas import ResponseModel router = APIRouter(include_in_schema=False) diff --git a/src/admin/service.py b/src/admin/service.py index 2bde0f8..027f6bc 100644 --- a/src/admin/service.py +++ b/src/admin/service.py @@ -1,11 +1,11 @@ from sqlalchemy import select from sqlalchemy.orm import Session -from cr.models import Cr, PendingCr -from diffs.models import CrDiff, CrDiffItem, MtrDiff, PendingCrDiff, PendingMtrDiff -from link.models import PendingRedirect, Redirect -from mtr.models import Mtr, PendingMtr -from mtr.service import get_pending_mtr +from src.cr.models import Cr, PendingCr +from src.diffs.models import CrDiff, CrDiffItem, MtrDiff, PendingCrDiff, PendingMtrDiff +from src.link.models import PendingRedirect, Redirect +from src.mtr.models import Mtr, PendingMtr +from src.mtr.service import get_pending_mtr def apply_pending_redirect(db: Session, resource: str) -> str | None: diff --git a/create_cr_and_diff.py b/src/cli_scripts/create_cr_and_diff.py similarity index 88% rename from create_cr_and_diff.py rename to src/cli_scripts/create_cr_and_diff.py index c4fa13c..4e52a89 100644 --- a/create_cr_and_diff.py +++ b/src/cli_scripts/create_cr_and_diff.py @@ -1,9 +1,9 @@ import json import os -from difftool.diffmaker import CRDiffMaker -from extractor.cr import extract_cr -from extractor.formatter import CRFormatterFactory +from src.difftool.diffmaker import CRDiffMaker +from src.extractor.cr import extract_cr +from src.extractor.formatter import CRFormatterFactory def diff(old_txt, new_txt, old_set_code=None, new_set_code=None, forced_matches=None): @@ -67,13 +67,13 @@ def diffall(): diff_save(old, new) -cr_in_dir = "src/static/raw_docs/cr" -cr_out_dir = "./gen/cr" -diff_dir = "./gen/diff_unchecked" -maps_dir = "./gen/map" -gloss_dir = "./gen/gloss" -key_dir = "./gen/keywords" -toc_dir = "./gen/toc" +cr_in_dir = "../static/raw_docs/cr" +cr_out_dir = "../../gen/cr" +diff_dir = "../../gen/diff_unchecked" +maps_dir = "../../gen/map" +gloss_dir = "../../gen/gloss" +key_dir = "../../gen/keywords" +toc_dir = "../../gen/toc" if __name__ == "__main__": # diffall() diff --git a/create_mtr_and_diff.py b/src/cli_scripts/create_mtr_and_diff.py similarity index 91% rename from create_mtr_and_diff.py rename to src/cli_scripts/create_mtr_and_diff.py index 143ba39..40c72f7 100644 --- a/create_mtr_and_diff.py +++ b/src/cli_scripts/create_mtr_and_diff.py @@ -5,8 +5,8 @@ from dotenv import load_dotenv -from difftool.diffmaker import MtrDiffMaker -from extractor.mtr.extract_mtr import extract +from src.difftool.diffmaker import MtrDiffMaker +from src.extractor.mtr.extract_mtr import extract load_dotenv() @@ -47,8 +47,8 @@ def diff_save(old: str, new: str): file.write(mtr_to_text(new_date, new_sections)) -mtr_out_dir = Path(".") / "gen" / "mtr" -mtr_diff_dir = Path(".") / "gen" / "mtr-diff" +mtr_out_dir = Path("../..") / "gen" / "mtr" +mtr_diff_dir = Path("../..") / "gen" / "mtr-diff" batch_input = [ # "2023-03-29", @@ -66,7 +66,7 @@ def diff_save(old: str, new: str): def diff_batch(): - mtrdir = Path("mtr") + mtrdir = Path("../../mtr") for i in range(1, len(batch_input)): old = mtrdir / ("mtr-" + batch_input[i] + ".pdf") new = mtrdir / ("mtr-" + batch_input[i - 1] + ".pdf") diff --git a/src/cli_scripts/run.py b/src/cli_scripts/run.py new file mode 100644 index 0000000..9c2a468 --- /dev/null +++ b/src/cli_scripts/run.py @@ -0,0 +1,35 @@ +from typing import Annotated, Union + +import typer +import uvicorn +from dotenv import load_dotenv + +app = typer.Typer() + + +@app.command() +def run( + port: int = 8000, + host: str = "localhost", + reload: Annotated[ + bool, typer.Option(help="Whether the server should automatically restart on file change.") + ] = False, +): + """ + Start the API web server. + """ + uvicorn.run("src.main:app", port=port, host=host, reload=reload) + + +@app.callback() +def options( + envfile: Annotated[ + Union[str, None], typer.Option(help="Optional location of a .env file that should be loaded") + ] = None +): + if envfile: + load_dotenv(dotenv_path=envfile) + + +if __name__ == "__main__": + app() diff --git a/src/cr/keyword_def.py b/src/cr/keyword_def.py index fba19c5..0483a9e 100644 --- a/src/cr/keyword_def.py +++ b/src/cr/keyword_def.py @@ -1,6 +1,6 @@ import re -from cr import service as cr_service +from src.cr import service as cr_service keyword_regex = r"702.(?:[2-9]|\d\d+)" keyword_action_regex = r"701.(?:[2-9]|\d\d+)" diff --git a/src/cr/models.py b/src/cr/models.py index efdbc70..d186489 100644 --- a/src/cr/models.py +++ b/src/cr/models.py @@ -1,7 +1,7 @@ from sqlalchemy import Column, Date, Integer, String, Text from sqlalchemy.dialects.postgresql import JSONB -from models import Base +from src.models import Base class Cr(Base): diff --git a/src/cr/router.py b/src/cr/router.py index 9ad19d6..e5fe5c0 100644 --- a/src/cr/router.py +++ b/src/cr/router.py @@ -6,14 +6,14 @@ from sqlalchemy.orm import Session from thefuzz import fuzz, process -from cr import schemas, service -from cr.keyword_def import get_best_rule -from db import get_db -from openapi.no422 import no422 -from openapi.strings import crTag, filesTag -from resources import static_paths as paths -from resources.cache import GlossaryCache -from schemas import Error, FileFormat +from src.cr import schemas, service +from src.cr.keyword_def import get_best_rule +from src.db import get_db +from src.openapi.no422 import no422 +from src.openapi.strings import crTag, filesTag +from src.resources import static_paths as paths +from src.resources.cache import GlossaryCache +from src.schemas import Error, FileFormat router = APIRouter() glossary = GlossaryCache() diff --git a/src/cr/schemas.py b/src/cr/schemas.py index 3d5fb69..ba8a8af 100644 --- a/src/cr/schemas.py +++ b/src/cr/schemas.py @@ -2,8 +2,8 @@ from pydantic import Field -from diffs.schemas import CrDiffMetadata -from schemas import Error, ResponseModel +from src.diffs.schemas import CrDiffMetadata +from src.schemas import Error, ResponseModel class RuleError(Error): @@ -72,6 +72,6 @@ class TraceDiffRule(ResponseModel): class TraceItem(ResponseModel): action: TraceItemAction - old: TraceDiffRule | None + old: TraceDiffRule | None = None new: TraceDiffRule diff: CrDiffMetadata diff --git a/src/cr/service.py b/src/cr/service.py index 5560cfe..7da9abe 100644 --- a/src/cr/service.py +++ b/src/cr/service.py @@ -1,11 +1,11 @@ from sqlalchemy import select from sqlalchemy.orm import Session -from cr import utils -from cr.models import Cr -from cr.schemas import TraceItem -from diffs.models import CrDiff, CrDiffItem -from diffs.schemas import CrDiffMetadata +from src.cr import utils +from src.cr.models import Cr +from src.cr.schemas import TraceItem +from src.diffs.models import CrDiff, CrDiffItem +from src.diffs.schemas import CrDiffMetadata def get_latest_cr(db: Session) -> Cr: diff --git a/src/cr/utils.py b/src/cr/utils.py index e2d37f9..be82d52 100644 --- a/src/cr/utils.py +++ b/src/cr/utils.py @@ -1,6 +1,6 @@ -from cr.schemas import TraceDiffRule, TraceItem, TraceItemAction -from diffs.models import CrDiffItem, DiffItemKind -from diffs.schemas import CrDiffMetadata +from src.cr.schemas import TraceDiffRule, TraceItem, TraceItemAction +from src.diffs.models import CrDiffItem, DiffItemKind +from src.diffs.schemas import CrDiffMetadata def format_trace_item(db_item: CrDiffItem) -> TraceItem: diff --git a/src/diffs/models.py b/src/diffs/models.py index 6f395c3..3b69ad0 100644 --- a/src/diffs/models.py +++ b/src/diffs/models.py @@ -4,7 +4,7 @@ from sqlalchemy.dialects.postgresql import JSONB from sqlalchemy.orm import relationship -from models import Base +from src.models import Base class CrDiff(Base): diff --git a/src/diffs/router.py b/src/diffs/router.py index 29f6176..7bfafbe 100644 --- a/src/diffs/router.py +++ b/src/diffs/router.py @@ -5,11 +5,11 @@ from fastapi.responses import RedirectResponse from sqlalchemy.orm import Session -from db import get_db -from diffs import schemas, service -from diffs.models import PendingCrDiff -from difftool.diffsorter import CRDiffSorter -from openapi.strings import diffTag +from src.db import get_db +from src.diffs import schemas, service +from src.diffs.models import PendingCrDiff +from src.difftool.diffsorter import CRDiffSorter +from src.openapi.strings import diffTag router = APIRouter(tags=[diffTag.name]) diff --git a/src/diffs/schemas.py b/src/diffs/schemas.py index 6c9f694..30bedfa 100644 --- a/src/diffs/schemas.py +++ b/src/diffs/schemas.py @@ -2,8 +2,8 @@ from pydantic import Field -from mtr.schemas import MtrChunk -from schemas import Error, ResponseModel +from src.mtr.schemas import MtrChunk +from src.schemas import Error, ResponseModel class CrDiffError(Error): @@ -16,8 +16,8 @@ class MtrDiffError(Error): class CRDiffError(Error): - old: str - new: str + old: str | None = None + new: str | None = None class CRDiffRule(ResponseModel): @@ -26,8 +26,8 @@ class CRDiffRule(ResponseModel): class CRDiffItem(ResponseModel): - old: CRDiffRule | None - new: CRDiffRule | None + old: CRDiffRule | None = None + new: CRDiffRule | None = None class CRMoveItem(ResponseModel): @@ -50,8 +50,8 @@ class CRDiffNavigation(ResponseModel): class CRDiff(CrDiffMetadata): creation_day: datetime.date = Field(..., alias="creationDay") changes: list[CRDiffItem] = Field(...) - moves: list[CRMoveItem] = Field(description="List of moved rules") - nav: CRDiffNavigation | None = Field(description="Navigational information.") + moves: list[CRMoveItem] = Field([], description="List of moved rules") + nav: CRDiffNavigation | None = Field(None, description="Navigational information.") class PendingCRDiff(ResponseModel): @@ -64,8 +64,8 @@ class PendingCRDiffResponse(ResponseModel): class MtrDiffItem(ResponseModel): - old: MtrChunk | None - new: MtrChunk | None + old: MtrChunk | None = None + new: MtrChunk | None = None class MtrDiff(ResponseModel): diff --git a/src/diffs/service.py b/src/diffs/service.py index 1dc6df0..07f6350 100644 --- a/src/diffs/service.py +++ b/src/diffs/service.py @@ -3,9 +3,9 @@ from sqlalchemy import select from sqlalchemy.orm import Session, aliased -from cr.models import Cr -from diffs.models import CrDiff, CrDiffItem, MtrDiff, PendingCrDiff, PendingMtrDiff -from mtr.models import Mtr +from src.cr.models import Cr +from src.diffs.models import CrDiff, CrDiffItem, MtrDiff, PendingCrDiff, PendingMtrDiff +from src.mtr.models import Mtr def get_cr_diff(db: Session, old_code: str | None, new_code: str | None) -> CrDiff | None: diff --git a/src/difftool/diffmaker.py b/src/difftool/diffmaker.py index 1eb0050..2336440 100644 --- a/src/difftool/diffmaker.py +++ b/src/difftool/diffmaker.py @@ -1,8 +1,8 @@ from dataclasses import dataclass -from .diffsorter import CRDiffSorter, DiffSorter, MtrDiffSorter -from .itemdiffer import CRItemDiffer, ItemDiffer, MtrItemDiffer -from .matcher import CRMatcher, Matcher, MtrMatcher +from src.difftool.diffsorter import CRDiffSorter, DiffSorter, MtrDiffSorter +from src.difftool.itemdiffer import CRItemDiffer, ItemDiffer, MtrItemDiffer +from src.difftool.matcher import CRMatcher, Matcher, MtrMatcher @dataclass diff --git a/src/difftool/matcher.py b/src/difftool/matcher.py index fe7fa03..c67af09 100644 --- a/src/difftool/matcher.py +++ b/src/difftool/matcher.py @@ -1,7 +1,7 @@ import difflib from abc import ABC, abstractmethod -from .matchscoregraph import MatchScoreGraph +from src.difftool.matchscoregraph import MatchScoreGraph class Matcher(ABC): diff --git a/src/extractor/cr/extract_cr.py b/src/extractor/cr/extract_cr.py index eb3d0dc..ce621ed 100644 --- a/src/extractor/cr/extract_cr.py +++ b/src/extractor/cr/extract_cr.py @@ -1,9 +1,9 @@ import json import re -from cr.keyword_def import ability_words_rule, keyword_action_regex, keyword_regex -from cr.schemas import ToCSection, ToCSubsection -from resources import static_paths as paths +from src.cr.keyword_def import ability_words_rule, keyword_action_regex, keyword_regex +from src.cr.schemas import ToCSection, ToCSubsection +from src.resources import static_paths as paths # TODO rework into new class hierarchy diff --git a/src/extractor/cr/refresh_cr.py b/src/extractor/cr/refresh_cr.py index 0e4769e..758e1f1 100644 --- a/src/extractor/cr/refresh_cr.py +++ b/src/extractor/cr/refresh_cr.py @@ -5,17 +5,17 @@ from sqlalchemy import select from sqlalchemy.orm import Session -from cr import service as cr_service -from cr.models import Cr, PendingCr -from db import SessionLocal -from diffs.models import PendingCrDiff -from difftool.diffmaker import CRDiffMaker -from extractor.cr import extract_cr -from link import service as links_service -from resources import static_paths as paths -from resources.cache import GlossaryCache, KeywordCache -from utils import notifier -from utils.logger import logger +from src.cr import service as cr_service +from src.cr.models import Cr, PendingCr +from src.db import SessionLocal +from src.diffs.models import PendingCrDiff +from src.difftool.diffmaker import CRDiffMaker +from src.extractor.cr import extract_cr +from src.link import service as links_service +from src.resources import static_paths as paths +from src.resources.cache import GlossaryCache, KeywordCache +from src.utils import notifier +from src.utils.logger import logger def get_response_text(response: requests.Response) -> str | None: diff --git a/src/extractor/download_doc.py b/src/extractor/download_doc.py index 0dd09b4..90102e0 100644 --- a/src/extractor/download_doc.py +++ b/src/extractor/download_doc.py @@ -3,7 +3,7 @@ import requests -from resources import static_paths as paths +from src.resources import static_paths as paths def download_doc(link: str, kind: Literal["mtr", "ipg"]): diff --git a/src/extractor/ipg/refresh_ipg.py b/src/extractor/ipg/refresh_ipg.py index e35b848..7a79012 100644 --- a/src/extractor/ipg/refresh_ipg.py +++ b/src/extractor/ipg/refresh_ipg.py @@ -1,6 +1,6 @@ -from db import SessionLocal -from extractor.download_doc import download_doc -from ipg.service import upload_ipg +from src.db import SessionLocal +from src.extractor.download_doc import download_doc +from src.ipg.service import upload_ipg def refresh_ipg(link: str): diff --git a/src/extractor/mtr/extract_mtr.py b/src/extractor/mtr/extract_mtr.py index 895b6e5..cf4741d 100644 --- a/src/extractor/mtr/extract_mtr.py +++ b/src/extractor/mtr/extract_mtr.py @@ -5,7 +5,7 @@ from tika import parser -from mtr.schemas import MtrChunk +from src.mtr.schemas import MtrChunk class ParagraphSplitter: diff --git a/src/extractor/mtr/refresh_mtr.py b/src/extractor/mtr/refresh_mtr.py index eb21de3..00892fc 100644 --- a/src/extractor/mtr/refresh_mtr.py +++ b/src/extractor/mtr/refresh_mtr.py @@ -1,13 +1,13 @@ from datetime import date from pathlib import Path -from db import SessionLocal -from diffs.models import PendingMtrDiff -from difftool.diffmaker import MtrDiffMaker -from extractor.download_doc import download_doc -from extractor.mtr.extract_mtr import extract -from mtr.models import PendingMtr -from mtr.service import get_current_mtr +from src.db import SessionLocal +from src.diffs.models import PendingMtrDiff +from src.difftool.diffmaker import MtrDiffMaker +from src.extractor.download_doc import download_doc +from src.extractor.mtr.extract_mtr import extract +from src.mtr.models import PendingMtr +from src.mtr.service import get_current_mtr def refresh_mtr(link: str): diff --git a/src/ipg/models.py b/src/ipg/models.py index cd31697..bed329c 100644 --- a/src/ipg/models.py +++ b/src/ipg/models.py @@ -1,6 +1,6 @@ from sqlalchemy import Column, Date, Integer, Text -from models import Base +from src.models import Base class Ipg(Base): diff --git a/src/ipg/router.py b/src/ipg/router.py index 3930653..4c5f1a5 100644 --- a/src/ipg/router.py +++ b/src/ipg/router.py @@ -4,10 +4,10 @@ from fastapi.responses import FileResponse from sqlalchemy.orm import Session -from db import get_db -from ipg import schemas, service -from openapi.strings import filesTag -from schemas import Error +from src.db import get_db +from src.ipg import schemas, service +from src.openapi.strings import filesTag +from src.schemas import Error router = APIRouter() diff --git a/src/ipg/schemas.py b/src/ipg/schemas.py index 34272ad..1ee3361 100644 --- a/src/ipg/schemas.py +++ b/src/ipg/schemas.py @@ -2,7 +2,7 @@ from pydantic import Field -from schemas import ResponseModel +from src.schemas import ResponseModel class IpgMetadataItem(ResponseModel): diff --git a/src/ipg/service.py b/src/ipg/service.py index 442505c..8024d38 100644 --- a/src/ipg/service.py +++ b/src/ipg/service.py @@ -3,7 +3,7 @@ from sqlalchemy import select from sqlalchemy.orm import Session -from ipg.models import Ipg +from src.ipg.models import Ipg def get_ipg_by_creation_date(db: Session, date: datetime.date) -> Ipg | None: diff --git a/src/link/models.py b/src/link/models.py index c3944bf..24c720a 100644 --- a/src/link/models.py +++ b/src/link/models.py @@ -1,6 +1,6 @@ from sqlalchemy import Column, Text -from models import Base +from src.models import Base class Redirect(Base): diff --git a/src/link/router.py b/src/link/router.py index c6980ad..2c759c3 100644 --- a/src/link/router.py +++ b/src/link/router.py @@ -2,11 +2,11 @@ from fastapi.responses import RedirectResponse from sqlalchemy.orm import Session -from db import get_db -from link import service -from openapi.no422 import no422 -from openapi.strings import redirectTag -from schemas import Error +from src.db import get_db +from src.link import service +from src.openapi.no422 import no422 +from src.openapi.strings import redirectTag +from src.schemas import Error router = APIRouter(tags=[redirectTag.name]) diff --git a/src/link/service.py b/src/link/service.py index 9dc0378..54fa7cf 100644 --- a/src/link/service.py +++ b/src/link/service.py @@ -1,7 +1,7 @@ from sqlalchemy import select from sqlalchemy.orm import Session -from link.models import PendingRedirect, Redirect +from src.link.models import PendingRedirect, Redirect def get_redirect(db: Session, resource: str) -> str | None: diff --git a/src/main.py b/src/main.py index 6bea565..f0e19d6 100644 --- a/src/main.py +++ b/src/main.py @@ -8,14 +8,14 @@ from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import JSONResponse -from admin.router import router as admin_router -from cr.router import router as cr_router -from diffs.router import router as diff_router -from ipg.router import router as ipg_router -from link.router import router as link_router -from mtr.router import router as mtr_router -from openapi import strings -from openapi.openapi_decorators import ( +from src.admin.router import router as admin_router +from src.cr.router import router as cr_router +from src.diffs.router import router as diff_router +from src.ipg.router import router as ipg_router +from src.link.router import router as link_router +from src.mtr.router import router as mtr_router +from src.openapi import strings +from src.openapi.openapi_decorators import ( ApiLogoDecorator, BaseResolver, CachingDecorator, @@ -23,10 +23,9 @@ TagGroupsDecorator, ValidationErrorSchemaDecorator, ) - -from .resources import seeder -from .utils.logger import logger -from .utils.scheduler import Scheduler +from src.resources import seeder +from src.utils.logger import logger +from src.utils.scheduler import Scheduler logging.basicConfig(format="%(asctime)s:%(levelname)s:%(name)s:%(message)s", datefmt="%Y-%m-%d %H:%M:%S") diff --git a/src/mtr/models.py b/src/mtr/models.py index 502f282..0f2a338 100644 --- a/src/mtr/models.py +++ b/src/mtr/models.py @@ -1,7 +1,7 @@ from sqlalchemy import Column, Date, Integer, Text from sqlalchemy.dialects.postgresql import JSONB -from models import Base +from src.models import Base class Mtr(Base): diff --git a/src/mtr/router.py b/src/mtr/router.py index 5ffd862..8e33da8 100644 --- a/src/mtr/router.py +++ b/src/mtr/router.py @@ -4,10 +4,10 @@ from fastapi.responses import FileResponse from sqlalchemy.orm import Session -from db import get_db -from mtr import schemas, service -from openapi.strings import filesTag, mtrTag -from schemas import Error +from src.db import get_db +from src.mtr import schemas, service +from src.openapi.strings import filesTag, mtrTag +from src.schemas import Error router = APIRouter() diff --git a/src/mtr/schemas.py b/src/mtr/schemas.py index 84dc122..f526008 100644 --- a/src/mtr/schemas.py +++ b/src/mtr/schemas.py @@ -2,7 +2,7 @@ from pydantic import Field -from schemas import Error, ResponseModel +from src.schemas import Error, ResponseModel class SectionError(Error): diff --git a/src/mtr/service.py b/src/mtr/service.py index 56bae6d..a4b9a6c 100644 --- a/src/mtr/service.py +++ b/src/mtr/service.py @@ -3,7 +3,7 @@ from sqlalchemy import select from sqlalchemy.orm import Session -from mtr.models import Mtr, PendingMtr +from src.mtr.models import Mtr, PendingMtr def get_current_mtr(db: Session) -> Mtr: diff --git a/src/openapi/openapi_decorators.py b/src/openapi/openapi_decorators.py index 36e9d8e..0403304 100644 --- a/src/openapi/openapi_decorators.py +++ b/src/openapi/openapi_decorators.py @@ -5,8 +5,8 @@ from fastapi.routing import APIRoute from starlette.routing import BaseRoute -from openapi.no422 import is_marked_no422 -from openapi.strings import Tag +from src.openapi.no422 import is_marked_no422 +from src.openapi.strings import Tag class OpenApiResolver(ABC): diff --git a/src/resources/cache.py b/src/resources/cache.py index f72e255..e7d828a 100644 --- a/src/resources/cache.py +++ b/src/resources/cache.py @@ -1,7 +1,7 @@ import json from pathlib import Path -from . import static_paths as paths +from src.resources import static_paths as paths # global cache store _caches = {} diff --git a/src/resources/seeder.py b/src/resources/seeder.py index 7d2f1b5..589a0d5 100644 --- a/src/resources/seeder.py +++ b/src/resources/seeder.py @@ -1,9 +1,8 @@ import os from pathlib import Path -from utils.logger import logger - -from . import static_paths as paths +from src.resources import static_paths as paths +from src.utils.logger import logger def seed_dir(path): diff --git a/src/scraper/cr_scraper.py b/src/scraper/cr_scraper.py index 2706d8b..515e975 100644 --- a/src/scraper/cr_scraper.py +++ b/src/scraper/cr_scraper.py @@ -3,10 +3,10 @@ import requests from bs4 import BeautifulSoup -from db import SessionLocal -from link import service as links_service -from utils.logger import logger -from utils.notifier import notify_new_cr, notify_scrape_error +from src.db import SessionLocal +from src.link import service as links_service +from src.utils.logger import logger +from src.utils.notifier import notify_new_cr, notify_scrape_error rules_page_uri = "https://magic.wizards.com/en/rules/" diff --git a/src/scraper/docs_scraper.py b/src/scraper/docs_scraper.py index f68ae6a..3ff73e6 100644 --- a/src/scraper/docs_scraper.py +++ b/src/scraper/docs_scraper.py @@ -5,11 +5,11 @@ import requests from sqlalchemy.orm import Session -from db import SessionLocal -from link import service as links_service -from link.models import PendingRedirect -from utils.logger import logger -from utils.notifier import notify_new_doc, notify_scrape_error +from src.db import SessionLocal +from src.link import service as links_service +from src.link.models import PendingRedirect +from src.utils.logger import logger +from src.utils.notifier import notify_new_doc, notify_scrape_error docs_page_uri = "https://wpn.wizards.com/en/rules-documents/" diff --git a/src/utils/backup.py b/src/utils/backup.py index dc980a5..ef83727 100644 --- a/src/utils/backup.py +++ b/src/utils/backup.py @@ -5,9 +5,9 @@ from botocore.config import Config from botocore.exceptions import ClientError -from resources import static_paths as paths -from utils import notifier -from utils.logger import logger +from src.resources import static_paths as paths +from src.utils import notifier +from src.utils.logger import logger class Backup: diff --git a/src/utils/notifier.py b/src/utils/notifier.py index 1452586..aad7e35 100644 --- a/src/utils/notifier.py +++ b/src/utils/notifier.py @@ -1,9 +1,7 @@ import os import requests -from dotenv import load_dotenv -load_dotenv() _uri = "https://api.pushover.net/1/messages.json" _token = os.environ.get("PUSHOVER_APP_TOKEN") _user = os.environ.get("PUSHOVER_USER_KEY") diff --git a/src/utils/scheduler.py b/src/utils/scheduler.py index 181b3bd..b01b22b 100644 --- a/src/utils/scheduler.py +++ b/src/utils/scheduler.py @@ -1,11 +1,10 @@ from apscheduler.jobstores.memory import MemoryJobStore from apscheduler.schedulers.asyncio import AsyncIOScheduler -from scraper.cr_scraper import scrape_rules_page -from scraper.docs_scraper import scrape_docs_page - -from .backup import run_backup -from .logger import logger +from src.scraper.cr_scraper import scrape_rules_page +from src.scraper.docs_scraper import scrape_docs_page +from src.utils.backup import run_backup +from src.utils.logger import logger class Scheduler: