The resource could not be found.
+ + + + """ + + +# HTTP +HTTP_HEADER = { + "Accept": "application/json; charset=utf-8" +} +RESPONSE_NORMAL_CODE = 200 +RESPONSE_NOT_FOUND_CODE = 404 +RESPONSE_DATA = "data" diff --git a/kdp_catalog_manager/config/__init__.py b/kdp_catalog_manager/config/__init__.py new file mode 100644 index 0000000..ff6ea1e --- /dev/null +++ b/kdp_catalog_manager/config/__init__.py @@ -0,0 +1,2 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- diff --git a/kdp_catalog_manager/config/base_config.py b/kdp_catalog_manager/config/base_config.py new file mode 100644 index 0000000..1395f87 --- /dev/null +++ b/kdp_catalog_manager/config/base_config.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +import os + + +CACHE_EXPIRE = 60*60*24*365*10 + + +CATALOG_DIR = "catalog" +APPS_DIR = "apps" + +CATALOG_README_DIR = "readme/catalog" +CATALOG_FROM_README_DIR = "readme/form" + + +DEFAULT_LANG = "zh" +SUPPORT_LANG = (os.environ.get("SUPPORT_LANG") or "zh,en").split(",") + +# httpx connect other server setting +HTTP_TIME_OUT = int(os.environ.get("HTTP_TIME_OUT") or 10) +HTTP_MAX_RETRIES = int(os.environ.get("HTTP_MAX_RETRIES") or 3) +HTTP = "http" + + +# OAM config +OAM_BASE_URL = os.environ.get( + "OAM_BASE_URL", + default=f"{HTTP}://kdp-oam-apiserver:8000" +) + + +# markdown to html +EXTEND_EXTENSION = os.environ.get("EXTENSION", default=None) + + +# worker +WORKER_NUM = int(os.environ.get("WORKER_NUM") or 4) +LIMIT_MAX_REQUESTS = os.environ.get("LIMIT_MAX_REQUESTS", None) +TIMEOUT_KEEP_ALIVE = int(os.environ.get("TIMEOUT_KEEP_ALIVE") or 5) + + +# monitor +DASHBOARD_URL = os.environ.get("DASHBOARD_URL", default="https://grafana.com") diff --git a/kdp_catalog_manager/config/gunicorn.py b/kdp_catalog_manager/config/gunicorn.py new file mode 100644 index 0000000..2fcf9b5 --- /dev/null +++ b/kdp_catalog_manager/config/gunicorn.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- + +# 监听内网端口8000 +bind = "0.0.0.0:8888" +# 工作模式协程。 +worker_class = "uvicorn.workers.UvicornWorker" +# 不设置守护进程 +daemon = 'false' +# 日志输出到stdout、stderr +errorlog = '-' +accesslog = '-' +# 日志格式 +logconfig_dict = { + 'formatters': { + "generic": { + # 打日志的格式 + "format": "[%(asctime)s] [%(levelname)s] [PID:%(process)d][ThreadID:%(thread)d-%(threadName)s] %(message)s", + "class": "logging.Formatter" + } + } +} \ No newline at end of file diff --git a/kdp_catalog_manager/config/log_config.py b/kdp_catalog_manager/config/log_config.py new file mode 100644 index 0000000..7db9567 --- /dev/null +++ b/kdp_catalog_manager/config/log_config.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +import os + + +LOG_DIR = os.environ.get("LOG_DIR", "logs") +LOG_LEVEL = os.environ.get('LOG_LEVEL', 'INFO') +LOG_FILE = os.environ.get("LOG_FILE", "kdp-catalog-manager.log") +LOG_FILE_MAX_BYTES = os.environ.get('LOG_FILE_MAX_BYTES', 1024 * 1024 * 5) +LOG_FILE_BACKUP_COUNT = os.environ.get('LOG_FILE_BACKUP_COUNT', 3) + + +LOGGING_CONFIG = { + "version": 1, + "disable_existing_loggers": False, + "formatters": { + "default": { + "()": "uvicorn.logging.DefaultFormatter", + "fmt": "[%(asctime)s] [%(levelname)s]: %(message)s", + "use_colors": None + }, + "access": { + "()": "uvicorn.logging.AccessFormatter", + "fmt": "[%(asctime)s] [%(levelname)s] [PID:%(process)d][ThreadID:%(thread)d-%(threadName)s] - [%(status_code)s]: %(message)s" + } + }, + "handlers": { + "default": { + "formatter": "default", + "class": "logging.StreamHandler", + "stream": "ext://sys.stderr" + }, + "access": { + "formatter": "access", + "class": "logging.StreamHandler", + "stream": "ext://sys.stderr" + } + }, + "loggers": { + "uvicorn": { + "handlers": ["default"], + "level": "ERROR", + "propagate": False + }, + "uvicorn.error": { + "level": "INFO" + }, + "uvicorn.access": { + "handlers": ["access"], + "level": "INFO", + "propagate": False + } + } +} \ No newline at end of file diff --git a/kdp_catalog_manager/domain/__init__.py b/kdp_catalog_manager/domain/__init__.py new file mode 100644 index 0000000..ff6ea1e --- /dev/null +++ b/kdp_catalog_manager/domain/__init__.py @@ -0,0 +1,2 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- diff --git a/kdp_catalog_manager/domain/format/__init__.py b/kdp_catalog_manager/domain/format/__init__.py new file mode 100644 index 0000000..ff6ea1e --- /dev/null +++ b/kdp_catalog_manager/domain/format/__init__.py @@ -0,0 +1,2 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- diff --git a/kdp_catalog_manager/domain/format/base.py b/kdp_catalog_manager/domain/format/base.py new file mode 100644 index 0000000..0e2b3b4 --- /dev/null +++ b/kdp_catalog_manager/domain/format/base.py @@ -0,0 +1,7 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- + + +class Format(object): + def __init__(self, raw): + self.raw = raw diff --git a/kdp_catalog_manager/domain/format/format_catalog.py b/kdp_catalog_manager/domain/format/format_catalog.py new file mode 100644 index 0000000..0905f13 --- /dev/null +++ b/kdp_catalog_manager/domain/format/format_catalog.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +from kdp_catalog_manager.domain.format.base import Format +from kdp_catalog_manager.utils.dictutils import DictUtils +from kdp_catalog_manager.config.base_config import DEFAULT_LANG +from kdp_catalog_manager.common.constants import I18N + + +class FormatCatalog(Format): + def __init__(self, raw): + super().__init__(raw) + + def get_name(self): + return DictUtils().get_items(self.raw, ["name"]) + + def get_type(self): + return DictUtils().get_items(self.raw, ["isGlobal"], False) + + def get_category(self, lang=DEFAULT_LANG): + if lang == DEFAULT_LANG: + return DictUtils().get_items(self.raw, ["category"]) + return DictUtils().get_items(self.raw, [I18N, lang, "category"]) + + def get_description(self, lang=DEFAULT_LANG): + if lang == DEFAULT_LANG: + return DictUtils().get_items(self.raw, ["description"]) + return DictUtils().get_items(self.raw, [I18N, lang, "description"]) diff --git a/kdp_catalog_manager/domain/format/format_catalog_form.py b/kdp_catalog_manager/domain/format/format_catalog_form.py new file mode 100644 index 0000000..5d4dfb4 --- /dev/null +++ b/kdp_catalog_manager/domain/format/format_catalog_form.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +from kdp_catalog_manager.common.constants import I18N +from kdp_catalog_manager.config.base_config import DEFAULT_LANG +from kdp_catalog_manager.domain.format.base import Format +from kdp_catalog_manager.utils.dictutils import DictUtils +from kdp_catalog_manager.config.base_config import DASHBOARD_URL + + +class FormatCatalogForm(Format): + def __init__(self, raw): + super().__init__(raw) + + def get_version(self): + return DictUtils().get_items(self.raw, ["version"]) + + def get_type(self): + return DictUtils().get_items(self.raw, ["isGlobal"], False) + + def get_alias(self): + return DictUtils().get_items(self.raw, ["alias"]) + + def get_invisible(self): + return DictUtils().get_items(self.raw, ["invisible"], False) + + def get_description(self, lang=DEFAULT_LANG): + if lang == DEFAULT_LANG: + return DictUtils().get_items(self.raw, ["description"]) + return DictUtils().get_items( + self.raw, [I18N, lang, "description"]) + + def get_dashboard(self, lang=DEFAULT_LANG): + """ + get catalog from dashboard info + :param lang: + :return: [{"name": "xx", "id": "xxx"}] + """ + dashboard_list = [] + dashboards = DictUtils().get_items(self.raw, ["dashboard"]) + if not dashboards: + return dashboard_list + for dashboard in dashboards: + dashboard_id = DictUtils().get_items(dashboard, ["id"]) + dashboard_name = None + if lang == DEFAULT_LANG: + dashboard_name = DictUtils().get_items( + dashboard, ["name"]) + if lang != DEFAULT_LANG: + dashboard_name = DictUtils().get_items( + dashboard, [I18N, lang]) + dashboard_list.append({ + "name": dashboard_name, + "id": dashboard_id, + "link": f"{DASHBOARD_URL}/d/{dashboard_id}" + }) + return dashboard_list + + +def format_catalog_form_metadata(catalog_form_metadatas, filter_lang, filter_invisible): + catalog_form_metadata_info = {} + catalog_form_metadata_all = catalog_form_metadatas + for catalog, catalog_form_metadatas in catalog_form_metadata_all.items(): + + catalog_catalog_form_metadatas_info = {} + for catalog_form, catalog_form_metadata in catalog_form_metadatas.items(): + + catalog_form_metadata_format_obj = FormatCatalogForm( + catalog_form_metadata) + invisible = catalog_form_metadata_format_obj.get_invisible() + if filter_invisible != invisible: + continue + catalog_catalog_form_metadatas_info[catalog_form] = { + "name": catalog_form, + "version": catalog_form_metadata_format_obj.get_version(), + "alias": catalog_form_metadata_format_obj.get_alias(), + "invisible": catalog_form_metadata_format_obj.get_invisible(), + "isGlobal": catalog_form_metadata_format_obj.get_type(), + "description": catalog_form_metadata_format_obj.get_description( + filter_lang), + "catalog": catalog, + "dashboardUrl": catalog_form_metadata_format_obj.get_dashboard( + filter_lang) + } + catalog_form_metadata_info[ + catalog] = catalog_catalog_form_metadatas_info + return catalog_form_metadata_info diff --git a/kdp_catalog_manager/domain/format/format_catalog_runtime_application.py b/kdp_catalog_manager/domain/format/format_catalog_runtime_application.py new file mode 100644 index 0000000..2d09a69 --- /dev/null +++ b/kdp_catalog_manager/domain/format/format_catalog_runtime_application.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +from kdp_catalog_manager.domain.format.base import Format +from kdp_catalog_manager.utils.dictutils import DictUtils +from kdp_catalog_manager.utils.log import log + + +class FormatCatalogRuntimeApplication(Format): + def __init__(self, raw): + super().__init__(raw) + + def get_bdc_name(self): + return DictUtils().get_items(self.raw, ["bdc", "name"]) + + def get_bdc_org(self): + return DictUtils().get_items(self.raw, ["bdc", "orgName"]) + + def get_bdc_status(self): + return DictUtils().get_items(self.raw, ["bdc", "status"]) + + def get_app_name(self): + return DictUtils().get_items(self.raw, ["name"]) + + def get_app_form(self): + return DictUtils().get_items(self.raw, ["appFormName"]) + + def get_app_runtime_name(self): + return DictUtils().get_items(self.raw, ["appRuntimeName"]) + + def get_app_createtime(self): + return DictUtils().get_items(self.raw, ["createTime"]) + + def get_app_updatetime(self): + return DictUtils().get_items(self.raw, ["updateTime"]) + + def get_app_status(self): + return DictUtils().get_items(self.raw, ["status"]) + + +def format_application( + all_application, + form_info, + filter_catalog, + filter_form +): + applications = [] + # deal with get application data is empty list or None + if not all_application: + return applications + for application_data in all_application: + application = application_data + application_format_obj = ( + FormatCatalogRuntimeApplication(application)) + application_form = application_format_obj.get_app_form() + application_catalog = DictUtils().get_items( + form_info, [application_form]) + + if not application_catalog: + log.warning(f"{application_form} not match catalog, " + f"get data is {application_catalog}") + continue + + # filter application + if filter_catalog and filter_catalog != application_catalog: + continue + if filter_form and filter_form != application_form: + continue + + application["catalog"] = application_catalog + applications.append(application) + return applications diff --git a/kdp_catalog_manager/domain/format/test/__init__.py b/kdp_catalog_manager/domain/format/test/__init__.py new file mode 100644 index 0000000..ff6ea1e --- /dev/null +++ b/kdp_catalog_manager/domain/format/test/__init__.py @@ -0,0 +1,2 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- diff --git a/kdp_catalog_manager/domain/format/test/test_format_catalog.py b/kdp_catalog_manager/domain/format/test/test_format_catalog.py new file mode 100644 index 0000000..b39ebcf --- /dev/null +++ b/kdp_catalog_manager/domain/format/test/test_format_catalog.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- + +from unittest import TestCase + +from ..format_catalog import FormatCatalog + +TEST_SYSTEM_CATEGORY = "系统/大数据开发工具" + + +class TestFormCatalog(TestCase): + def setUp(self): + self.catalog_data = { + "name": "mysql", + "category": "系统/大数据开发工具", + "description": "mysql", + "i18n": { + "en": { + "category": "system.dataManagement", + "description": "Mysql M" + } + } + } + self.catalog_format_obj = FormatCatalog(self.catalog_data) + + def test_get_name(self): + rt = self.catalog_format_obj.get_name() + self.assertEqual(rt, "mysql") + + def test_get_type(self): + rt = self.catalog_format_obj.get_type() + self.assertEqual(rt, False) + + def test_get_type_exists(self): + self.catalog_format_obj.raw["isGlobal"] = True + rt = self.catalog_format_obj.get_type() + self.assertEqual(rt, True) + + def test_get_category_default(self): + rt = self.catalog_format_obj.get_category("zh") + self.assertEqual(rt, TEST_SYSTEM_CATEGORY) + + def test_get_category_en(self): + rt = self.catalog_format_obj.get_category("en") + self.assertEqual(rt, "system.dataManagement") + + def test_get_category(self): + rt = self.catalog_format_obj.get_category() + self.assertEqual(rt, TEST_SYSTEM_CATEGORY) + + def test_get_description(self): + rt = self.catalog_format_obj.get_description() + self.assertEqual(rt, "mysql") + + def test_get_description_zh(self): + rt = self.catalog_format_obj.get_description("zh") + self.assertEqual(rt, "mysql") + + def test_get_description_en(self): + rt = self.catalog_format_obj.get_description("en") + self.assertEqual(rt, "Mysql M") diff --git a/kdp_catalog_manager/domain/format/test/test_format_catalog_form.py b/kdp_catalog_manager/domain/format/test/test_format_catalog_form.py new file mode 100644 index 0000000..d2a3eba --- /dev/null +++ b/kdp_catalog_manager/domain/format/test/test_format_catalog_form.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- + +from unittest import TestCase + +from ..format_catalog_form import FormatCatalogForm + + +TEST_DESC = "mysql test" + + +class TestFormatCatalogForm(TestCase): + def setUp(self): + self.catalog_form_metadata = { + "version": "1.0.0", + "alias": "mysql", + "description": TEST_DESC, + "isGlobal": False, + "i18n": { + "en": { + "category": "system.dataManagement", + "description": "Mysql M" + } + } + } + self.dashboard = [{ + "id": "xxx", + "name": "x1", + "i18n": { + "en": "x11" + } + }] + self.catalog_form_format_obj = FormatCatalogForm(self.catalog_form_metadata) + + def test_get_version(self): + rt = self.catalog_form_format_obj.get_version() + self.assertEqual(rt, "1.0.0") + + def test_get_version_not_exists(self): + del self.catalog_form_format_obj.raw["version"] + rt = self.catalog_form_format_obj.get_version() + self.assertEqual(rt, None) + + def test_get_type(self): + rt = self.catalog_form_format_obj.get_type() + self.assertEqual(rt, False) + + def test_get_type_exists(self): + self.catalog_form_format_obj.raw["isGlobal"] = True + rt = self.catalog_form_format_obj.get_type() + self.assertEqual(rt, True) + + def test_get_alias(self): + rt = self.catalog_form_format_obj.get_alias() + self.assertEqual(rt, "mysql") + + def test_get_invisible(self): + rt = self.catalog_form_format_obj.get_invisible() + self.assertEqual(rt, False) + + def test_get_invisible_exists(self): + self.catalog_form_format_obj.raw["invisible"] = True + rt = self.catalog_form_format_obj.get_invisible() + self.assertEqual(rt, True) + + def test_get_description(self): + rt = self.catalog_form_format_obj.get_description() + self.assertEqual(rt, TEST_DESC) + + def test_get_description_zh(self): + rt = self.catalog_form_format_obj.get_description("zh") + self.assertEqual(rt, TEST_DESC) + + def test_get_description_en(self): + rt = self.catalog_form_format_obj.get_description("en") + self.assertEqual(rt, "Mysql M") + + def test_dashboard(self): + rt = self.catalog_form_format_obj.get_dashboard() + self.assertEqual(rt, []) + + def test_dashboard_exists(self): + self.catalog_form_format_obj.raw["dashboard"] = self.dashboard + rt = self.catalog_form_format_obj.get_dashboard() + self.assertEqual(rt, [{ + "id": "xxx", "name": "x1", 'link': 'https://grafana.com/d/xxx'}]) + + def test_dashboard_exists_en(self): + self.catalog_form_format_obj.raw["dashboard"] = self.dashboard + rt = self.catalog_form_format_obj.get_dashboard(lang="en") + self.assertEqual(rt, [{ + "id": "xxx", "name": "x11", 'link': 'https://grafana.com/d/xxx'}]) diff --git a/kdp_catalog_manager/domain/format/test/test_format_catalog_runtime_application.py b/kdp_catalog_manager/domain/format/test/test_format_catalog_runtime_application.py new file mode 100644 index 0000000..32e4ad2 --- /dev/null +++ b/kdp_catalog_manager/domain/format/test/test_format_catalog_runtime_application.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- + +from unittest import TestCase + +from ..format_catalog_runtime_application import FormatCatalogRuntimeApplication, format_application + + +TEST_TIME = "0001-01-01T00:00:00Z" + + +CATALOG_RUNTIME_APPLICATION = { + "name": "admin-admin-bdos-file-registry", + "appFormName": "mysql", + "appTemplateType": "file-registry", + "appRuntimeName": "", + "appRuntimeNs": "admin", + "bdc": { + "name": "admin-admin", + "alias": "", + "description": "", + "orgName": "admin", + "status": "" + }, + "createTime": TEST_TIME, + "updateTime": TEST_TIME, + "labels": { + "app": "bdos-file-registry", + "app.core.bdos/type": "system", + "bdc.bdos.io/name": "admin-admin", + "bdc.bdos.io/org": "admin" + }, + "status": "running" +} + + +class TestFormatCatalogRuntimeApplication(TestCase): + def setUp(self): + self.maxDiff = None + self.catalog_runtime_application_data = CATALOG_RUNTIME_APPLICATION + self.catalog_runtime_application_format_obj = ( + FormatCatalogRuntimeApplication( + self.catalog_runtime_application_data + ) + ) + + def test_get_bdc_name(self): + rt = self.catalog_runtime_application_format_obj.get_bdc_name() + self.assertEqual(rt, "admin-admin") + + def test_get_bdc_org(self): + rt = self.catalog_runtime_application_format_obj.get_bdc_org() + self.assertEqual(rt, "admin") + + def test_get_bdc_status(self): + rt = self.catalog_runtime_application_format_obj.get_bdc_status() + self.assertEqual(rt, "") + + def test_get_app_name(self): + rt = self.catalog_runtime_application_format_obj.get_app_name() + self.assertEqual(rt, "admin-admin-bdos-file-registry") + + def test_get_app_form(self): + rt = self.catalog_runtime_application_format_obj.get_app_form() + self.assertEqual(rt, "mysql") + + def test_get_app_runtime_name(self): + rt = self.catalog_runtime_application_format_obj.get_app_runtime_name() + self.assertEqual(rt, "") + + def test_get_app_createtime(self): + rt = self.catalog_runtime_application_format_obj.get_app_createtime() + self.assertEqual(rt, TEST_TIME) + + def test_get_app_updatetime(self): + rt = self.catalog_runtime_application_format_obj.get_app_updatetime() + self.assertEqual(rt, TEST_TIME) + + def test_get_app_status(self): + rt = self.catalog_runtime_application_format_obj.get_app_status() + self.assertEqual(rt, "running") + + +class TestFormatApplication(TestCase): + def setUp(self): + self.maxDiff = None + self.all_application = CATALOG_RUNTIME_APPLICATION + self.form_info = {"mysql": "mysql", "spark": "spark"} + self.filter_catalog = None + self.filter_form = None + + self.rt = CATALOG_RUNTIME_APPLICATION + self.add_application = { + "appFormName": "test", + "bdc": { + "name": "", + "orgName": "admin" + } + } + + def test_format_application(self): + rt = format_application( + [self.all_application], + self.form_info, + self.filter_catalog, + self.filter_form + ) + self.assertEqual(rt, [self.rt]) + + def test_format_application_not_match_catalog(self): + rt = format_application( + [self.all_application, self.add_application], + self.form_info, + self.filter_catalog, + self.filter_form + ) + self.assertEqual(rt, [self.rt]) + + def test_format_application_filter_catalog(self): + self.add_application["appFormName"] = "spark" + rt = format_application( + [self.all_application, self.add_application], + self.form_info, + "mysql", + self.filter_form + ) + self.assertEqual(rt, [self.rt]) + + def test_format_application_filter_catalog1(self): + self.add_application["appFormName"] = "spark" + rt = format_application( + [self.all_application, self.add_application], + self.form_info, + "spark", + self.filter_form + ) + self.assertEqual(rt, [self.add_application]) + + def test_format_application_filter_catalog_form(self): + self.add_application["appFormName"] = "spark" + rt = format_application( + [self.all_application, self.add_application], + self.form_info, + self.filter_catalog, + "mysql" + ) + self.assertEqual(rt, [self.rt]) diff --git a/kdp_catalog_manager/domain/model/__init__.py b/kdp_catalog_manager/domain/model/__init__.py new file mode 100644 index 0000000..ff6ea1e --- /dev/null +++ b/kdp_catalog_manager/domain/model/__init__.py @@ -0,0 +1,2 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- diff --git a/kdp_catalog_manager/domain/model/base.py b/kdp_catalog_manager/domain/model/base.py new file mode 100644 index 0000000..3f6ef3b --- /dev/null +++ b/kdp_catalog_manager/domain/model/base.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +from pydantic import BaseModel, Field + + +class Response(BaseModel): + status: int = Field(default=0, description="状态码， 1：失败， 0：成功") + message: str = Field(description="描述信息") + data: dict = Field(description="返回数据") + error: dict = Field(description="错误信息") diff --git a/kdp_catalog_manager/domain/model/catalog.py b/kdp_catalog_manager/domain/model/catalog.py new file mode 100644 index 0000000..793f1b7 --- /dev/null +++ b/kdp_catalog_manager/domain/model/catalog.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +from typing import List, Dict + +from pydantic import BaseModel, Field + +from kdp_catalog_manager.domain.model.base import Response + + +class CatalogData(BaseModel): + name: str = Field(description="应用目录") + description: str = Field(description="应用目录说明") + category: str = Field(description="应用目录分类") + + +class CatalogDataOut(Response): + data: CatalogData | Dict + + +class CatalogList(Response): + data: List[CatalogData] | List diff --git a/kdp_catalog_manager/domain/model/catalog_app_form.py b/kdp_catalog_manager/domain/model/catalog_app_form.py new file mode 100644 index 0000000..55ada91 --- /dev/null +++ b/kdp_catalog_manager/domain/model/catalog_app_form.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +from typing import List, Dict + +from pydantic import BaseModel, Field + +from kdp_catalog_manager.domain.model.base import Response + + +class CatalogFormDataDashboardUrl(BaseModel): + name: str = Field(description="面板名称") + id: str = Field(description="面板id") + link: str = Field(description="面板地址") + + +class CatalogFormData(BaseModel): + name: str | None = Field(description="应用模板名称") + version: str | None = Field(description="应用模板版本") + alias: str | None = Field(description="应用模板别名") + invisible: bool = Field( + default=False, description="应用模板是否隐藏") + isGlobal: bool = Field(default=False, description="是否为全局级别") + description: str | None = Field(description="应用模板说明") + catalog: str = Field(description="应用目录") + dashboardUrl: List[CatalogFormDataDashboardUrl] | List + + +class CatalogFormList(Response): + data: List[CatalogFormData] | List + + +class CatalogForm(Response): + data: CatalogFormData | Dict + + +class CatalogFormInstallDataList(BaseModel): + org: str = Field(description="机构名称") + bdc: list = [] + + +class CatalogFormInstallData(BaseModel): + name: str = Field(description="form名称") + installtion: List[CatalogFormInstallDataList] + + +class CatalogFormInstall(Response): + data: CatalogFormInstallData diff --git a/kdp_catalog_manager/domain/model/catalog_app_runtime.py b/kdp_catalog_manager/domain/model/catalog_app_runtime.py new file mode 100644 index 0000000..d6212b7 --- /dev/null +++ b/kdp_catalog_manager/domain/model/catalog_app_runtime.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +from typing import List + +from pydantic import BaseModel, Field + +from kdp_catalog_manager.domain.model.base import Response + + +class CatalogAppListDataMetadata(BaseModel): + catalog: str = Field(description="应用目录名称") + category: str = Field(description="应用目录分类") + appForm: str = Field(description="应用模板名称") + appName: str = Field(description="应用名称") + appRuntime: str = Field(description="应用实例名称") + org: str = Field(description="机构名称") + bdc: str = Field(description="大数据集群名称") + + +class CatalogAppListData(BaseModel): + name: str = Field(description="应用名称") + isGlobal: bool = Field(default=False, description="是否为全局级别") + status: str = Field(description="应用状态") + updataTime: str = Field(description="应用更新时间") + metadata: CatalogAppListDataMetadata + + +class CatalogAppList(Response): + data: List diff --git a/kdp_catalog_manager/domain/service/__init__.py b/kdp_catalog_manager/domain/service/__init__.py new file mode 100644 index 0000000..ff6ea1e --- /dev/null +++ b/kdp_catalog_manager/domain/service/__init__.py @@ -0,0 +1,2 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- diff --git a/kdp_catalog_manager/domain/service/application.py b/kdp_catalog_manager/domain/service/application.py new file mode 100644 index 0000000..159216a --- /dev/null +++ b/kdp_catalog_manager/domain/service/application.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +from kdp_catalog_manager.domain.format.format_catalog_runtime_application import \ + format_application +from kdp_catalog_manager.domain.service.catalog_form import \ + CatalogFormController +from kdp_catalog_manager.exceptions.exception import GetDataError, \ + get_exception +from kdp_catalog_manager.modules.http_requests.application.oam_application import \ + OamApplication +from kdp_catalog_manager.utils.log import log + + +class ApplicationController(object): + def __init__(self, bdc=None, catalog=None, app_form=None): + self.bdc = bdc + self.catalog = catalog + self.app_form = app_form + + def get_applications(self, label_selector): + try: + form_info = CatalogFormController().get_form_info() + except Exception: + log.error(get_exception()) + raise GetDataError("get form catalog error") + + params = {} + if self.bdc: + params["bdcName"] = self.bdc + if label_selector: + params["labelSelector"] = label_selector + if not params: + params = None + all_application = OamApplication().list_all(params) + applications = format_application( + all_application, form_info, self.catalog, self.app_form) + return applications diff --git a/kdp_catalog_manager/domain/service/catalog.py b/kdp_catalog_manager/domain/service/catalog.py new file mode 100644 index 0000000..60630b1 --- /dev/null +++ b/kdp_catalog_manager/domain/service/catalog.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +import os.path + +from kdp_catalog_manager.common.constants import CATALOG_KEY, I18N, README_HTML +from kdp_catalog_manager.config.base_config import DEFAULT_LANG, \ + CATALOG_README_DIR +from kdp_catalog_manager.domain.format.format_catalog import FormatCatalog +from kdp_catalog_manager.exceptions.exception import DataTypeError, \ + APIRequestedInvalidParamsError +from kdp_catalog_manager.modules.cache.cache import cache_instance +from kdp_catalog_manager.utils.dictutils import DictUtils +from kdp_catalog_manager.utils.fileutils import FileUtils + + +class CatalogController(object): + def __init__(self, catalog=None, lang=DEFAULT_LANG): + self.catalog = catalog + self.lang = lang + + def get_catalogs_info(self): + catalogs_info = {} + catalogs_data = cache_instance.get(CATALOG_KEY) + if not isinstance(catalogs_data, dict): + raise DataTypeError(f"data type is {type(catalogs_data)}") + for catalog, catalog_info in catalogs_data.items(): + if not catalog_info: + catalogs_info[catalog] = {} + continue + catalog_format_obj = FormatCatalog(catalog_info) + catalogs_info[catalog] = { + "name": catalog_format_obj.get_name(), + "description": catalog_format_obj.get_description(self.lang), + "category": catalog_format_obj.get_category(self.lang) + } + return catalogs_info + + def get_catalogs(self): + catalogs = [] + catalogs_info = self.get_catalogs_info() + if not catalogs_info: + return catalogs + for catalog_info in catalogs_info.values(): + catalogs.append(catalog_info) + return catalogs + + def get_catalog(self): + catalogs_info = self.get_catalogs_info() + catalog_data = DictUtils().get_items(catalogs_info, [self.catalog], {}) + return catalog_data + + def get_catalog_readme(self): + if self.catalog is None: + raise APIRequestedInvalidParamsError("catalog is None") + catalog_file = os.path.join( + CATALOG_README_DIR, + self.catalog, + I18N, + self.lang, + README_HTML + ) + return FileUtils().read_file(catalog_file) diff --git a/kdp_catalog_manager/domain/service/catalog_form.py b/kdp_catalog_manager/domain/service/catalog_form.py new file mode 100644 index 0000000..cb21a7a --- /dev/null +++ b/kdp_catalog_manager/domain/service/catalog_form.py @@ -0,0 +1,170 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +import os + +from kdp_catalog_manager.common.constants import CATALOG_FORM_KEY, \ + CATALOG_FROM_DATA_KEY, I18N, README_HTML +from kdp_catalog_manager.config.base_config import DEFAULT_LANG, \ + CATALOG_FROM_README_DIR +from kdp_catalog_manager.domain.format.format_catalog_form import \ + FormatCatalogForm, format_catalog_form_metadata +from kdp_catalog_manager.exceptions.exception import CacheNotExistsError, \ + CatalogFormMatchError, APIRequestedInvalidParamsError +from kdp_catalog_manager.modules.cache.cache import cache_instance +from kdp_catalog_manager.utils.dictutils import DictUtils +from kdp_catalog_manager.utils.fileutils import FileUtils +from kdp_catalog_manager.modules.http_requests.application.oam_application import OamApplication +from kdp_catalog_manager.domain.format.format_catalog_runtime_application import FormatCatalogRuntimeApplication +from kdp_catalog_manager.utils.log import log + + +class CatalogFormController(object): + def __init__( + self, catalog=None, app_form=None, + lang=DEFAULT_LANG, invisible=False + ): + self.catalog = catalog + self.app_form = app_form + self.lang = lang + self.invisible = invisible + + def get_catalog_form_metadata_for_cache(self): + """ + get catalog from metadata from cache + :return: + """ + catalog_form_metadatas = cache_instance.get(CATALOG_FORM_KEY) + if not catalog_form_metadatas: + raise CacheNotExistsError( + "catalog from cache not exists, please re-initialize") + return catalog_form_metadatas + + def get_format_catalog_metadata(self): + """ + get format catalog + :return: + { + "catalog": + { + "catalog_form": + { + "name": "xxxx", + "version": "", + "alias": "Mysql", + "isGlobal": false, + "description": "xxx", + "invisible": false + } + } + } + """ + catalog_form_metadata_cache = self.get_catalog_form_metadata_for_cache() + catalog_form_metadata = format_catalog_form_metadata( + catalog_form_metadata_cache, + self.lang, + self.invisible + ) + return catalog_form_metadata + + def get_catalogs_forms(self): + catalog_forms = [] + catalog_form_metadata = self.get_format_catalog_metadata() + catalog_forms_metadta = DictUtils().get_items( + catalog_form_metadata, self.catalog) + if not catalog_forms_metadta: + return catalog_forms + for catalog_form_info in catalog_forms_metadta.values(): + catalog_forms.append(catalog_form_info) + return catalog_forms + + def get_catalogs_form(self): + catalog_form = {} + catalog_form_metadata = self.get_format_catalog_metadata() + catalog_forms_metadta = DictUtils().get_items( + catalog_form_metadata, self.catalog) + if not catalog_forms_metadta: + return catalog_form + + catalog_form = DictUtils().get_items( + catalog_forms_metadta, [self.app_form]) + if not catalog_form: + raise CatalogFormMatchError( + f"{self.catalog} not found {self.app_form}") + + return catalog_form + + def get_catalog_form_data(self): + catalog_form_data = cache_instance.get( + CATALOG_FROM_DATA_KEY.format(self.catalog, self.app_form) + ) + if not catalog_form_data: + return {} + return catalog_form_data + + def get_catalog_forms_install(self, org=None, bdc=None): + params = None + if bdc: + params = { + "bdcName": bdc + } + form_application_install = {} + all_application_data = OamApplication().list_all(params) + # deal with get application data is empty list or None + if all_application_data: + for application_data in all_application_data: + if not application_data: + continue + application_format_obj = ( + FormatCatalogRuntimeApplication(application_data)) + + application_org = application_format_obj.get_bdc_org() + application_bdc = application_format_obj.get_bdc_name() + application_form = application_format_obj.get_app_form() + if self.app_form != application_form: + continue + if org and org != application_org: + continue + + if application_org not in form_application_install: + form_application_install[application_org] = [application_bdc] + continue + if application_bdc not in form_application_install[application_org]: + form_application_install[application_org].append(application_bdc) + + install_data = [] + if form_application_install: + for org, bdc in form_application_install.items(): + install_data.append({ + "org": org, + "bdc": bdc + }) + + catalog_forms_install = { + "name": self.app_form, + "installtion": install_data + } + return catalog_forms_install + + def get_catalog_form_readme(self): + if self.catalog is None or self.app_form is None: + raise APIRequestedInvalidParamsError("catalog or app form is None") + catalog_file = os.path.join( + CATALOG_FROM_README_DIR, + self.app_form, + I18N, + self.lang, + README_HTML + ) + return FileUtils().read_file(catalog_file) + + def get_form_info(self): + """ + get form info + :return: {"form_name": "catalog_name"} + """ + form_info = {} + catalog_form_metadata = self.get_format_catalog_metadata() + for catalog, catalog_forms in catalog_form_metadata.items(): + for catalog_form in catalog_forms.keys(): + form_info[catalog_form] = catalog + return form_info diff --git a/kdp_catalog_manager/domain/service/markdown_transform_html.py b/kdp_catalog_manager/domain/service/markdown_transform_html.py new file mode 100644 index 0000000..8de02b0 --- /dev/null +++ b/kdp_catalog_manager/domain/service/markdown_transform_html.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +import os + +from kdp_catalog_manager.common.constants import README, I18N +from kdp_catalog_manager.config.base_config import CATALOG_DIR, APPS_DIR, \ + CATALOG_README_DIR, CATALOG_FROM_README_DIR, DEFAULT_LANG +from kdp_catalog_manager.utils.log import log +from kdp_catalog_manager.utils.markdown_to_html.markdownToHtml import \ + MarkdownToHtlm + + +def deal_readme(base_dir, output_file, output_base_dir=CATALOG_README_DIR): + for base_dir_name in os.listdir(base_dir): + form_name = base_dir_name + if base_dir_name.endswith(".app"): + form_name = base_dir_name.split(".")[0] + + # 处理默认语言readme + default_reademe_file = os.path.join(base_dir, base_dir_name, README) + output_dir = os.path.join( + output_base_dir, form_name, I18N, DEFAULT_LANG) + MarkdownToHtlm().markdown_to_html( + default_reademe_file, + output_file, + output_dir + ) + + # 处理其他语言readme + readme_lang_dir = os.path.join(base_dir, base_dir_name, I18N) + for readme_lang in os.listdir(readme_lang_dir): + readme_lang_file = os.path.join(readme_lang_dir, readme_lang, README) + output_dir = os.path.join( + output_base_dir, form_name, I18N, readme_lang + ) + MarkdownToHtlm().markdown_to_html( + readme_lang_file, + output_file, + output_dir + ) + + +def catalog_readme(): + output_file = README.replace("md", "html") + deal_readme(CATALOG_DIR, output_file, CATALOG_README_DIR) + log.info("transform catalog readme to html success") + + +def catalog_form_readme(): + output_file = README.replace("md", "html") + for catalog in os.listdir(CATALOG_DIR): + catalog_form_dir = os.path.join(CATALOG_DIR, catalog, APPS_DIR) + deal_readme(catalog_form_dir, output_file, CATALOG_FROM_README_DIR) + + log.info("transform catalog app form readme to html success") diff --git a/kdp_catalog_manager/domain/service/save_data_cache.py b/kdp_catalog_manager/domain/service/save_data_cache.py new file mode 100644 index 0000000..f6f34c9 --- /dev/null +++ b/kdp_catalog_manager/domain/service/save_data_cache.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +import os + +from kdp_catalog_manager.common.constants import METADATA_YAML, CATALOG_KEY, \ + CATALOG_FORM_KEY, APP_YAML, CATALOG_FROM_DATA_KEY +from kdp_catalog_manager.config.base_config import CACHE_EXPIRE +from kdp_catalog_manager.config.base_config import CATALOG_DIR, APPS_DIR +from kdp_catalog_manager.exceptions.exception import CacheOperatorError +from kdp_catalog_manager.modules.cache.cache import cache_instance +from kdp_catalog_manager.utils.log import log +from kdp_catalog_manager.utils.yamlutils import YAMLUtils + + +class SaveDataToCache(object): + def __init__(self): + self.cache = cache_instance + + +class SaveCatalogDataCache(SaveDataToCache): + def __init__(self): + super().__init__() + self.catalog_dir = CATALOG_DIR + + def get_catalog_data(self): + catalog_data = {} + for catalog in os.listdir(self.catalog_dir): + catalog_metadata_file = os.path.join( + CATALOG_DIR, catalog, METADATA_YAML) + if not os.path.exists(catalog_metadata_file): + continue + + catalog_metadata = YAMLUtils().load_all_yaml(catalog_metadata_file) + catalog_data[catalog] = catalog_metadata + rt = self.cache.set(CATALOG_KEY, catalog_data, CACHE_EXPIRE) + if rt: + log.info("initialization catalog data to cache successful") + else: + raise CacheOperatorError() + + +class SaveCatalogFormDataCache(SaveDataToCache): + def __init__(self): + super().__init__() + self.catalog_dir = CATALOG_DIR + self.apps = APPS_DIR + + def get_catalog_form_data(self): + catalog_form = {} + for catalog in os.listdir(self.catalog_dir): + catalog_form_dir = os.path.join( + self.catalog_dir, catalog, self.apps) + catalog_form_metadata_info = {} + for catalog_from_app in os.listdir(catalog_form_dir): + if not catalog_from_app.endswith(".app"): + continue + + # 存储元数据至缓存中 + catalog_form_metadata_file = os.path.join( + catalog_form_dir, catalog_from_app, METADATA_YAML) + if not os.path.exists(catalog_form_metadata_file): + continue + + catalog_form_metadata = YAMLUtils().load_all_yaml( + catalog_form_metadata_file) + + catalog_form_app_name = catalog_from_app.split(".")[0] + catalog_form_metadata_info[catalog_form_app_name] = ( + catalog_form_metadata) + + # 存储应用模板信息至缓存中 + catalog_form_data_file = os.path.join( + catalog_form_dir, catalog_from_app, APP_YAML + ) + if not os.path.exists(catalog_form_data_file): + continue + catalog_form_data = YAMLUtils().load_all_yaml( + catalog_form_data_file) + + self.cache.set( + CATALOG_FROM_DATA_KEY.format(catalog, catalog_form_app_name), + catalog_form_data, + CACHE_EXPIRE + ) + + catalog_form[catalog] = catalog_form_metadata_info + rt = self.cache.set(CATALOG_FORM_KEY, catalog_form, CACHE_EXPIRE) + if rt: + log.info("initialization catalog from data to cache successful") + else: + raise CacheOperatorError("save catalog form data to cache failed") diff --git a/kdp_catalog_manager/domain/service/test/__init__.py b/kdp_catalog_manager/domain/service/test/__init__.py new file mode 100644 index 0000000..ff6ea1e --- /dev/null +++ b/kdp_catalog_manager/domain/service/test/__init__.py @@ -0,0 +1,2 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- diff --git a/kdp_catalog_manager/domain/service/test/test_application.py b/kdp_catalog_manager/domain/service/test/test_application.py new file mode 100644 index 0000000..e69de29 diff --git a/kdp_catalog_manager/domain/service/test/test_catalog.py b/kdp_catalog_manager/domain/service/test/test_catalog.py new file mode 100644 index 0000000..445b6b5 --- /dev/null +++ b/kdp_catalog_manager/domain/service/test/test_catalog.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +from unittest import TestCase + +from kdp_catalog_manager.common.constants import CATALOG_KEY +from kdp_catalog_manager.exceptions.exception import FileNotExistsError, \ + APIRequestedInvalidParamsError +from kdp_catalog_manager.modules.cache.cache import cache_instance +from ..catalog import CatalogController + + +class TestCatalogController(TestCase): + def setUp(self): + self.catalog_data = { + "mysql": { + "category": "系统/大数据开发工具", + "description": "mysql", + "i18n": { + "en": { + "category": "system.dataManagement", + "description": "mysql" + } + } + } + } + cache_instance.set(CATALOG_KEY, self.catalog_data) + + def test_get_catalog_data_is_None(self): + self.catalog_data = {"mysql": None} + cache_instance.set(CATALOG_KEY, self.catalog_data) + rt = CatalogController(catalog="mysql").get_catalog() + self.assertEqual(rt, {}) + + def test_get_catalog(self): + self.catalog_data = {"mysql": {}} + cache_instance.set(CATALOG_KEY, self.catalog_data) + rt = CatalogController(catalog="mysql").get_catalog() + self.assertEqual(rt, {}) + + def test_get_catalog_readme(self): + try: + CatalogController(catalog="mysql", lang="zhs").get_catalog_readme() + except FileNotExistsError: + self.assertEqual(True, True) + + def test_get_catalog_readme_catalog_is_None(self): + try: + CatalogController(lang="zhs").get_catalog_readme() + except APIRequestedInvalidParamsError: + self.assertEqual(True, True) diff --git a/kdp_catalog_manager/domain/service/test/test_catalog_form.py b/kdp_catalog_manager/domain/service/test/test_catalog_form.py new file mode 100644 index 0000000..8a7300b --- /dev/null +++ b/kdp_catalog_manager/domain/service/test/test_catalog_form.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +from unittest import TestCase + +from kdp_catalog_manager.common.constants import CATALOG_FORM_KEY +from kdp_catalog_manager.exceptions.exception import FileNotExistsError, \ + APIRequestedInvalidParamsError +from kdp_catalog_manager.modules.cache.cache import cache_instance +from ..catalog_form import CatalogFormController, CacheNotExistsError, \ + CatalogFormMatchError + +TEST_DESC = "Mysql T" + + +class TestCatalogFormController(TestCase): + def setUp(self): + self.catalog_form_metadata = { + "mysql": { + "mysql": { + "version": "1.0.0", + "alias": "mysql", + "description": TEST_DESC, + "isGlobal": False, + "i18n": { + "en": { + "category": "system.dataManagement", + "description": "Mysql M" + } + } + } + } + } + + self.catalog_form_data = { + "apiVersion": "bdc.bdos.io/v1alpha1", + "kind": "Application" + } + + self.rt = { + "mysql": { + "mysql": { + "name": "mysql", + "version": "1.0.0", + "alias": "mysql", + "isGlobal": False, + "description": TEST_DESC, + "invisible": False, + "catalog": "mysql", + "dashboardUrl": [] + } + } + } + + def set_cache(self): + cache_instance.set(CATALOG_FORM_KEY, self.catalog_form_metadata) + + def test_get_catalog_form_metadata_from_cache_not_exists(self): + self.catalog_form_metadata = {} + self.set_cache() + try: + CatalogFormController().get_catalog_form_metadata_for_cache() + except CacheNotExistsError: + self.assertEqual(True, True) + + def test_get_catalog_form_metadata_from_cache(self): + self.set_cache() + rt = CatalogFormController().get_catalog_form_metadata_for_cache() + self.assertEqual(rt, self.catalog_form_metadata) + + def test_get_catalog_form_metadata(self): + self.set_cache() + rt = CatalogFormController().get_format_catalog_metadata() + self.assertEqual(rt, self.rt) + + def test_get_catalogs_forms(self): + self.set_cache() + rt = CatalogFormController( + catalog="mysql", app_form="mysql").get_catalogs_forms() + self.assertEqual(rt, [{ + "name": "mysql", "version": "1.0.0", "alias": 'mysql', + 'invisible': False, 'isGlobal': False, + 'description': TEST_DESC, "catalog": "mysql", 'dashboardUrl': []}]) + + def test_get_catalogs_forms_invisible(self): + self.set_cache() + rt = CatalogFormController( + catalog="mysql", app_form="mysql", invisible=True + ).get_catalogs_forms() + self.assertEqual(rt, []) + + def test_get_catalogs_form_catalog_not_found_from(self): + self.set_cache() + try: + CatalogFormController( + catalog="mysql", app_form="spark").get_catalogs_form() + except CatalogFormMatchError: + self.assertEqual(True, True) + + def test_get_catalogs_form_invisible(self): + self.set_cache() + cache_instance.set("mysql", self.catalog_form_metadata) + rt = CatalogFormController( + catalog="mysql", app_form="mysql", invisible=True + ).get_catalogs_form() + self.assertEqual(rt, {}) + cache_instance.delete("mysql") + + def test_get_catalogs_form(self): + self.set_cache() + cache_instance.set("mysql", self.catalog_form_metadata) + rt = CatalogFormController( + catalog="mysql", app_form="mysql").get_catalogs_form() + self.assertEqual(rt, { + 'name': 'mysql', 'version': '1.0.0', 'alias': 'mysql', + 'isGlobal': False, 'description': 'Mysql T', + 'invisible': False, "catalog": "mysql", 'dashboardUrl': []}) + cache_instance.delete("mysql") + + def test_get_catalog_form_data(self): + self.set_cache() + cache_instance.set("mysql-mysql-data", self.catalog_form_data) + rt = CatalogFormController( + catalog="mysql", app_form="mysql").get_catalog_form_data() + self.assertEqual(rt, self.catalog_form_data) + + def test_get_catalog_form_data_not_exists(self): + self.set_cache() + cache_instance.set("mysql-mysql-data", self.catalog_form_data) + rt = CatalogFormController( + catalog="mysql", app_form="mysql1").get_catalog_form_data() + self.assertEqual(rt, {}) + + def test_get_catalog_form_readme_not_exists(self): + try: + CatalogFormController( + catalog="mysql", app_form="mysql1").get_catalog_form_readme() + except FileNotExistsError: + self.assertEqual(True, True) + + def test_get_catalog_form_readme_not_catalog(self): + try: + CatalogFormController().get_catalog_form_readme() + except APIRequestedInvalidParamsError: + self.assertEqual(True, True) + + def test_get_form_info(self): + rt = CatalogFormController().get_form_info() + self.assertEqual(rt, {"mysql": "mysql"}) diff --git a/kdp_catalog_manager/exceptions/__init__.py b/kdp_catalog_manager/exceptions/__init__.py new file mode 100644 index 0000000..ff6ea1e --- /dev/null +++ b/kdp_catalog_manager/exceptions/__init__.py @@ -0,0 +1,2 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- diff --git a/kdp_catalog_manager/exceptions/err_map.json b/kdp_catalog_manager/exceptions/err_map.json new file mode 100644 index 0000000..51909f8 --- /dev/null +++ b/kdp_catalog_manager/exceptions/err_map.json @@ -0,0 +1,50 @@ +{ + "KdpCatalogManagerError": { + "code": 100000, + "error-name": "KdpCatalogManagerError", + "description": "", + "solution": "请查看日志获取更多错误详情", + "manual": "", + "app": "KDP-Catalog-Manager" + }, + "LoadJSONFileError": { + "code": 100001, + "error-name": "LoadJSONFileError", + "description": "读取的{name} 配置文件内容不是合法的JSON格式", + "solution": "检查文件内容", + "manual": "", + "app": "" + }, + "LangNotSupport":{ + "code": 300001, + "error-name": "LangNotSupport", + "description": "语言不支持", + "solution": "请传输正确语言参数", + "manual": "", + "app": "" + }, + "CatalogFormMatchError": { + "code": 300001, + "error-name": "CatalogFormMatchError", + "description": "catalog 与from 不匹配", + "solution": "检查查询参数是否正确", + "manual": "", + "app": "" + }, + "CacheOperatorError": { + "code": 100101, + "error-name": "CacheOperatorError", + "description": "缓存操作失败: {name}", + "solution": "检查应用缓存数据或重启应用", + "manual": "", + "app": "" + }, + "DataTypeError": { + "code": 100201, + "error-name": "DataTypeError", + "description": "缓存操作失败: {name}", + "solution": "检查应用缓存数据或重启应用", + "manual": "", + "app": "" + } +} \ No newline at end of file diff --git a/kdp_catalog_manager/exceptions/exception.py b/kdp_catalog_manager/exceptions/exception.py new file mode 100644 index 0000000..87747e7 --- /dev/null +++ b/kdp_catalog_manager/exceptions/exception.py @@ -0,0 +1,196 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +import sys +import traceback +import os +from kdp_catalog_manager.utils.log import logger + + +def get_exception(): + # 获取异常的跟踪信息 + exc_traceback = traceback.format_exc() + + # 提取文件名部分并打印跟踪信息 + basename_traceback = os.path.basename(exc_traceback) + logger.error(basename_traceback) + + +class KdpCatalogManagerError(Exception): + def __init__(self, error_cname=None): + self.error_name = self.__class__.__name__ + self.error_details = get_exception() + if not error_cname: + self.error_cname = "Kdp Catalog Manager Error" + else: + self.error_cname = "{}".format(error_cname) + super().__init__(self.error_cname) + + def __repr__(self): + return "'{}({})'".format(self.error_name, self.error_cname) + + +class LangNotSupport(KdpCatalogManagerError): + def __init__(self, name): + self.error_cname = "语言不支持:{}".format(name) + KdpCatalogManagerError.__init__(self, self.error_cname) + + +class DirectoryNotExistsError(KdpCatalogManagerError): + def __init__(self, file_path): + self.error_cname = "路径 {} 不存在".format(file_path) + KdpCatalogManagerError.__init__(self, self.error_cname) + + +class FileNotExistsError(KdpCatalogManagerError): + def __init__(self, file_path): + self.error_cname = "文件 {} 不存在".format(file_path) + KdpCatalogManagerError.__init__(self, self.error_cname) + + +class ReadFileError(KdpCatalogManagerError): + def __init__(self, file_path): + self.error_cname = "读取文件 {} 失败".format(file_path) + KdpCatalogManagerError.__init__(self, self.error_cname) + + +class HTTPRequestError(KdpCatalogManagerError): + def __init__(self, method, url, params=None, payload=None, msg=None): + __basic_error_cname = "请求method={} url={}失败 {},请检查该服务是否运行正常".format(method, url, msg) + self.error_cname = __basic_error_cname + if params: + self.error_cname = "{}(请求参数Params={})".format(__basic_error_cname, params) + if payload: + self.error_cname = "{}(请求参数Body={})".format(__basic_error_cname, payload) + if params and payload: + self.error_cname = "{}(请求参数Params={},Body={})".format(__basic_error_cname, params, payload) + KdpCatalogManagerError.__init__(self, self.error_cname) + + +class APIRequestedURLNotFoundError(KdpCatalogManagerError): + def __init__(self): + self.error_cname = "API 请求URL不存在" + KdpCatalogManagerError.__init__(self, self.error_cname) + + +class APIRequestedInvalidParamsError(KdpCatalogManagerError): + def __init__(self, error_msg=""): + self.error_cname = "API 请求参数不正确:{}".format(error_msg) + KdpCatalogManagerError.__init__(self, self.error_cname) + + +class WriteFileError(KdpCatalogManagerError): + def __init__(self, file_path): + self.error_cname = "保存(写)文件 {} 失败".format(file_path) + KdpCatalogManagerError.__init__(self, self.error_cname) + + +class FileOpsForbidden(KdpCatalogManagerError): + def __init__(self, file_path): + self.error_cname = "文件路径 {} 不合法".format(file_path) + KdpCatalogManagerError.__init__(self, self.error_cname) + + +class CreateSoftLinkError(KdpCatalogManagerError): + def __init__(self, src, dst): + self.error_cname = "创建软链 {}->{} 失败".format(src, dst) + KdpCatalogManagerError.__init__(self, self.error_cname) + + +class MaxTryError(KdpCatalogManagerError): + """超过最大重试次数""" + + +class LoadYamlError(KdpCatalogManagerError): + def __init__(self, name=""): + self.error_cname = "获取YAML失败:{}".format(name) + KdpCatalogManagerError.__init__(self, self.error_cname) + + +class CopyFileError(KdpCatalogManagerError): + def __init__(self, error_msg): + self.error_cname = "拷贝文件失败 {}".format(error_msg) + KdpCatalogManagerError.__init__(self, self.error_cname) + + +class DirNotExistsError(KdpCatalogManagerError): + def __init__(self, dir_path): + self.error_cname = "目录 {} 不存在".format(dir_path) + KdpCatalogManagerError.__init__(self, self.error_cname) + + +class CopyDirError(KdpCatalogManagerError): + def __init__(self, error_msg): + self.error_cname = "拷贝目录失败 {}".format(error_msg) + KdpCatalogManagerError.__init__(self, self.error_cname) + + +class RemoveDirError(KdpCatalogManagerError): + def __init__(self, error_msg): + self.error_cname = "删除目录失败 {}".format(error_msg) + KdpCatalogManagerError.__init__(self, self.error_cname) + + +class GetAppInfoError(KdpCatalogManagerError): + def __init__(self, error_msg=""): + self.error_cname = "获取应用描述信息失败 {}".format(error_msg) + KdpCatalogManagerError.__init__(self, self.error_cname) + + +class ConvertEngineDataError(KdpCatalogManagerError): + def __init__(self, name=""): + self.error_cname = "[实例化应用]转换配置失败 {}".format(name) + KdpCatalogManagerError.__init__(self, self.error_cname) + + +class JSONSchemaError(KdpCatalogManagerError): + def __init__(self, name=""): + self.error_cname = "JSONSchemaError:{}".format(name) + KdpCatalogManagerError.__init__(self, self.error_cname) + + +class CacheNotSupportError(KdpCatalogManagerError): + def __init__(self, name=""): + self.error_cname = f"缓存类型{name}不支持" + KdpCatalogManagerError.__init__(self, self.error_cname) + + +class CacheOperatorError(KdpCatalogManagerError): + def __init__(self, name=""): + self.error_cname = f"缓存操作失败: {name}" + KdpCatalogManagerError.__init__(self, self.error_cname) + + +class CacheNotExistsError(KdpCatalogManagerError): + def __init__(self, name=""): + self.error_cname = f"缓存不存在: {name}" + KdpCatalogManagerError.__init__(self, self.error_cname) + + +class LoadJSONFileError(KdpCatalogManagerError): + def __init__(self, file_path): + self.error_cname = f"读取的{file_path} 配置文件内容不是合法的JSON格式" + KdpCatalogManagerError.__init__(self, self.error_cname) + + +class DataNoneError(KdpCatalogManagerError): + def __init__(self, file_path): + self.error_cname = f"内容为空:{file_path}" + KdpCatalogManagerError.__init__(self, self.error_cname) + + +class CatalogFormMatchError(KdpCatalogManagerError): + def __init__(self, error_msg): + self.error_cname = f"应用目录与应用模板不匹配: {error_msg}" + KdpCatalogManagerError.__init__(self, self.error_cname) + + +class DataTypeError(KdpCatalogManagerError): + def __init__(self, error_msg): + self.error_cname = f"数据类型异常: {error_msg}" + KdpCatalogManagerError.__init__(self, self.error_cname) + + +class GetDataError(KdpCatalogManagerError): + def __init__(self, error_msg): + self.error_cname = f"获取数据类型: {error_msg}" + KdpCatalogManagerError.__init__(self, self.error_cname) \ No newline at end of file diff --git a/kdp_catalog_manager/main.py b/kdp_catalog_manager/main.py new file mode 100644 index 0000000..1b31910 --- /dev/null +++ b/kdp_catalog_manager/main.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +import os +from contextlib import asynccontextmanager + +import fastapi.openapi.utils as fu +from fastapi import FastAPI, status, Request, HTTPException +from fastapi.encoders import jsonable_encoder +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse + +from kdp_catalog_manager.api.api_v1.api import api_router +from kdp_catalog_manager.domain.service.markdown_transform_html import \ + catalog_readme, catalog_form_readme +from kdp_catalog_manager.domain.service.save_data_cache import \ + SaveCatalogDataCache, SaveCatalogFormDataCache +from kdp_catalog_manager.utils.format_return import FormatReturn +from kdp_catalog_manager.utils.log import log +from prometheus_fastapi_instrumentator import Instrumentator + + +@asynccontextmanager +async def lifespan(app: FastAPI): + print("prestart runing ......") + instrumentator.expose(app) + yield + print("running before closing ......") + + +app = FastAPI( + title="KDP Catalog Manager", + version="1.0.0", + description="应用目录管理", + license_info={ + "name": "Apache 2.0", + "url": "https://www.apache.org/licenses/LICENSE-2.0.html", + }, + openapi_url="/api/v1/openapi.json", + lifespan=lifespan +) + +# 添加监控指标 +instrumentator = Instrumentator().instrument(app) + +# 添加路由前缀 +app.include_router(api_router, prefix="/api/v1") + + +@app.get("/") +async def index(): + return {"ping": "pong"} + + +@app.exception_handler(RequestValidationError) +async def validation_exception_handler(request: Request, exc: RequestValidationError): + error_info = FormatReturn().format_error_info( + "RequestValidationError", + exc.errors(), + error_msg=exc.errors() + ) + rtn = FormatReturn().format_return_json( + {}, status=1, msg=exc.errors(), error_info=error_info) + return JSONResponse( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + content=jsonable_encoder( + rtn + ) + ) + + +@app.exception_handler(404) +def not_found_exception_handler(request: Request, exc: HTTPException): + return JSONResponse( + status_code=404, + content=FormatReturn().format_return_json( + data=exc.detail, + status=1, + msg=f"{request.query_params} {exc.detail}" + ) + ) + + +fu.validation_error_response_definition = { + "title": "HTTPValidationError", + "type": "object", + "properties": { + "status": {"title": "err status", "type": "int", "default": 1}, + "message": {"title": "Message", "type": "string"}, + "data": {"title": "data Message", "type": "object"}, + "error": {"title": "err Message", "type": "object"} + } +} + +if not os.environ.get("Test"): + SaveCatalogDataCache().get_catalog_data() + SaveCatalogFormDataCache().get_catalog_form_data() + + catalog_readme() + log.info("initialization catalog readme transform to html successful") + catalog_form_readme() + log.info("initialization catalog form readme transform to html successful") + + +if __name__ == '__main__': + import uvicorn + uvicorn.run( + "main:app", host="0.0.0.0", port=8888, + ) diff --git a/kdp_catalog_manager/modules/__init__.py b/kdp_catalog_manager/modules/__init__.py new file mode 100644 index 0000000..ff6ea1e --- /dev/null +++ b/kdp_catalog_manager/modules/__init__.py @@ -0,0 +1,2 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- diff --git a/kdp_catalog_manager/modules/cache/__init__.py b/kdp_catalog_manager/modules/cache/__init__.py new file mode 100644 index 0000000..ff6ea1e --- /dev/null +++ b/kdp_catalog_manager/modules/cache/__init__.py @@ -0,0 +1,2 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- diff --git a/kdp_catalog_manager/modules/cache/cache.py b/kdp_catalog_manager/modules/cache/cache.py new file mode 100644 index 0000000..9ffb635 --- /dev/null +++ b/kdp_catalog_manager/modules/cache/cache.py @@ -0,0 +1,6 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +from cachelib import SimpleCache + + +cache_instance = SimpleCache() diff --git a/kdp_catalog_manager/modules/http_requests/__init__.py b/kdp_catalog_manager/modules/http_requests/__init__.py new file mode 100644 index 0000000..ff6ea1e --- /dev/null +++ b/kdp_catalog_manager/modules/http_requests/__init__.py @@ -0,0 +1,2 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- diff --git a/kdp_catalog_manager/modules/http_requests/application/__init__.py b/kdp_catalog_manager/modules/http_requests/application/__init__.py new file mode 100644 index 0000000..ff6ea1e --- /dev/null +++ b/kdp_catalog_manager/modules/http_requests/application/__init__.py @@ -0,0 +1,2 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- diff --git a/kdp_catalog_manager/modules/http_requests/application/oam_application.py b/kdp_catalog_manager/modules/http_requests/application/oam_application.py new file mode 100644 index 0000000..46a9c26 --- /dev/null +++ b/kdp_catalog_manager/modules/http_requests/application/oam_application.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +from kdp_catalog_manager.modules.http_requests.base_requests import BaseRequests +from kdp_catalog_manager.common.constants import HTTP_HEADER, RESPONSE_DATA +from kdp_catalog_manager.config.base_config import OAM_BASE_URL, HTTP_TIME_OUT, \ + HTTP_MAX_RETRIES + + +class OamApplication(BaseRequests): + def __init__(self, timeout=HTTP_TIME_OUT, max_retries=HTTP_MAX_RETRIES): + super().__init__(timeout=timeout, max_retries=max_retries) + self.headers = HTTP_HEADER + + def list_all(self, params=None): + _url = "{}/api/v1/applications".format(OAM_BASE_URL) + _response = self.http_request( + "GET", _url, self.headers, params=params + ) + return _response.get(RESPONSE_DATA) diff --git a/kdp_catalog_manager/modules/http_requests/application/test/__init__.py b/kdp_catalog_manager/modules/http_requests/application/test/__init__.py new file mode 100644 index 0000000..ff6ea1e --- /dev/null +++ b/kdp_catalog_manager/modules/http_requests/application/test/__init__.py @@ -0,0 +1,2 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- diff --git a/kdp_catalog_manager/modules/http_requests/application/test/test_oam_application.py b/kdp_catalog_manager/modules/http_requests/application/test/test_oam_application.py new file mode 100644 index 0000000..3db9219 --- /dev/null +++ b/kdp_catalog_manager/modules/http_requests/application/test/test_oam_application.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +from unittest import TestCase, mock +from ..oam_application import OamApplication +from kdp_catalog_manager.exceptions.exception import HTTPRequestError + + +class TestOamApplication(TestCase): + def setUp(self): + self.application = OamApplication(1, 2) + + def test_list_all_normal(self): + with self.assertRaises(HTTPRequestError): + self.application.list_all() + + def test_list_all_success(self): + self.rt = { + "status": 0, + "message": "ok", + "data": [ + ] + } + self.application.http_request = mock.Mock(return_value=self.rt) + response = self.application.list_all() + self.assertEqual(response, []) diff --git a/kdp_catalog_manager/modules/http_requests/base_requests.py b/kdp_catalog_manager/modules/http_requests/base_requests.py new file mode 100644 index 0000000..d84c20a --- /dev/null +++ b/kdp_catalog_manager/modules/http_requests/base_requests.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +import traceback + +import httpx + +from kdp_catalog_manager.common.constants import RESPONSE_NORMAL_CODE, \ + RESPONSE_NOT_FOUND_CODE +from kdp_catalog_manager.config.base_config import HTTP_TIME_OUT, \ + HTTP_MAX_RETRIES +from kdp_catalog_manager.exceptions.exception import HTTPRequestError, \ + APIRequestedURLNotFoundError +from kdp_catalog_manager.utils.log import log + + +class BaseRequests(object): + def __init__(self, timeout=HTTP_TIME_OUT, max_retries=HTTP_MAX_RETRIES): + self.timeout = timeout + self.max_retries = max_retries + + def http_request( + self, + method, + url, + headers, + data=None, + params=None + ): + try: + log.info("request url: [{}]{}".format(method, url)) + log.info("request body: {}".format(data)) + log.info("request params: {}".format(params)) + transport = httpx.HTTPTransport(retries=self.max_retries) + with httpx.Client(transport=transport) as client: + response = client.request( + method, + url, + headers=headers, + params=params, + timeout=self.timeout, + json=data + ) + if response.status_code == RESPONSE_NORMAL_CODE: + return response.json() + if response.status_code == RESPONSE_NOT_FOUND_CODE and \ + response.text == "404: Page Not Found": + raise APIRequestedURLNotFoundError() + else: + raise HTTPRequestError( + method, url, + msg=f"{response.status_code}, response.json()") + except Exception as e: + print(traceback.format_exc()) + raise HTTPRequestError(method, url, msg=str(e)) diff --git a/kdp_catalog_manager/modules/http_requests/test/__init__.py b/kdp_catalog_manager/modules/http_requests/test/__init__.py new file mode 100644 index 0000000..ff6ea1e --- /dev/null +++ b/kdp_catalog_manager/modules/http_requests/test/__init__.py @@ -0,0 +1,2 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- diff --git a/kdp_catalog_manager/modules/http_requests/test/test_base_requests.py b/kdp_catalog_manager/modules/http_requests/test/test_base_requests.py new file mode 100644 index 0000000..56de120 --- /dev/null +++ b/kdp_catalog_manager/modules/http_requests/test/test_base_requests.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +from unittest import TestCase +from unittest import mock + +from kdp_catalog_manager.exceptions.exception import HTTPRequestError +from kdp_catalog_manager.modules.http_requests.base_requests import BaseRequests, RESPONSE_NOT_FOUND_CODE, APIRequestedURLNotFoundError + + +class TestHttpRequest(TestCase): + def setUp(self): + self.maxDiff = None + self.header = {"Content-Type": "application/json; charset=UTF-8"} + self.base_requests = BaseRequests(timeout=1, max_retries=1) + + def test_http_request(self): + rt = True + try: + self.base_requests.http_request( + "GET", "http://127.0.0.1", + self.header + ) + except HTTPRequestError: + self.assertEqual(rt, True) + + def test_http_request1(self): + rt = self.base_requests.http_request( + "GET", "https://wenku.baidu.com/message/getnotice", + self.header) + self.assertEqual(rt.get("status").get("code"), 0) + self.assertEqual(rt, { + "status": {"code": 0, "msg": "get notice success"}, + "data": {"system": [], "diy": [], "advert": [], + "errstr": "get notice success"} + }) + + @mock.patch('httpx.Client') + def test_http_request_404(self, mock_client): + mock_response = mock.MagicMock() + mock_response.status_code = 404 + mock_response.text = "404: Page Not Found" + mock_client.return_value.__enter__.return_value.request.return_value = mock_response + with self.assertRaises(HTTPRequestError): + self.base_requests.http_request( + "GET", "http://127.0.0.1:8002/api/v1/applicationss", + self.header) + + @mock.patch('httpx.Client') + def test_http_request_500(self, mock_client): + mock_response = mock.MagicMock() + mock_response.status_code = 500 + with self.assertRaises(HTTPRequestError): + self.base_requests.http_request( + "GET", "http://127.0.0.1:8002/api/v1/applicationss", + self.header) diff --git a/kdp_catalog_manager/setup.py b/kdp_catalog_manager/setup.py new file mode 100644 index 0000000..e47f412 --- /dev/null +++ b/kdp_catalog_manager/setup.py @@ -0,0 +1,49 @@ +import os +import re +import sys +from Cython.Build import cythonize +from Cython.Compiler import Options +from distutils.core import Extension, setup + +__ABS_DIR_PATH = os.path.abspath('') +exclude_so = ["__init__.py", "setup.py", "main.py", "gunicorn.py"] +sources = ['./'] + +extensions = [] +sources_files = [] +for source in sources: + for dir_path, folder_names, filenames in os.walk(source): + for filename in filter(lambda x: re.match(r'.*[.]py$', x), filenames): + print(filename) + file_path = os.path.join(dir_path, filename) + print(file_path) + if filename not in exclude_so: + sources_files.append(file_path) + if filename not in exclude_so: + print("debug point ", file_path[:-3].replace('/', '.')[2:]) + extensions.append( + Extension(file_path[:-3].replace('/', '.')[2:], [file_path], extra_compile_args=["-Os", "-g0"], + extra_link_args=["-Wl,--strip-all"])) + +print(sources_files) +Options.docstrings = False +compiler_directives = {'optimize.unpack_method_calls': False, 'always_allow_keywords': True} + +try: + setup( + ext_modules=cythonize( + extensions, + exclude=None, + nthreads=20, + quiet=True, + language_level=3, + compiler_directives=compiler_directives + ) + ) + print("Start remove source files...") + for sf in sources_files: + os.remove(sf) + os.remove('{}.c'.format(sf[:-3])) +except Exception as ex: + print("error! ", str(ex)) + sys.exit(1) diff --git a/kdp_catalog_manager/test_main.py b/kdp_catalog_manager/test_main.py new file mode 100644 index 0000000..96f3a47 --- /dev/null +++ b/kdp_catalog_manager/test_main.py @@ -0,0 +1,12 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +from .main import app +from fastapi.testclient import TestClient + +client = TestClient(app) + + +def test_read_main(): + response = client.get("/") + assert response.status_code == 200 + assert response.json() == {"ping": "pong"} diff --git a/kdp_catalog_manager/utils/__init__.py b/kdp_catalog_manager/utils/__init__.py new file mode 100644 index 0000000..ff6ea1e --- /dev/null +++ b/kdp_catalog_manager/utils/__init__.py @@ -0,0 +1,2 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- diff --git a/kdp_catalog_manager/utils/dictutils.py b/kdp_catalog_manager/utils/dictutils.py new file mode 100644 index 0000000..c934c48 --- /dev/null +++ b/kdp_catalog_manager/utils/dictutils.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from functools import reduce + + +class DictUtils(object): + + @staticmethod + def get_items(obj, items, default=None): + """递归获取数据 + + :param obj: + :param default: + :param items: 键列表：['foo', 'bar']，或者用 "." 连接的键路径： ".foo.bar" 或 "foo.bar" + """ + if isinstance(items, str): + items = items.strip(".").split(".") + try: + return reduce(lambda x, i: x[i], items, obj) + except (IndexError, KeyError, TypeError): + return default diff --git a/kdp_catalog_manager/utils/fileutils.py b/kdp_catalog_manager/utils/fileutils.py new file mode 100644 index 0000000..a19bfe0 --- /dev/null +++ b/kdp_catalog_manager/utils/fileutils.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +import os + +from kdp_catalog_manager.exceptions.exception import FileNotExistsError, \ + ReadFileError, WriteFileError +from kdp_catalog_manager.utils.log import log + + +class FileUtils(object): + + @staticmethod + def check_file_exists(file, log_err=True): + if not os.path.lexists(file): + if log_err: + log.debug("File {} not exists!".format(file)) + return False + else: + return True + + def read_file(self, file): + if self.check_file_exists(file): + try: + with open(file) as f: + rtn = f.read() + return rtn + except IOError as e: + log.exception(e) + raise ReadFileError(file) + else: + log.warning("File {} not exists".format(file)) + raise FileNotExistsError(file) + + @staticmethod + def write_file(content, target_file): + try: + with open(target_file, 'w') as f: + f.write(content) + except Exception as err: + log.error(str(err)) + raise WriteFileError(target_file) diff --git a/kdp_catalog_manager/utils/format_return.py b/kdp_catalog_manager/utils/format_return.py new file mode 100644 index 0000000..4d39c5d --- /dev/null +++ b/kdp_catalog_manager/utils/format_return.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +import json +from kdp_catalog_manager.utils.log import log + + +class FormatReturn(object): + + @staticmethod + def format_return_json(data, status=0, msg="", error_info={}): + if error_info: + status = 1 + msg = "BdosException occurred" if msg == "" else msg + data = None if data is None else data + + rtn = {'status': status, 'data': data, + 'message': msg, "error": error_info} + return rtn + + @staticmethod + def format_error_info( + error_name, raw_exception, app_name=None, func_args=None, + error_msg=None, exception_type="error" + ): + log.error(f"{error_name}: {error_msg}") + with open("kdp_catalog_manager/exceptions/err_map.json") as fp: + errors_map_list = json.load(fp) + error_info = { + "type": exception_type, + "exception": raw_exception, + "args": [] if func_args is None else func_args + } + error_data_info = errors_map_list.get( + error_name, + errors_map_list.get("KdpCatalogManagerError") + ) + error_info["info"] = error_data_info + if error_msg: + error_info["info"]['description'] = error_msg + if app_name: + error_info["app"] = app_name + else: + error_info["app"] = error_data_info["app"] + del error_data_info['app'] + + return error_info diff --git a/kdp_catalog_manager/utils/log.py b/kdp_catalog_manager/utils/log.py new file mode 100644 index 0000000..a8db13b --- /dev/null +++ b/kdp_catalog_manager/utils/log.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +import os +import logging +from concurrent_log_handler import ConcurrentRotatingFileHandler +from kdp_catalog_manager.config import log_config + + +# 日志记录器 +logger = logging.getLogger() +# 设置日志级别 +logger.setLevel(log_config.LOG_LEVEL) + +# 设置日志格式 +formatter = logging.Formatter( + "[%(asctime)s] [%(levelname)s] [PID:%(process)d][ThreadID:%(thread)d-%(threadName)s] %(module)s %(funcName)s line:%(lineno)d %(message)s" +) + +to_console = logging.StreamHandler() +to_console.setFormatter(formatter) + + +to_file = ConcurrentRotatingFileHandler( + os.path.join(log_config.LOG_DIR, log_config.LOG_FILE), + mode="a", + maxBytes=log_config.LOG_FILE_MAX_BYTES, + backupCount=log_config.LOG_FILE_BACKUP_COUNT +) +to_file.setFormatter(formatter) + +# 将日志输出至屏幕 +logger.addHandler(to_console) +# 将日志输出至文件 +logger.addHandler(to_file) + +log = logger + + +if not os.path.exists(log_config.LOG_DIR): + os.makedirs(log_config.LOG_DIR) diff --git a/kdp_catalog_manager/utils/markdown_to_html/__init__.py b/kdp_catalog_manager/utils/markdown_to_html/__init__.py new file mode 100644 index 0000000..ff6ea1e --- /dev/null +++ b/kdp_catalog_manager/utils/markdown_to_html/__init__.py @@ -0,0 +1,2 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- diff --git a/kdp_catalog_manager/utils/markdown_to_html/markdownToHtml.py b/kdp_catalog_manager/utils/markdown_to_html/markdownToHtml.py new file mode 100644 index 0000000..b4f86a4 --- /dev/null +++ b/kdp_catalog_manager/utils/markdown_to_html/markdownToHtml.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- + +import os + +import markdown + +from kdp_catalog_manager.config.base_config import EXTEND_EXTENSION +from kdp_catalog_manager.utils.fileutils import FileUtils + +MD_HTML_CSS = "kdp_catalog_manager/utils/markdown_to_html/md_html.css" + + +class HtmlController(object): + def __init__(self, css_file=MD_HTML_CSS): + self.css_file = css_file + self.html_css = os.path.join(os.getcwd(), self.css_file) + + def get_css(self): + return FileUtils().read_file(self.html_css) + + def create(self, data, file_name: str): + css_data = self.get_css() + html_data = css_data + data + FileUtils().write_file(html_data, file_name) + + +class MarkdownToHtlm(HtmlController): + def __init__(self): + super().__init__() + self.extensions = [ + 'extra', + 'tables', + 'codehilite' + ] + self.extend_extensions() + + def extend_extensions(self): + if EXTEND_EXTENSION: + custom_extensions = EXTEND_EXTENSION.split(",") + self.extensions.extend(custom_extensions) + + def mutate_markdown_data(self, input_file): + try: + markdown_data = FileUtils().read_file(input_file) + html_data = markdown.markdown( + markdown_data, + extensions=self.extensions + ) + return html_data + except Exception: + return None + + def markdown_to_html(self, input_file, output_file, output_dir=None): + markdown_html_data = self.mutate_markdown_data(input_file) + if output_dir is not None: + if not os.path.exists(output_dir): + os.makedirs(output_dir) + output_file = os.path.join(output_dir, output_file) + self.create(markdown_html_data, output_file) diff --git a/kdp_catalog_manager/utils/markdown_to_html/md_html.css b/kdp_catalog_manager/utils/markdown_to_html/md_html.css new file mode 100644 index 0000000..1a66764 --- /dev/null +++ b/kdp_catalog_manager/utils/markdown_to_html/md_html.css @@ -0,0 +1,259 @@ + + \ No newline at end of file diff --git a/kdp_catalog_manager/utils/markdown_to_html/test_markdownToHtml.py b/kdp_catalog_manager/utils/markdown_to_html/test_markdownToHtml.py new file mode 100644 index 0000000..278d517 --- /dev/null +++ b/kdp_catalog_manager/utils/markdown_to_html/test_markdownToHtml.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +import os.path +from unittest import TestCase + +from .markdownToHtml import HtmlController, MarkdownToHtlm + + +class TestHtmlController(TestCase): + def setUp(self): + + self.css_file = HtmlController().css_file + with open(self.css_file, "r") as f: + self.css_data = f.read() + + def test_get_css(self): + rt = HtmlController().get_css() + self.assertEqual(rt, self.css_data) + + def test_create(self): + output_file = "test.html" + HtmlController().create("test", output_file) + if os.path.exists(output_file): + self.assertEqual(True, True) + os.remove("test.html") + + +class TestMarkdownToHtlm(TestCase): + def setUp(self): + self.markdown_data = "# desc" + self.markdown_file = "test.md" + with open(self.markdown_file, "w") as f: + f.write(self.markdown_data) + + def test_mutate_markdown_data(self): + rt = MarkdownToHtlm().mutate_markdown_data(self.markdown_file) + self.assertEqual(rt, '