diff --git a/README.md b/README.md index 3f0ecb3..2396a99 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ -# Terarium TA1 Middleware Service +# Terarium Knowledge Middleware -[![codecov](https://codecov.io/github/DARPA-ASKEM/TA1-Service/branch/main/graph/badge.svg?token=XEARJHESHY)](https://codecov.io/github/DARPA-ASKEM/TA1-Service) +[![codecov](https://codecov.io/github/DARPA-ASKEM/knowledge-middleware/branch/main/graph/badge.svg?token=XEARJHESHY)](https://codecov.io/github/DARPA-ASKEM/knowledge-middleware) -The TA1 Middleware Service is designed to provide an intermediate job queue for management of long running TA1 extraction and profiling tasks. It enables the Terarium HMI to request TA1 tasks to be performed asynchronously, with robust error handling, and with customized ETL of TA1 responses into Terarium specific schemas and specifications. It currently supports the following functions: +The Knowledge Middleware is designed to provide an intermediate job queue for management of long-running extraction and profiling tasks. It enables the Terarium HMI to request large knowledge discovery and curation tasks to be performed asynchronously, with robust error handling, and with customized ETL of backend service responses into Terarium specific schemas and specifications. It currently supports the following functions: 1. Equation to AMR: both LaTeX and MathML 2. Code to AMR: code snippets only @@ -44,8 +44,8 @@ pytest --cov . tests ### Testing Configuration -To modify your test configuration, edit the `tests/conftest.py` file. In particular note that setting `LIVE` to `TRUE` and adding the correct endpoints for TA1 will send real payloads to TA1 services and validate the results. +To modify your test configuration, edit the `tests/conftest.py` file. In particular note that setting `LIVE` to `TRUE` and adding the correct endpoints for will send real payloads to backend knowledge-middleware services and validate the results. ## License -[Apache License 2.0](LICENSE) \ No newline at end of file +[Apache License 2.0](LICENSE) diff --git a/api/Dockerfile b/api/Dockerfile index 0ecf4de..4984124 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -15,10 +15,10 @@ RUN pip install --no-cache-dir poetry==1.5.1 RUN poetry config virtualenvs.create false && \ poetry install --no-root --no-cache --with api -COPY api ta1-service/api -COPY lib ta1-service/lib -WORKDIR /ta1-service +COPY api knowledge-middleware/api +COPY lib knowledge-middleware/lib +WORKDIR /knowledge-middleware EXPOSE 8000 -ENV PYTHONPATH=/ta1-service +ENV PYTHONPATH=/knowledge-middleware CMD [ "uvicorn", "api.server:app", "--reload", "--host", "0.0.0.0", "--port", "8000" ] diff --git a/api/server.py b/api/server.py index 622398e..448a3d6 100644 --- a/api/server.py +++ b/api/server.py @@ -21,8 +21,8 @@ def build_api(*args) -> FastAPI: api = FastAPI( - title="Terarium TA1 Middleware Service", - description="Middleware for managing interactions with various TA1 services.", + title="Terarium Knowledge Middleware Service", + description="Middleware for managing interactions with various services.", docs_url="/", ) origins = [ diff --git a/docker-bake.hcl b/docker-bake.hcl index 3240550..f702e4f 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -28,11 +28,11 @@ function "check_suffix" { # ---------------------------------------------------------------------------------------------------------------------- group "prod" { - targets = ["ta1-service-api", "ta1-service-worker"] + targets = ["knowledge-middleware-api", "knowledge-middleware-worker"] } group "default" { - targets = ["ta1-service-api-base", "ta1-service-worker-base"] + targets = ["knowledge-middleware-api-base", "knowledge-middleware-worker-base"] } # ---------------------------------------------------------------------------------------------------------------------- @@ -41,22 +41,22 @@ target "_platforms" { platforms = ["linux/amd64", "linux/arm64"] } -target "ta1-service-api-base" { +target "knowledge-middleware-api-base" { context = "." - tags = tag("ta1-service-api", "", "") + tags = tag("knowledge-middleware-api", "", "") dockerfile = "api/Dockerfile" } -target "ta1-service-worker-base" { +target "knowledge-middleware-worker-base" { context = "." - tags = tag("ta1-service-worker", "", "") + tags = tag("knowledge-middleware-worker", "", "") dockerfile = "worker/Dockerfile" } -target "ta1-service-api" { - inherits = ["_platforms", "ta1-service-api-base"] +target "knowledge-middleware-api" { + inherits = ["_platforms", "knowledge-middleware-api-base"] } -target "ta1-service-worker" { - inherits = ["_platforms", "ta1-service-worker-base"] +target "knowledge-middleware-worker" { + inherits = ["_platforms", "knowledge-middleware-worker-base"] } diff --git a/docker-compose.prod.yaml b/docker-compose.prod.yaml index 532ba1b..5e95cfd 100644 --- a/docker-compose.prod.yaml +++ b/docker-compose.prod.yaml @@ -1,12 +1,12 @@ --- version: "3.9" networks: - ta1-service: + knowledge-middleware: driver: bridge - name: ta1-service + name: knowledge-middleware services: api: - container_name: api-ta1-service + container_name: api-knowledge-middleware build: context: ./ dockerfile: api/Dockerfile @@ -15,7 +15,7 @@ services: env_file: - .env networks: - - ta1-service + - knowledge-middleware depends_on: - redis volumes: @@ -23,14 +23,14 @@ services: extra_hosts: - "host.docker.internal:host-gateway" redis: - container_name: redis-ta1-service + container_name: redis-knowledge-middleware image: redis ports: - "6379:6379" networks: - - ta1-service + - knowledge-middleware rqworker: - container_name: worker-ta1-service + container_name: worker-knowledge-middleware build: context: ./ dockerfile: worker/Dockerfile @@ -39,4 +39,4 @@ services: depends_on: - redis networks: - - ta1-service + - knowledge-middleware diff --git a/docker-compose.yaml b/docker-compose.yaml index d49b30b..937cdc3 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,14 +1,14 @@ --- version: "3.9" networks: - ta1-service: + knowledge-middleware: driver: bridge - name: ta1-service + name: knowledge-middleware data-api: external: true services: extraction-api: - container_name: api-ta1-service + container_name: api-knowledge-middleware build: context: ./ dockerfile: api/Dockerfile @@ -17,7 +17,7 @@ services: env_file: - .env networks: - - ta1-service + - knowledge-middleware - data-api depends_on: - redis @@ -26,14 +26,14 @@ services: extra_hosts: - "host.docker.internal:host-gateway" redis: - container_name: redis-ta1-service + container_name: redis-knowledge-middleware image: redis ports: - "6379:6379" networks: - - ta1-service + - knowledge-middleware rqworker: - container_name: worker-ta1-service + container_name: worker-knowledge-middleware build: context: ./ dockerfile: worker/Dockerfile @@ -42,26 +42,26 @@ services: depends_on: - redis networks: - - ta1-service + - knowledge-middleware - data-api mit-tr: extends: file: ./askem-ta1-dockervm/end-to-end-rest/docker-compose.yml service: mit-tr networks: - - ta1-service + - knowledge-middleware skema-tr: extends: file: ./askem-ta1-dockervm/end-to-end-rest/docker-compose.yml service: skema-tr networks: - - ta1-service + - knowledge-middleware skema-py: extends: file: ./askem-ta1-dockervm/end-to-end-rest/docker-compose.yml service: skema-py networks: - - ta1-service + - knowledge-middleware skema-unified: extends: file: ./askem-ta1-dockervm/end-to-end-rest/docker-compose.yml @@ -71,25 +71,25 @@ services: file: ./askem-ta1-dockervm/end-to-end-rest/docker-compose.yml service: skema-rs networks: - - ta1-service + - knowledge-middleware graphdb: extends: file: ./askem-ta1-dockervm/end-to-end-rest/docker-compose.yml service: graphdb networks: - - ta1-service + - knowledge-middleware eq2mml: extends: file: ./askem-ta1-dockervm/end-to-end-rest/docker-compose.yml service: eq2mml networks: - - ta1-service + - knowledge-middleware mathjax: extends: file: ./askem-ta1-dockervm/end-to-end-rest/docker-compose.yml service: mathjax networks: - - ta1-service + - knowledge-middleware volumes: mg_lib: diff --git a/lib/settings.py b/lib/settings.py index 3eddc98..1233def 100644 --- a/lib/settings.py +++ b/lib/settings.py @@ -2,7 +2,7 @@ class Settings(BaseSettings): LIVE: bool = False - REDIS_HOST: str = "redis.ta1-service" + REDIS_HOST: str = "redis.knowledge-middleware" REDIS_PORT: int = 6379 TA1_UNIFIED_URL: str = "http://ta1:5" SKEMA_RS_URL: str = "http://skema-rs.staging.terarium.ai" @@ -12,4 +12,4 @@ class Settings(BaseSettings): LOG_LEVEL: str = "INFO" -settings = Settings() \ No newline at end of file +settings = Settings() diff --git a/pyproject.toml b/pyproject.toml index 1513a77..bfa670b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [tool.poetry] -name = "ta1-service" +name = "knowledge-middleware" version = "1.6.0" description = "" authors = ["Powell Fendley"] @@ -41,4 +41,4 @@ markers = ["resource"] [build-system] requires = ["poetry-core"] -build-backend = "poetry.core.masonry.api" \ No newline at end of file +build-backend = "poetry.core.masonry.api" diff --git a/worker/operations.py b/worker/operations.py index 986f1d0..845a691 100644 --- a/worker/operations.py +++ b/worker/operations.py @@ -37,7 +37,7 @@ logger.addHandler(handler) -# Worker jobs for TA1 services +# Worker jobs for knowledge services def equations_to_amr(*args, **kwargs): equation_type = kwargs.get("equation_type") equations = kwargs.get("equations") @@ -57,7 +57,7 @@ def equations_to_amr(*args, **kwargs): headers = {"Content-Type": "application/json"} - logger.info(f"Sending equations of type {equation_type} to TA1 at {url}") + logger.info(f"Sending equations of type {equation_type} to backend knowledge services at {url}") if equation_type == "mathml": amr_response = requests.put( url, data=json.dumps(put_payload, default=str), headers=headers @@ -70,7 +70,7 @@ def equations_to_amr(*args, **kwargs): amr_json = amr_response.json() logger.debug(f"TA 1 response object: {amr_response}") except: - logger.error(f"Failed to parse response from TA1 Service: {amr_response.text}") + logger.error(f"Failed to parse response from backend knowledge service: {amr_response.text}") if amr_response.status_code == 200 and amr_json: tds_responses = put_amr_to_tds(amr_json, name, description) @@ -108,11 +108,11 @@ def pdf_to_text(*args, **kwargs): try: logger.info( - f"Sending PDF to TA1 service with artifact id {artifact_id} at {unified_text_reading_url}" + f"Sending PDF to backend knowledge service with artifact id {artifact_id} at {unified_text_reading_url}" ) response = requests.post(unified_text_reading_url, files=put_payload) logger.info( - f"Response received from TA1 with status code: {response.status_code}" + f"Response received from backend knowledge service with status code: {response.status_code}" ) extraction_json = response.json() logger.debug(f"TA 1 response object: {extraction_json}") @@ -171,11 +171,11 @@ def pdf_extractions(*args, **kwargs): try: logger.info( - f"Sending PDF to TA1 service with artifact id {artifact_id} at {unified_text_reading_url}" + f"Sending PDF to backend knowledge service with artifact id {artifact_id} at {unified_text_reading_url}" ) response = requests.post(unified_text_reading_url, json=payload) logger.info( - f"Response received from TA1 with status code: {response.status_code}" + f"Response received from backend knowledge service with status code: {response.status_code}" ) extraction_json = response.json() logger.debug(f"TA 1 response object: {response.text}") @@ -184,14 +184,14 @@ def pdf_extractions(*args, **kwargs): if isinstance(outputs, dict): if extraction_json.get("outputs", {"data": None}).get("data", None) is None: raise ValueError( - f"Malformed or empty response from TA1: {extraction_json}" + f"Malformed or empty response from backend knowledge service: {extraction_json}" ) else: extraction_json = extraction_json.get("outputs").get("data") elif isinstance(outputs, list): if extraction_json.get("outputs")[0].get("data") is None: raise ValueError( - f"Malformed or empty response from TA1: {extraction_json}" + f"Malformed or empty response from backend knowledge service: {extraction_json}" ) else: extraction_json = [extraction_json.get("outputs")[0].get("data")] @@ -380,7 +380,7 @@ def model_card(*args, **kwargs): raise Exception(f"Failed to generate model card for {model_id}: {e}") else: - raise Exception(f"Bad response from TA1 for {model_id}: {resp.status_code}") + raise Exception(f"Bad response from backend knowledge service for {model_id}: {resp.status_code}") # dccde3a0-0132-430c-afd8-c67953298f48 @@ -438,7 +438,7 @@ def link_amr(*args, **kwargs): "message": "Model enriched and updated in TDS", } else: - raise Exception("Response from TA1 service was not 200: {response.text}") + raise Exception("Response from backend knowledge service was not 200: {response.text}") # 60e539e4-6969-4369-a358-c601a3a583da @@ -459,13 +459,13 @@ def code_to_amr(*args, **kwargs): } logger.info( - f"Sending code to TA1 service with artifact id: {artifact_id} at {code_amr_workflow_url}" + f"Sending code to backend knowledge service with artifact id: {artifact_id} at {code_amr_workflow_url}" ) amr_response = requests.post( code_amr_workflow_url, json=json.loads(json.dumps(request_payload)) ) logger.info( - f"Response received from TA1 with status code: {amr_response.status_code}" + f"Response received from backend knowledge service with status code: {amr_response.status_code}" ) amr_json = amr_response @@ -474,7 +474,7 @@ def code_to_amr(*args, **kwargs): amr_json = amr_response.json() logger.debug(f"TA 1 response object: {amr_json}") except: - logger.error(f"Failed to parse response from TA1 Service:\n{amr_response.text}") + logger.error(f"Failed to parse response from backend knowledge service:\n{amr_response.text}") if amr_response.status_code == 200 and amr_json: tds_responses = put_amr_to_tds(amr_json, name, description)