renamed TA1-service to Knowledge-Middleware

README.md

@@ -1,8 +1,8 @@
-# Terarium TA1 Middleware Service
+# Terarium Knowledge Middleware
 
-[![codecov](https://codecov.io/github/DARPA-ASKEM/TA1-Service/branch/main/graph/badge.svg?token=XEARJHESHY)](https://codecov.io/github/DARPA-ASKEM/TA1-Service)
+[![codecov](https://codecov.io/github/DARPA-ASKEM/knowledge-middleware/branch/main/graph/badge.svg?token=XEARJHESHY)](https://codecov.io/github/DARPA-ASKEM/knowledge-middleware)
 
-The TA1 Middleware Service is designed to provide an intermediate job queue for management of long running TA1 extraction and profiling tasks. It enables the Terarium HMI to request TA1 tasks to be performed asynchronously, with robust error handling, and with customized ETL of TA1 responses into Terarium specific schemas and specifications. It currently supports the following functions:
+The Knowledge Middleware is designed to provide an intermediate job queue for management of long-running extraction and profiling tasks. It enables the Terarium HMI to request large knowledge discovery and curation tasks to be performed asynchronously, with robust error handling, and with customized ETL of backend service responses into Terarium specific schemas and specifications. It currently supports the following functions:
 
 1. Equation to AMR: both LaTeX and MathML
 2. Code to AMR: code snippets only
@@ -44,8 +44,8 @@ pytest --cov . tests
 
 ### Testing Configuration
 
-To modify your test configuration, edit the `tests/conftest.py` file. In particular note that setting `LIVE` to `TRUE` and adding the correct endpoints for TA1 will send real payloads to TA1 services and validate the results.
+To modify your test configuration, edit the `tests/conftest.py` file. In particular note that setting `LIVE` to `TRUE` and adding the correct endpoints for will send real payloads to backend knowledge-middleware services and validate the results.
 
 ## License
 
-[Apache License 2.0](LICENSE)
+[Apache License 2.0](LICENSE)

api/Dockerfile

@@ -15,10 +15,10 @@ RUN pip install --no-cache-dir poetry==1.5.1
 RUN poetry config virtualenvs.create false && \
  poetry install --no-root --no-cache --with api
 
-COPY api ta1-service/api
-COPY lib ta1-service/lib
-WORKDIR /ta1-service
+COPY api knowledge-middleware/api
+COPY lib knowledge-middleware/lib
+WORKDIR /knowledge-middleware
 
 EXPOSE 8000
-ENV PYTHONPATH=/ta1-service
+ENV PYTHONPATH=/knowledge-middleware
 CMD [ "uvicorn", "api.server:app", "--reload", "--host", "0.0.0.0", "--port", "8000" ]

api/server.py

@@ -21,8 +21,8 @@
 
 def build_api(*args) -> FastAPI:
  api = FastAPI(
- title="Terarium TA1 Middleware Service",
- description="Middleware for managing interactions with various TA1 services.",
+ title="Terarium Knowledge Middleware Service",
+ description="Middleware for managing interactions with various services.",
  docs_url="/",
  )
  origins = [

docker-bake.hcl

@@ -28,11 +28,11 @@ function "check_suffix" {
 # ----------------------------------------------------------------------------------------------------------------------
 
 group "prod" {
- targets = ["ta1-service-api", "ta1-service-worker"]
+ targets = ["knowledge-middleware-api", "knowledge-middleware-worker"]
 }
 
 group "default" {
- targets = ["ta1-service-api-base", "ta1-service-worker-base"]
+ targets = ["knowledge-middleware-api-base", "knowledge-middleware-worker-base"]
 }
 
 # ----------------------------------------------------------------------------------------------------------------------
@@ -41,22 +41,22 @@ target "_platforms" {
  platforms = ["linux/amd64", "linux/arm64"]
 }
 
-target "ta1-service-api-base" {
+target "knowledge-middleware-api-base" {
  context = "."
- tags = tag("ta1-service-api", "", "")
+ tags = tag("knowledge-middleware-api", "", "")
  dockerfile = "api/Dockerfile"
 }
 
-target "ta1-service-worker-base" {
+target "knowledge-middleware-worker-base" {
  context = "."
- tags = tag("ta1-service-worker", "", "")
+ tags = tag("knowledge-middleware-worker", "", "")
  dockerfile = "worker/Dockerfile"
 }
 
-target "ta1-service-api" {
- inherits = ["_platforms", "ta1-service-api-base"]
+target "knowledge-middleware-api" {
+ inherits = ["_platforms", "knowledge-middleware-api-base"]
 }
 
-target "ta1-service-worker" {
- inherits = ["_platforms", "ta1-service-worker-base"]
+target "knowledge-middleware-worker" {
+ inherits = ["_platforms", "knowledge-middleware-worker-base"]
 }

docker-compose.prod.yaml

@@ -1,12 +1,12 @@
 ---
 version: "3.9"
 networks:
- ta1-service:
+ knowledge-middleware:
  driver: bridge
- name: ta1-service
+ name: knowledge-middleware
 services:
  api:
- container_name: api-ta1-service
+ container_name: api-knowledge-middleware
  build:
  context: ./
  dockerfile: api/Dockerfile
@@ -15,22 +15,22 @@ services:
  env_file:
  - .env
  networks:
- - ta1-service
+ - knowledge-middleware
  depends_on:
  - redis
  volumes:
  - $PWD/api:/api
  extra_hosts:
  - "host.docker.internal:host-gateway"
  redis:
- container_name: redis-ta1-service
+ container_name: redis-knowledge-middleware
  image: redis
  ports:
  - "6379:6379"
  networks:
- - ta1-service
+ - knowledge-middleware
  rqworker:
- container_name: worker-ta1-service
+ container_name: worker-knowledge-middleware
  build:
  context: ./
  dockerfile: worker/Dockerfile
@@ -39,4 +39,4 @@ services:
  depends_on:
  - redis
  networks:
- - ta1-service
+ - knowledge-middleware

docker-compose.yaml

@@ -1,14 +1,14 @@
 ---
 version: "3.9"
 networks:
- ta1-service:
+ knowledge-middleware:
  driver: bridge
- name: ta1-service
+ name: knowledge-middleware
  data-api:
  external: true
 services:
  extraction-api:
- container_name: api-ta1-service
+ container_name: api-knowledge-middleware
  build:
  context: ./
  dockerfile: api/Dockerfile
@@ -17,7 +17,7 @@ services:
  env_file:
  - .env
  networks:
- - ta1-service
+ - knowledge-middleware
  - data-api
  depends_on:
  - redis
@@ -26,14 +26,14 @@ services:
  extra_hosts:
  - "host.docker.internal:host-gateway"
  redis:
- container_name: redis-ta1-service
+ container_name: redis-knowledge-middleware
  image: redis
  ports:
  - "6379:6379"
  networks:
- - ta1-service
+ - knowledge-middleware
  rqworker:
- container_name: worker-ta1-service
+ container_name: worker-knowledge-middleware
  build:
  context: ./
  dockerfile: worker/Dockerfile
@@ -42,26 +42,26 @@ services:
  depends_on:
  - redis
  networks:
- - ta1-service
+ - knowledge-middleware
  - data-api
  mit-tr:
  extends:
  file: ./askem-ta1-dockervm/end-to-end-rest/docker-compose.yml
  service: mit-tr
  networks:
- - ta1-service
+ - knowledge-middleware
  skema-tr:
  extends:
  file: ./askem-ta1-dockervm/end-to-end-rest/docker-compose.yml
  service: skema-tr
  networks:
- - ta1-service
+ - knowledge-middleware
  skema-py:
  extends:
  file: ./askem-ta1-dockervm/end-to-end-rest/docker-compose.yml
  service: skema-py
  networks:
- - ta1-service
+ - knowledge-middleware
  skema-unified:
  extends:
  file: ./askem-ta1-dockervm/end-to-end-rest/docker-compose.yml
@@ -71,25 +71,25 @@ services:
  file: ./askem-ta1-dockervm/end-to-end-rest/docker-compose.yml
  service: skema-rs
  networks:
- - ta1-service
+ - knowledge-middleware
  graphdb:
  extends:
  file: ./askem-ta1-dockervm/end-to-end-rest/docker-compose.yml
  service: graphdb
  networks:
- - ta1-service
+ - knowledge-middleware
  eq2mml:
  extends:
  file: ./askem-ta1-dockervm/end-to-end-rest/docker-compose.yml
  service: eq2mml
  networks:
- - ta1-service
+ - knowledge-middleware
  mathjax:
  extends:
  file: ./askem-ta1-dockervm/end-to-end-rest/docker-compose.yml
  service: mathjax
  networks:
- - ta1-service
+ - knowledge-middleware
 
 volumes:
  mg_lib:

lib/settings.py

@@ -2,7 +2,7 @@
 
 class Settings(BaseSettings):
  LIVE: bool = False
- REDIS_HOST: str = "redis.ta1-service"
+ REDIS_HOST: str = "redis.knowledge-middleware"
  REDIS_PORT: int = 6379
  TA1_UNIFIED_URL: str = "http://ta1:5"
  SKEMA_RS_URL: str = "http://skema-rs.staging.terarium.ai"
@@ -12,4 +12,4 @@ class Settings(BaseSettings):
  LOG_LEVEL: str = "INFO"
 
 
-settings = Settings()
+settings = Settings()

pyproject.toml

@@ -1,5 +1,5 @@
 [tool.poetry]
-name = "ta1-service"
+name = "knowledge-middleware"
 version = "1.6.0"
 description = ""
 authors = ["Powell Fendley"]
@@ -41,4 +41,4 @@ markers = ["resource"]
 
 [build-system]
 requires = ["poetry-core"]
-build-backend = "poetry.core.masonry.api"
+build-backend = "poetry.core.masonry.api"

worker/operations.py

@@ -37,7 +37,7 @@
 logger.addHandler(handler)
 
 
-# Worker jobs for TA1 services
+# Worker jobs for knowledge services
 def equations_to_amr(*args, **kwargs):
  equation_type = kwargs.get("equation_type")
  equations = kwargs.get("equations")
@@ -57,7 +57,7 @@ def equations_to_amr(*args, **kwargs):
 
  headers = {"Content-Type": "application/json"}
 
- logger.info(f"Sending equations of type {equation_type} to TA1 at {url}")
+ logger.info(f"Sending equations of type {equation_type} to backend knowledge services at {url}")
  if equation_type == "mathml":
  amr_response = requests.put(
  url, data=json.dumps(put_payload, default=str), headers=headers
@@ -70,7 +70,7 @@ def equations_to_amr(*args, **kwargs):
  amr_json = amr_response.json()
  logger.debug(f"TA 1 response object: {amr_response}")
  except:
- logger.error(f"Failed to parse response from TA1 Service: {amr_response.text}")
+ logger.error(f"Failed to parse response from backend knowledge service: {amr_response.text}")
 
  if amr_response.status_code == 200 and amr_json:
  tds_responses = put_amr_to_tds(amr_json, name, description)
@@ -108,11 +108,11 @@ def pdf_to_text(*args, **kwargs):
 
  try:
  logger.info(
- f"Sending PDF to TA1 service with artifact id {artifact_id} at {unified_text_reading_url}"
+ f"Sending PDF to backend knowledge service with artifact id {artifact_id} at {unified_text_reading_url}"
  )
  response = requests.post(unified_text_reading_url, files=put_payload)
  logger.info(
- f"Response received from TA1 with status code: {response.status_code}"
+ f"Response received from backend knowledge service with status code: {response.status_code}"
  )
  extraction_json = response.json()
  logger.debug(f"TA 1 response object: {extraction_json}")
@@ -171,11 +171,11 @@ def pdf_extractions(*args, **kwargs):
 
  try:
  logger.info(
- f"Sending PDF to TA1 service with artifact id {artifact_id} at {unified_text_reading_url}"
+ f"Sending PDF to backend knowledge service with artifact id {artifact_id} at {unified_text_reading_url}"
  )
  response = requests.post(unified_text_reading_url, json=payload)
  logger.info(
- f"Response received from TA1 with status code: {response.status_code}"
+ f"Response received from backend knowledge service with status code: {response.status_code}"
  )
  extraction_json = response.json()
  logger.debug(f"TA 1 response object: {response.text}")
@@ -184,14 +184,14 @@ def pdf_extractions(*args, **kwargs):
  if isinstance(outputs, dict):
  if extraction_json.get("outputs", {"data": None}).get("data", None) is None:
  raise ValueError(
- f"Malformed or empty response from TA1: {extraction_json}"
+ f"Malformed or empty response from backend knowledge service: {extraction_json}"
  )
  else:
  extraction_json = extraction_json.get("outputs").get("data")
  elif isinstance(outputs, list):
  if extraction_json.get("outputs")[0].get("data") is None:
  raise ValueError(
- f"Malformed or empty response from TA1: {extraction_json}"
+ f"Malformed or empty response from backend knowledge service: {extraction_json}"
  )
  else:
  extraction_json = [extraction_json.get("outputs")[0].get("data")]
@@ -380,7 +380,7 @@ def model_card(*args, **kwargs):
  raise Exception(f"Failed to generate model card for {model_id}: {e}")
 
  else:
- raise Exception(f"Bad response from TA1 for {model_id}: {resp.status_code}")
+ raise Exception(f"Bad response from backend knowledge service for {model_id}: {resp.status_code}")
 
 
 # dccde3a0-0132-430c-afd8-c67953298f48
@@ -438,7 +438,7 @@ def link_amr(*args, **kwargs):
  "message": "Model enriched and updated in TDS",
  }
  else:
- raise Exception("Response from TA1 service was not 200: {response.text}")
+ raise Exception("Response from backend knowledge service was not 200: {response.text}")
 
 
 # 60e539e4-6969-4369-a358-c601a3a583da
@@ -459,13 +459,13 @@ def code_to_amr(*args, **kwargs):
  }
 
  logger.info(
- f"Sending code to TA1 service with artifact id: {artifact_id} at {code_amr_workflow_url}"
+ f"Sending code to backend knowledge service with artifact id: {artifact_id} at {code_amr_workflow_url}"
  )
  amr_response = requests.post(
  code_amr_workflow_url, json=json.loads(json.dumps(request_payload))
  )
  logger.info(
- f"Response received from TA1 with status code: {amr_response.status_code}"
+ f"Response received from backend knowledge service with status code: {amr_response.status_code}"
  )
 
  amr_json = amr_response
@@ -474,7 +474,7 @@ def code_to_amr(*args, **kwargs):
  amr_json = amr_response.json()
  logger.debug(f"TA 1 response object: {amr_json}")
  except:
- logger.error(f"Failed to parse response from TA1 Service:\n{amr_response.text}")
+ logger.error(f"Failed to parse response from backend knowledge service:\n{amr_response.text}")
 
  if amr_response.status_code == 200 and amr_json:
  tds_responses = put_amr_to_tds(amr_json, name, description)