feat: Add integration tests (#185)

* Add integration tests

* Fix bug found by integration tests

* Move unit tests in `unit_tests` folder

* allow absence of config file for unit tests

* minor refactor

* remove pytest asyncio mark

* - fixes due to pr comments
-add retries on " event loop is closed" error

* minor refactor in project dependencies

* refactor: remove pytest_asyncio

* minor refactor

* Minor fix in comment

* Fix in naming

* fix comment

* another fix in test name and comment

* update vision tests

* Bunch of review fixes

* comment typo

* forgotten commit

* reduce max tokens count, minor refactor

* another bunch of review fixes

* Another bunch of review fixes

* More refactoring of text tests

* minor fix

* one more refactor of test cases

.gitignore

@@ -9,4 +9,5 @@ __pycache__
 dist
 .vscode/launch.json
 .pytest_cache
-~*
+~*
+integration_test_config.json

Makefile

@@ -29,6 +29,9 @@ format: install
 test: install
 	poetry run nox -s test -- $(ARGS)
 
+integration_test: install
+	poetry run nox -s integration_test -- $(ARGS)
+
 docker_serve:
 	docker build --platform $(PLATFORM) -t $(IMAGE_NAME):dev .
 	docker run --platform $(PLATFORM) --env-file ./.env --rm -p $(PORT):5000 $(IMAGE_NAME):dev

aidial_adapter_openai/app_config.py

@@ -1,6 +1,6 @@
 import json
 import os
-from typing import Dict, List
+from typing import Callable, Dict, List
 
 from pydantic import BaseModel
 
@@ -25,23 +25,32 @@ class ApplicationConfig(BaseModel):
     NON_STREAMING_DEPLOYMENTS: List[str] = []
     ELIMINATE_EMPTY_CHOICES: bool = False
 
+    DEPLOYMENT_TYPE_MAP: Dict[
+        ChatCompletionDeploymentType, Callable[["ApplicationConfig"], List[str]]
+    ] = {
+        ChatCompletionDeploymentType.DALLE3: lambda config: config.DALLE3_DEPLOYMENTS,
+        ChatCompletionDeploymentType.GPT4_VISION: lambda config: config.GPT4_VISION_DEPLOYMENTS,
+        ChatCompletionDeploymentType.MISTRAL: lambda config: config.MISTRAL_DEPLOYMENTS,
+        ChatCompletionDeploymentType.DATABRICKS: lambda config: config.DATABRICKS_DEPLOYMENTS,
+        ChatCompletionDeploymentType.GPT4O: lambda config: config.GPT4O_DEPLOYMENTS,
+        ChatCompletionDeploymentType.GPT4O_MINI: lambda config: config.GPT4O_MINI_DEPLOYMENTS,
+    }
+
     def get_chat_completion_deployment_type(
         self, deployment_id: str
     ) -> ChatCompletionDeploymentType:
-        if deployment_id in self.DALLE3_DEPLOYMENTS:
-            return ChatCompletionDeploymentType.DALLE3
-        elif deployment_id in self.GPT4_VISION_DEPLOYMENTS:
-            return ChatCompletionDeploymentType.GPT4_VISION
-        elif deployment_id in self.MISTRAL_DEPLOYMENTS:
-            return ChatCompletionDeploymentType.MISTRAL
-        elif deployment_id in self.DATABRICKS_DEPLOYMENTS:
-            return ChatCompletionDeploymentType.DATABRICKS
-        elif deployment_id in self.GPT4O_DEPLOYMENTS:
-            return ChatCompletionDeploymentType.GPT4O
-        elif deployment_id in self.GPT4O_MINI_DEPLOYMENTS:
-            return ChatCompletionDeploymentType.GPT4O_MINI
-        else:
-            return ChatCompletionDeploymentType.GPT_TEXT_ONLY
+        for deployment_type, config_getter in self.DEPLOYMENT_TYPE_MAP.items():
+            if deployment_id in config_getter(self):
+                return deployment_type
+        return ChatCompletionDeploymentType.GPT_TEXT_ONLY
+
+    def add_deployment(
+        self, deployment_id: str, deployment_type: ChatCompletionDeploymentType
+    ):
+        if deployment_type == ChatCompletionDeploymentType.GPT_TEXT_ONLY:
+            return
+        config_getter = self.DEPLOYMENT_TYPE_MAP[deployment_type]
+        config_getter(self).append(deployment_id)
 
     @classmethod
     def from_env(cls) -> "ApplicationConfig":

aidial_adapter_openai/constant.py

@@ -1,11 +1,11 @@
-from enum import StrEnum, auto
+from enum import StrEnum
 
 
 class ChatCompletionDeploymentType(StrEnum):
-    DALLE3 = auto()
-    MISTRAL = auto()
-    DATABRICKS = auto()
-    GPT4_VISION = auto()
-    GPT4O = auto()
-    GPT4O_MINI = auto()
-    GPT_TEXT_ONLY = auto()
+    DALLE3 = "DALLE3"
+    MISTRAL = "MISTRAL"
+    DATABRICKS = "DATABRICKS"
+    GPT4_VISION = "GPT4_VISION"
+    GPT4O = "GPT4O"
+    GPT4O_MINI = "GPT4O_MINI"
+    GPT_TEXT_ONLY = "GPT_TEXT_ONLY"

aidial_adapter_openai/utils/pydantic.py

@@ -0,0 +1,6 @@
+from pydantic import BaseModel
+
+
+class ExtraAllowedModel(BaseModel):
+    class Config:
+        extra = "allow"

aidial_adapter_openai/utils/streaming.py

@@ -244,7 +244,7 @@ def block_to_response(block: dict) -> Response:
         if emulate_stream:
             return stream_to_response(block_to_stream(block))
         else:
-            return JSONResponse(response)
+            return JSONResponse(block)
 
     if isinstance(response, AsyncIterator):
         return stream_to_response(response)

noxfile.py

@@ -37,4 +37,11 @@ def format(session: nox.Session):
 def test(session: nox.Session):
     """Runs unit tests"""
     session.run("poetry", "install", external=True)
-    session.run("pytest", "tests/")
+    session.run("pytest", "tests/unit_tests")
+
+
+@nox.session
+def integration_test(session: nox.Session):
+    """Runs integration tests. You need a"""
+    session.run("poetry", "install", external=True)
+    session.run("pytest", "tests/integration_tests")

pyproject.toml

@@ -8,9 +8,7 @@ documentation = "https://epam-rail.com/dial_api"
 license = "Apache-2.0"
 readme = "README.md"
 keywords = ["ai"]
-classifiers = [
-    "Topic :: Software Development :: Libraries :: Python Modules"
-]
+classifiers = ["Topic :: Software Development :: Libraries :: Python Modules"]
 repository = "https://github.com/epam/ai-dial-adapter-openai"
 
 [tool.poetry.scripts]
@@ -31,11 +29,12 @@ aiohttp = "^3.10.11"
 numpy = "^1.26.0"
 pillow = "^10.3.0"
 azure-identity = "^1.16.1"
-aidial-sdk = {version = "^0.16.0", extras = ["telemetry"]}
+aidial-sdk = { version = "^0.16.0", extras = ["telemetry"] }
 
 [tool.poetry.group.test.dependencies]
 pytest = "7.4.0"
 pytest-asyncio = "0.21.1"
+pytest-xdist = "^3.5.0"
 respx = "^0.21.1"
 
 [tool.poetry.group.lint.dependencies]
@@ -51,22 +50,18 @@ nox = "^2023.4.22"
 python-dotenv = "^1.0.1"
 
 [tool.pytest.ini_options]
+addopts = "-n=auto --asyncio-mode=auto"
 # muting warnings coming from opentelemetry and pkg_resources packages
 filterwarnings = [
-    "ignore::DeprecationWarning:opentelemetry.instrumentation.dependencies",
-    "ignore::DeprecationWarning:pkg_resources"
+  "ignore::DeprecationWarning:opentelemetry.instrumentation.dependencies",
+  "ignore::DeprecationWarning:pkg_resources",
 ]
 
 [tool.pyright]
 typeCheckingMode = "basic"
 reportUnusedVariable = "error"
 reportIncompatibleMethodOverride = "error"
-exclude = [
-    ".git",
-    ".venv",
-    ".nox",
-    "**/__pycache__"
-]
+exclude = [".git", ".venv", ".nox", "**/__pycache__"]
 
 [tool.black]
 line-length = 80
@@ -89,7 +84,4 @@ remove_all_unused_imports = true
 in_place = true
 recursive = true
 quiet = true
-exclude = [
-    "\\.venv",
-    "\\.nox",
-]
+exclude = ["\\.venv", "\\.nox"]

tests/conftest.py

@@ -1,22 +1,29 @@
 import httpx
 import pytest
-import pytest_asyncio
 from httpx import ASGITransport
+from openai import AsyncAzureOpenAI
 
 from aidial_adapter_openai.app import create_app
+from aidial_adapter_openai.utils.http_client import DEFAULT_TIMEOUT
 from aidial_adapter_openai.utils.request import get_app_config
+from tests.integration_tests.base import DeploymentConfig
+from tests.integration_tests.constants import TEST_DEPLOYMENTS_CONFIG
 
 
-@pytest.fixture
+@pytest.fixture(scope="session")
 def _app_instance():
-    return create_app(init_telemetry=False)
+    return create_app(
+        init_telemetry=False,
+        app_config=TEST_DEPLOYMENTS_CONFIG.app_config,
+    )
 
 
-@pytest_asyncio.fixture
+@pytest.fixture
 async def test_app(_app_instance):
     async with httpx.AsyncClient(
         transport=ASGITransport(app=_app_instance),
         base_url="http://test-app.com",
+        timeout=DEFAULT_TIMEOUT,
     ) as client:
         yield client
 
@@ -27,3 +34,20 @@ def eliminate_empty_choices(_app_instance):
     app_config.ELIMINATE_EMPTY_CHOICES = True
     yield
     app_config.ELIMINATE_EMPTY_CHOICES = False
+
+
+@pytest.fixture
+def get_openai_client(test_app: httpx.AsyncClient):
+    def _get_client(deployment_config: DeploymentConfig) -> AsyncAzureOpenAI:
+        return AsyncAzureOpenAI(
+            azure_endpoint=str(test_app.base_url),
+            azure_deployment=deployment_config.deployment_id,
+            api_version="2024-02-01",
+            api_key="dummy_key",
+            max_retries=5,
+            timeout=30,
+            http_client=test_app,
+            default_headers=deployment_config.upstream_headers,
+        )
+
+    yield _get_client