diff --git a/poetry.lock b/poetry.lock index 039f5191..dbc8fd05 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. [[package]] name = "aiohttp" @@ -1081,6 +1081,17 @@ files = [ {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, ] +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + [[package]] name = "jsonpatch" version = "1.33" @@ -1486,6 +1497,21 @@ files = [ [package.dependencies] ptyprocess = ">=0.5" +[[package]] +name = "pluggy" +version = "1.4.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.4.0-py3-none-any.whl", hash = "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981"}, + {file = "pluggy-1.4.0.tar.gz", hash = "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + [[package]] name = "prompt-toolkit" version = "3.0.36" @@ -1721,6 +1747,28 @@ files = [ [package.dependencies] tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} +[[package]] +name = "pytest" +version = "8.0.0" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-8.0.0-py3-none-any.whl", hash = "sha256:50fb9cbe836c3f20f0dfa99c565201fb75dc54c8d76373cd1bde06b06657bdb6"}, + {file = "pytest-8.0.0.tar.gz", hash = "sha256:249b1b0864530ba251b7438274c4d251c58d868edaaec8762893ad4a0d71c36c"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "sys_platform == \"win32\""} +exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=1.3.0,<2.0" +tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} + +[package.extras] +testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] + [[package]] name = "python-dateutil" version = "2.8.2" @@ -1946,6 +1994,39 @@ urllib3 = ">=1.21.1,<3" socks = ["PySocks (>=1.5.6,!=1.5.7)"] use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] +[[package]] +name = "requests-mock" +version = "1.11.0" +description = "Mock out responses from the requests package" +optional = false +python-versions = "*" +files = [ + {file = "requests-mock-1.11.0.tar.gz", hash = "sha256:ef10b572b489a5f28e09b708697208c4a3b2b89ef80a9f01584340ea357ec3c4"}, + {file = "requests_mock-1.11.0-py2.py3-none-any.whl", hash = "sha256:f7fae383f228633f6bececebdab236c478ace2284d6292c6e7e2867b9ab74d15"}, +] + +[package.dependencies] +requests = ">=2.3,<3" +six = "*" + +[package.extras] +fixture = ["fixtures"] +test = ["fixtures", "mock", "purl", "pytest", "requests-futures", "sphinx", "testtools"] + +[[package]] +name = "respx" +version = "0.20.2" +description = "A utility for mocking out the Python HTTPX and HTTP Core libraries." +optional = false +python-versions = ">=3.7" +files = [ + {file = "respx-0.20.2-py2.py3-none-any.whl", hash = "sha256:ab8e1cf6da28a5b2dd883ea617f8130f77f676736e6e9e4a25817ad116a172c9"}, + {file = "respx-0.20.2.tar.gz", hash = "sha256:07cf4108b1c88b82010f67d3c831dae33a375c7b436e54d87737c7f9f99be643"}, +] + +[package.dependencies] +httpx = ">=0.21.0" + [[package]] name = "rich" version = "13.7.0" @@ -2147,7 +2228,7 @@ files = [ ] [package.dependencies] -greenlet = {version = "!=0.4.17", markers = "platform_machine == \"win32\" or platform_machine == \"WIN32\" or platform_machine == \"AMD64\" or platform_machine == \"amd64\" or platform_machine == \"x86_64\" or platform_machine == \"ppc64le\" or platform_machine == \"aarch64\""} +greenlet = {version = "!=0.4.17", markers = "platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\""} typing-extensions = ">=4.6.0" [package.extras] @@ -2743,4 +2824,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = ">=3.9,<4.0" -content-hash = "0f2bb90a7799f0aa7746cdc4da7ea934dddc2cca2eae6f40cc3ffe8fafaa9f0a" +content-hash = "5ccfe5bb68ef4370ad374eb4887c1bbc2547e2033214306741a4e97662ee22d4" diff --git a/pyproject.toml b/pyproject.toml index 0077e0f6..88377ff9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,6 +21,9 @@ packages = [ [tool.poetry.group.dev.dependencies] build = "^0.10.0" +pytest = "^8.0.0" +requests-mock = "^1.11.0" +respx = "^0.20.2" [project.urls] "Homepage" = "https://github.com/log10-io/log10" diff --git a/tests/test_requests.py b/tests/test_requests.py new file mode 100644 index 00000000..2c3f5c3b --- /dev/null +++ b/tests/test_requests.py @@ -0,0 +1,75 @@ +import asyncio +import uuid + +import httpx +import pytest +import requests_mock + +from log10.load import log_sync, log_async, OpenAI, log10_session +from log10.llm import LLM, Log10Config + + +def test_log_sync_500(): + payload = {'abc': '123'} + url = 'https://log10.io/api/completions' + + with requests_mock.Mocker() as m: + m.post(url, status_code=500) + log_sync(url, payload) + + +@pytest.mark.asyncio +async def test_log_async_500(): + payload = {'abc': '123'} + url = 'https://log10.io/api/completions' + + with requests_mock.Mocker() as m: + m.post(url, status_code=500) + await log_async(url, payload) + + +@pytest.mark.skip(reason="This is a very simple load test and doesn't need to be run as part of the test suite.") +@pytest.mark.asyncio +async def test_log_async_multiple_calls(): + simultaneous_calls = 100 + url = 'https://log10.io/api/completions' + + mock_resp = { + "role": "user", + "content": "Say this is a test", + } + + log10_config = Log10Config() + loop = asyncio.get_event_loop() + + def fake_logging(): + llm = LLM(log10_config=log10_config) + completion_id = llm.log_start(url, kind="chat") + print(completion_id) + llm.log_end(completion_id=completion_id, response=mock_resp, duration=5) + + await asyncio.gather(*[loop.run_in_executor(None, fake_logging) for _ in range(simultaneous_calls)]) + + +@pytest.mark.skip(reason="This is a very simple load test and doesn't need to be run as part of the test suite.") +@pytest.mark.asyncio +async def test_log_async_httpx_multiple_calls_with_tags(respx_mock): + simultaneous_calls = 100 + + mock_resp = { + "role": "user", + "content": "Say this is a test", + } + + client = OpenAI() + + respx_mock.post("https://api.openai.com/v1/chat/completions").mock(return_value=httpx.Response(200, json=mock_resp)) + + def better_logging(): + uuids = [str(uuid.uuid4()) for _ in range(5)] + with log10_session(tags=uuids) as s: + completion = client.chat.completions.create(model="gpt-3.5-turbo", messages=[ + {"role": "user", "content": "Say pong"}]) + + loop = asyncio.get_event_loop() + await asyncio.gather(*[loop.run_in_executor(None, better_logging) for _ in range(simultaneous_calls)])