From 24799668b05a83065a0a3a7da662ecd9b9db682e Mon Sep 17 00:00:00 2001 From: Michael Jin Date: Mon, 29 Jan 2024 16:38:33 -0800 Subject: [PATCH 1/6] Add sample pytest tests for 500 error codes as well as a skip test for (very) simple load testing. --- poetry.lock | 73 ++++++++++++++++++++++++++++++++++++++++-- pyproject.toml | 2 ++ tests/test_requests.py | 51 +++++++++++++++++++++++++++++ 3 files changed, 123 insertions(+), 3 deletions(-) create mode 100644 tests/test_requests.py diff --git a/poetry.lock b/poetry.lock index 039f5191..0ab087a8 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. [[package]] name = "aiohttp" @@ -1081,6 +1081,17 @@ files = [ {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, ] +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + [[package]] name = "jsonpatch" version = "1.33" @@ -1486,6 +1497,21 @@ files = [ [package.dependencies] ptyprocess = ">=0.5" +[[package]] +name = "pluggy" +version = "1.4.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.4.0-py3-none-any.whl", hash = "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981"}, + {file = "pluggy-1.4.0.tar.gz", hash = "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + [[package]] name = "prompt-toolkit" version = "3.0.36" @@ -1721,6 +1747,28 @@ files = [ [package.dependencies] tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} +[[package]] +name = "pytest" +version = "8.0.0" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-8.0.0-py3-none-any.whl", hash = "sha256:50fb9cbe836c3f20f0dfa99c565201fb75dc54c8d76373cd1bde06b06657bdb6"}, + {file = "pytest-8.0.0.tar.gz", hash = "sha256:249b1b0864530ba251b7438274c4d251c58d868edaaec8762893ad4a0d71c36c"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "sys_platform == \"win32\""} +exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=1.3.0,<2.0" +tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} + +[package.extras] +testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] + [[package]] name = "python-dateutil" version = "2.8.2" @@ -1946,6 +1994,25 @@ urllib3 = ">=1.21.1,<3" socks = ["PySocks (>=1.5.6,!=1.5.7)"] use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] +[[package]] +name = "requests-mock" +version = "1.11.0" +description = "Mock out responses from the requests package" +optional = false +python-versions = "*" +files = [ + {file = "requests-mock-1.11.0.tar.gz", hash = "sha256:ef10b572b489a5f28e09b708697208c4a3b2b89ef80a9f01584340ea357ec3c4"}, + {file = "requests_mock-1.11.0-py2.py3-none-any.whl", hash = "sha256:f7fae383f228633f6bececebdab236c478ace2284d6292c6e7e2867b9ab74d15"}, +] + +[package.dependencies] +requests = ">=2.3,<3" +six = "*" + +[package.extras] +fixture = ["fixtures"] +test = ["fixtures", "mock", "purl", "pytest", "requests-futures", "sphinx", "testtools"] + [[package]] name = "rich" version = "13.7.0" @@ -2147,7 +2214,7 @@ files = [ ] [package.dependencies] -greenlet = {version = "!=0.4.17", markers = "platform_machine == \"win32\" or platform_machine == \"WIN32\" or platform_machine == \"AMD64\" or platform_machine == \"amd64\" or platform_machine == \"x86_64\" or platform_machine == \"ppc64le\" or platform_machine == \"aarch64\""} +greenlet = {version = "!=0.4.17", markers = "platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\""} typing-extensions = ">=4.6.0" [package.extras] @@ -2743,4 +2810,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = ">=3.9,<4.0" -content-hash = "0f2bb90a7799f0aa7746cdc4da7ea934dddc2cca2eae6f40cc3ffe8fafaa9f0a" +content-hash = "d520ccbdcd4787fb6916d96bbdc161c0cb3d29f22121ace7278d2546cf2270a0" diff --git a/pyproject.toml b/pyproject.toml index dd732969..1f8fd77d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,6 +21,8 @@ packages = [ [tool.poetry.group.dev.dependencies] build = "^0.10.0" +pytest = "^8.0.0" +requests-mock = "^1.11.0" [project.urls] "Homepage" = "https://github.com/log10-io/log10" diff --git a/tests/test_requests.py b/tests/test_requests.py new file mode 100644 index 00000000..536c55e8 --- /dev/null +++ b/tests/test_requests.py @@ -0,0 +1,51 @@ +import asyncio + +import pytest +import requests_mock + +from log10.load import log_sync, log_async +from log10.llm import LLM, Log10Config + + +def test_log_sync_500(): + payload = {'abc': '123'} + url = 'https://log10.io/api/completions' + + with pytest.raises(Exception) as e: + with requests_mock.Mocker() as m: + m.post(url, status_code=500) + log_sync(url, payload) + + +@pytest.mark.asyncio +async def test_log_async_500(): + payload = {'abc': '123'} + url = 'https://log10.io/api/completions' + + with pytest.raises(Exception) as e: + with requests_mock.Mocker() as m: + m.post(url, status_code=500) + await log_async(url, payload) + + +@pytest.mark.skip(reason="This is a very simple load test and doesn't need to be run as part of the test suite.") +@pytest.mark.asyncio +async def test_log_async_multiple_calls(): + simultaneous_calls = 100 + url = 'https://log10.io/api/completions' + + mock_resp = { + "role": "user", + "content": "Say this is a test", + } + + log10_config = Log10Config() + loop = asyncio.get_event_loop() + + def fake_logging(): + llm = LLM(log10_config=log10_config) + completion_id = llm.log_start(url, kind="chat") + print(completion_id) + llm.log_end(completion_id=completion_id, response=mock_resp, duration=5) + + await asyncio.gather(*[loop.run_in_executor(None, fake_logging) for _ in range(simultaneous_calls)]) \ No newline at end of file From 0581f0e1a0bdb81aa4381bec73ae209e11401a2c Mon Sep 17 00:00:00 2001 From: Michael Jin Date: Mon, 29 Jan 2024 16:41:46 -0800 Subject: [PATCH 2/6] Newline at EOF. --- tests/test_requests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_requests.py b/tests/test_requests.py index 536c55e8..b0b898f9 100644 --- a/tests/test_requests.py +++ b/tests/test_requests.py @@ -48,4 +48,4 @@ def fake_logging(): print(completion_id) llm.log_end(completion_id=completion_id, response=mock_resp, duration=5) - await asyncio.gather(*[loop.run_in_executor(None, fake_logging) for _ in range(simultaneous_calls)]) \ No newline at end of file + await asyncio.gather(*[loop.run_in_executor(None, fake_logging) for _ in range(simultaneous_calls)]) From 03033a2206833a5fc1e3c084abdc7737971888fe Mon Sep 17 00:00:00 2001 From: Michael Jin Date: Tue, 30 Jan 2024 13:47:10 -0800 Subject: [PATCH 3/6] Add a skip test with oai chat completions endpoint mocked out. --- poetry.lock | 16 +++++++++++++++- pyproject.toml | 1 + tests/test_requests.py | 27 ++++++++++++++++++++++++++- 3 files changed, 42 insertions(+), 2 deletions(-) diff --git a/poetry.lock b/poetry.lock index 0ab087a8..dbc8fd05 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2013,6 +2013,20 @@ six = "*" fixture = ["fixtures"] test = ["fixtures", "mock", "purl", "pytest", "requests-futures", "sphinx", "testtools"] +[[package]] +name = "respx" +version = "0.20.2" +description = "A utility for mocking out the Python HTTPX and HTTP Core libraries." +optional = false +python-versions = ">=3.7" +files = [ + {file = "respx-0.20.2-py2.py3-none-any.whl", hash = "sha256:ab8e1cf6da28a5b2dd883ea617f8130f77f676736e6e9e4a25817ad116a172c9"}, + {file = "respx-0.20.2.tar.gz", hash = "sha256:07cf4108b1c88b82010f67d3c831dae33a375c7b436e54d87737c7f9f99be643"}, +] + +[package.dependencies] +httpx = ">=0.21.0" + [[package]] name = "rich" version = "13.7.0" @@ -2810,4 +2824,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = ">=3.9,<4.0" -content-hash = "d520ccbdcd4787fb6916d96bbdc161c0cb3d29f22121ace7278d2546cf2270a0" +content-hash = "5ccfe5bb68ef4370ad374eb4887c1bbc2547e2033214306741a4e97662ee22d4" diff --git a/pyproject.toml b/pyproject.toml index 1f8fd77d..9983cdd9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,6 +45,7 @@ backoff = "^2.2.1" anthropic = "^0.3.11" mosaicml-cli = "^0.5.30" together = "^0.2.7" +respx = "^0.20.2" [tool.ruff] # Never enforce `E501` (line length violations). diff --git a/tests/test_requests.py b/tests/test_requests.py index b0b898f9..a8c76277 100644 --- a/tests/test_requests.py +++ b/tests/test_requests.py @@ -1,9 +1,11 @@ import asyncio +import os +import httpx import pytest import requests_mock -from log10.load import log_sync, log_async +from log10.load import log_sync, log_async, OpenAI from log10.llm import LLM, Log10Config @@ -49,3 +51,26 @@ def fake_logging(): llm.log_end(completion_id=completion_id, response=mock_resp, duration=5) await asyncio.gather(*[loop.run_in_executor(None, fake_logging) for _ in range(simultaneous_calls)]) + + +@pytest.mark.skip(reason="This is a very simple load test and doesn't need to be run as part of the test suite.") +@pytest.mark.asyncio +async def test_log_async_httpx_multiple_calls(respx_mock): + simultaneous_calls = 100 + + mock_resp = { + "role": "user", + "content": "Say this is a test", + } + + + client = OpenAI() + + respx_mock.post("https://api.openai.com/v1/chat/completions").mock(return_value=httpx.Response(200, json=mock_resp)) + + def better_logging(): + completion = client.chat.completions.create(model="gpt-3.5-turbo", messages=[ + {"role": "user", "content": "Say pong"}]) + + loop = asyncio.get_event_loop() + await asyncio.gather(*[loop.run_in_executor(None, better_logging) for _ in range(simultaneous_calls)]) From f7bf77e257a49df054a6b1e32641c1d6ed906599 Mon Sep 17 00:00:00 2001 From: Michael Jin Date: Tue, 30 Jan 2024 14:46:59 -0800 Subject: [PATCH 4/6] Add random tags to async httpx test. --- tests/test_requests.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/test_requests.py b/tests/test_requests.py index a8c76277..f8d1a210 100644 --- a/tests/test_requests.py +++ b/tests/test_requests.py @@ -1,11 +1,11 @@ import asyncio -import os +import uuid import httpx import pytest import requests_mock -from log10.load import log_sync, log_async, OpenAI +from log10.load import log_sync, log_async, OpenAI, log10_session from log10.llm import LLM, Log10Config @@ -55,7 +55,7 @@ def fake_logging(): @pytest.mark.skip(reason="This is a very simple load test and doesn't need to be run as part of the test suite.") @pytest.mark.asyncio -async def test_log_async_httpx_multiple_calls(respx_mock): +async def test_log_async_httpx_multiple_calls_with_tags(respx_mock): simultaneous_calls = 100 mock_resp = { @@ -63,14 +63,15 @@ async def test_log_async_httpx_multiple_calls(respx_mock): "content": "Say this is a test", } - client = OpenAI() respx_mock.post("https://api.openai.com/v1/chat/completions").mock(return_value=httpx.Response(200, json=mock_resp)) def better_logging(): - completion = client.chat.completions.create(model="gpt-3.5-turbo", messages=[ - {"role": "user", "content": "Say pong"}]) + uuids = [str(uuid.uuid4()) for _ in range(5)] + with log10_session(tags=uuids) as s: + completion = client.chat.completions.create(model="gpt-3.5-turbo", messages=[ + {"role": "user", "content": "Say pong"}]) loop = asyncio.get_event_loop() await asyncio.gather(*[loop.run_in_executor(None, better_logging) for _ in range(simultaneous_calls)]) From d3a70bf47a476ba915b77307331422e622240ba3 Mon Sep 17 00:00:00 2001 From: Michael Jin Date: Tue, 30 Jan 2024 15:58:25 -0800 Subject: [PATCH 5/6] Reverse check in tests now that we swallow the exception. --- tests/test_requests.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/tests/test_requests.py b/tests/test_requests.py index f8d1a210..2c3f5c3b 100644 --- a/tests/test_requests.py +++ b/tests/test_requests.py @@ -13,10 +13,9 @@ def test_log_sync_500(): payload = {'abc': '123'} url = 'https://log10.io/api/completions' - with pytest.raises(Exception) as e: - with requests_mock.Mocker() as m: - m.post(url, status_code=500) - log_sync(url, payload) + with requests_mock.Mocker() as m: + m.post(url, status_code=500) + log_sync(url, payload) @pytest.mark.asyncio @@ -24,10 +23,9 @@ async def test_log_async_500(): payload = {'abc': '123'} url = 'https://log10.io/api/completions' - with pytest.raises(Exception) as e: - with requests_mock.Mocker() as m: - m.post(url, status_code=500) - await log_async(url, payload) + with requests_mock.Mocker() as m: + m.post(url, status_code=500) + await log_async(url, payload) @pytest.mark.skip(reason="This is a very simple load test and doesn't need to be run as part of the test suite.") From 4766e165d0a045a1a6755f4f65449f1397c3d333 Mon Sep 17 00:00:00 2001 From: Michael Jin Date: Tue, 30 Jan 2024 16:07:19 -0800 Subject: [PATCH 6/6] Move respx to dev dependency. --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index d07d752c..88377ff9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,6 +23,7 @@ packages = [ build = "^0.10.0" pytest = "^8.0.0" requests-mock = "^1.11.0" +respx = "^0.20.2" [project.urls] "Homepage" = "https://github.com/log10-io/log10" @@ -44,7 +45,6 @@ backoff = "^2.2.1" anthropic = "^0.3.11" mosaicml-cli = "^0.5.30" together = "^0.2.7" -respx = "^0.20.2" [tool.ruff] # Never enforce `E501` (line length violations).