From 64e45612b7995437045e756b1e36c3f835d1d66a Mon Sep 17 00:00:00 2001 From: Parker Hancock <633163+parkerhancock@users.noreply.github.com> Date: Thu, 7 Dec 2023 21:51:48 -0600 Subject: [PATCH 1/5] fixes for httpx --- tests/integration/test_httpx.py | 157 ++++++++++++++------------------ vcr/patch.py | 12 +-- vcr/stubs/httpx_stubs.py | 29 +----- 3 files changed, 77 insertions(+), 121 deletions(-) diff --git a/tests/integration/test_httpx.py b/tests/integration/test_httpx.py index 4425e02e..45415a0e 100644 --- a/tests/integration/test_httpx.py +++ b/tests/integration/test_httpx.py @@ -9,6 +9,14 @@ from vcr.stubs.httpx_stubs import HTTPX_REDIRECT_PARAM # noqa: E402 +@pytest.fixture(params=["https", "http"]) +def scheme(request): + """Fixture that returns both http and https.""" + return request.param + +@pytest.fixture +def httpbin(scheme): + return scheme + "://httpbin.org" class BaseDoRequest: _client_class = None @@ -16,6 +24,7 @@ class BaseDoRequest: def __init__(self, *args, **kwargs): self._client_args = args self._client_kwargs = kwargs + self._client_kwargs['follow_redirects'] = self._client_kwargs.get('follow_redirects', True) def _make_client(self): return self._client_class(*self._client_args, **self._client_kwargs) @@ -40,7 +49,10 @@ def client(self): def __call__(self, *args, **kwargs): return self.client.request(*args, timeout=60, **kwargs) - + + def stream(self, *args, **kwargs): + with self.client.stream(*args, **kwargs) as response: + return b"".join(response.iter_bytes()) class DoAsyncRequest(BaseDoRequest): _client_class = httpx.AsyncClient @@ -75,8 +87,22 @@ def __call__(self, *args, **kwargs): # Use one-time context and dispose of the loop/client afterwards with self: - return self(*args, **kwargs) + return self._loop.run_until_complete(self.client.request(*args, **kwargs)) + + async def _get_stream(self, *args, **kwargs): + async with self.client.stream(*args, **kwargs) as response: + content = b"" + async for c in response.aiter_bytes(): + content += c + return content + + def stream(self, *args, **kwargs): + if hasattr(self, "_loop"): + return self._loop.run_until_complete(self._get_stream(*args, **kwargs)) + # Use one-time context and dispose of the loop/client afterwards + with self: + return self._loop.run_until_complete(self._get_stream(*args, **kwargs)) def pytest_generate_tests(metafunc): if "do_request" in metafunc.fixturenames: @@ -89,8 +115,8 @@ def yml(tmpdir, request): @pytest.mark.online -def test_status(tmpdir, mockbin, do_request): - url = mockbin +def test_status(tmpdir, httpbin, do_request): + url = httpbin with vcr.use_cassette(str(tmpdir.join("status.yaml"))): response = do_request()("GET", url) @@ -102,8 +128,8 @@ def test_status(tmpdir, mockbin, do_request): @pytest.mark.online -def test_case_insensitive_headers(tmpdir, mockbin, do_request): - url = mockbin +def test_case_insensitive_headers(tmpdir, httpbin, do_request): + url = httpbin with vcr.use_cassette(str(tmpdir.join("whatever.yaml"))): do_request()("GET", url) @@ -116,8 +142,8 @@ def test_case_insensitive_headers(tmpdir, mockbin, do_request): @pytest.mark.online -def test_content(tmpdir, mockbin, do_request): - url = mockbin +def test_content(tmpdir, httpbin, do_request): + url = httpbin with vcr.use_cassette(str(tmpdir.join("cointent.yaml"))): response = do_request()("GET", url) @@ -129,23 +155,21 @@ def test_content(tmpdir, mockbin, do_request): @pytest.mark.online -def test_json(tmpdir, mockbin, do_request): - url = mockbin + "/request" - - headers = {"content-type": "application/json"} +def test_json(tmpdir, httpbin, do_request): + url = httpbin + "/json" with vcr.use_cassette(str(tmpdir.join("json.yaml"))): - response = do_request(headers=headers)("GET", url) + response = do_request()("GET", url) with vcr.use_cassette(str(tmpdir.join("json.yaml"))) as cassette: - cassette_response = do_request(headers=headers)("GET", url) + cassette_response = do_request()("GET", url) assert cassette_response.json() == response.json() assert cassette.play_count == 1 @pytest.mark.online -def test_params_same_url_distinct_params(tmpdir, mockbin, do_request): - url = mockbin + "/request" +def test_params_same_url_distinct_params(tmpdir, httpbin, do_request): + url = httpbin + "/get" headers = {"Content-Type": "application/json"} params = {"a": 1, "b": False, "c": "c"} @@ -165,22 +189,20 @@ def test_params_same_url_distinct_params(tmpdir, mockbin, do_request): @pytest.mark.online -def test_redirect(mockbin, yml, do_request): - url = mockbin + "/redirect/303/2" - - redirect_kwargs = {HTTPX_REDIRECT_PARAM.name: True} +def test_redirect(httpbin, yml, do_request): + url = httpbin + "/redirect-to" - response = do_request()("GET", url, **redirect_kwargs) + response = do_request()("GET", url) with vcr.use_cassette(yml): - response = do_request()("GET", url, **redirect_kwargs) + response = do_request()("GET", url, params={"url": "./get", "status_code": 302}) with vcr.use_cassette(yml) as cassette: - cassette_response = do_request()("GET", url, **redirect_kwargs) + cassette_response = do_request()("GET", url, params={"url": "./get", "status_code": 302}) assert cassette_response.status_code == response.status_code assert len(cassette_response.history) == len(response.history) - assert len(cassette) == 3 - assert cassette.play_count == 3 + assert len(cassette) == 2 + assert cassette.play_count == 2 # Assert that the real response and the cassette response have a similar # looking request_info. @@ -190,8 +212,8 @@ def test_redirect(mockbin, yml, do_request): @pytest.mark.online -def test_work_with_gzipped_data(mockbin, do_request, yml): - url = mockbin + "/gzip?foo=bar" +def test_work_with_gzipped_data(httpbin, do_request, yml): + url = httpbin + "/gzip?foo=bar" headers = {"accept-encoding": "deflate, gzip"} with vcr.use_cassette(yml): @@ -216,56 +238,32 @@ def test_simple_fetching(do_request, yml, url): assert str(cassette_response.request.url) == url assert cassette.play_count == 1 - -def test_behind_proxy(do_request): - # This is recorded because otherwise we should have a live proxy somewhere. - yml = ( - os.path.dirname(os.path.realpath(__file__)) + "/cassettes/" + "test_httpx_test_test_behind_proxy.yml" - ) - url = "https://mockbin.org/headers" - proxy = "http://localhost:8080" - proxies = {"http://": proxy, "https://": proxy} - - with vcr.use_cassette(yml): - response = do_request(proxies=proxies, verify=False)("GET", url) - - with vcr.use_cassette(yml) as cassette: - cassette_response = do_request(proxies=proxies, verify=False)("GET", url) - assert str(cassette_response.request.url) == url - assert cassette.play_count == 1 - - assert cassette_response.headers["Via"] == "my_own_proxy", str(cassette_response.headers) - assert cassette_response.request.url == response.request.url - - @pytest.mark.online -def test_cookies(tmpdir, mockbin, do_request): +def test_cookies(tmpdir, httpbin, do_request): def client_cookies(client): return list(client.client.cookies) def response_cookies(response): return list(response.cookies) - url = mockbin + "/bin/26148652-fe25-4f21-aaf5-689b5b4bf65f" - headers = {"cookie": "k1=v1;k2=v2"} + url = httpbin + "/cookies/set" + params = {"k1": "v1", "k2": "v2"} - with do_request(headers=headers) as client: + with do_request(params=params, follow_redirects=False) as client: assert client_cookies(client) == [] - redirect_kwargs = {HTTPX_REDIRECT_PARAM.name: True} - testfile = str(tmpdir.join("cookies.yml")) with vcr.use_cassette(testfile): - r1 = client("GET", url, **redirect_kwargs) + r1 = client("GET", url) assert response_cookies(r1) == ["k1", "k2"] - r2 = client("GET", url, **redirect_kwargs) + r2 = client("GET", url) assert response_cookies(r2) == ["k1", "k2"] assert client_cookies(client) == ["k1", "k2"] - with do_request(headers=headers) as new_client: + with do_request(params=params, follow_redirects=False) as new_client: assert client_cookies(new_client) == [] with vcr.use_cassette(testfile) as cassette: @@ -275,42 +273,19 @@ def response_cookies(response): assert response_cookies(cassette_response) == ["k1", "k2"] assert client_cookies(new_client) == ["k1", "k2"] - @pytest.mark.online -def test_relative_redirects(tmpdir, scheme, do_request, mockbin): - redirect_kwargs = {HTTPX_REDIRECT_PARAM.name: True} +def test_stream(tmpdir, httpbin, do_request): + url = httpbin + "/stream-bytes/512" + testfile = str(tmpdir.join("stream.yml")) - url = mockbin + "/redirect/301?to=/redirect/301?to=/request" - testfile = str(tmpdir.join("relative_redirects.yml")) with vcr.use_cassette(testfile): - response = do_request()("GET", url, **redirect_kwargs) - assert len(response.history) == 2, response - assert response.json()["url"].endswith("request") + response_content = do_request().stream("GET", url) + assert len(response_content) == 512 + with vcr.use_cassette(testfile) as cassette: - response = do_request()("GET", url, **redirect_kwargs) - assert len(response.history) == 2 - assert response.json()["url"].endswith("request") - - assert cassette.play_count == 3 - - -@pytest.mark.online -def test_redirect_wo_allow_redirects(do_request, mockbin, yml): - url = mockbin + "/redirect/308/5" - - redirect_kwargs = {HTTPX_REDIRECT_PARAM.name: False} - - with vcr.use_cassette(yml): - response = do_request()("GET", url, **redirect_kwargs) - - assert str(response.url).endswith("308/5") - assert response.status_code == 308 - - with vcr.use_cassette(yml) as cassette: - response = do_request()("GET", url, **redirect_kwargs) - - assert str(response.url).endswith("308/5") - assert response.status_code == 308 - + cassette_content = do_request().stream("GET", url) + assert cassette_content == response_content + assert len(cassette_content) == 512 assert cassette.play_count == 1 + \ No newline at end of file diff --git a/vcr/patch.py b/vcr/patch.py index afcaab57..f69ae768 100644 --- a/vcr/patch.py +++ b/vcr/patch.py @@ -95,8 +95,8 @@ except ImportError: # pragma: no cover pass else: - _HttpxSyncClient_send = httpx.Client.send - _HttpxAsyncClient_send = httpx.AsyncClient.send + _HttpxSyncClient_send_single_request = httpx.Client._send_single_request + _HttpxAsyncClient_send_single_request = httpx.AsyncClient._send_single_request class CassettePatcherBuilder: @@ -307,11 +307,11 @@ def _httpx(self): else: from .stubs.httpx_stubs import async_vcr_send, sync_vcr_send - new_async_client_send = async_vcr_send(self._cassette, _HttpxAsyncClient_send) - yield httpx.AsyncClient, "send", new_async_client_send + new_async_client_send = async_vcr_send(self._cassette, _HttpxAsyncClient_send_single_request) + yield httpx.AsyncClient, "_send_single_request", new_async_client_send - new_sync_client_send = sync_vcr_send(self._cassette, _HttpxSyncClient_send) - yield httpx.Client, "send", new_sync_client_send + new_sync_client_send = sync_vcr_send(self._cassette, _HttpxSyncClient_send_single_request) + yield httpx.Client, "_send_single_request", new_sync_client_send def _urllib3_patchers(self, cpool, conn, stubs): http_connection_remover = ConnectionRemover( diff --git a/vcr/stubs/httpx_stubs.py b/vcr/stubs/httpx_stubs.py index 515855ee..453d3290 100644 --- a/vcr/stubs/httpx_stubs.py +++ b/vcr/stubs/httpx_stubs.py @@ -38,7 +38,7 @@ def _to_serialized_response(httpx_response): "status_code": httpx_response.status_code, "http_version": httpx_response.http_version, "headers": _transform_headers(httpx_response), - "content": httpx_response.content.decode("utf-8", "ignore"), + "content": httpx_response.content#.decode("utf-8", "ignore"), } @@ -57,7 +57,7 @@ def _from_serialized_headers(headers): @patch("httpx.Response.close", MagicMock()) @patch("httpx.Response.read", MagicMock()) def _from_serialized_response(request, serialized_response, history=None): - content = serialized_response.get("content").encode() + content = serialized_response.get("content") response = httpx.Response( status_code=serialized_response.get("status_code"), request=request, @@ -106,33 +106,12 @@ def _record_responses(cassette, vcr_request, real_response): def _play_responses(cassette, request, vcr_request, client, kwargs): - history = [] - - allow_redirects = kwargs.get( - HTTPX_REDIRECT_PARAM.name, - HTTPX_REDIRECT_PARAM.default, - ) vcr_response = cassette.play_response(vcr_request) response = _from_serialized_response(request, vcr_response) - - while allow_redirects and 300 <= response.status_code <= 399: - next_url = response.headers.get("location") - if not next_url: - break - - vcr_request = VcrRequest("GET", next_url, None, dict(response.headers)) - vcr_request = cassette.find_requests_with_most_matches(vcr_request)[0][0] - - history.append(response) - # add cookies from response to session cookie store - client.cookies.extract_cookies(response) - - vcr_response = cassette.play_response(vcr_request) - response = _from_serialized_response(vcr_request, vcr_response, history) - return response + async def _async_vcr_send(cassette, real_send, *args, **kwargs): vcr_request, response = _shared_vcr_send(cassette, real_send, *args, **kwargs) if response: @@ -141,6 +120,7 @@ async def _async_vcr_send(cassette, real_send, *args, **kwargs): return response real_response = await real_send(*args, **kwargs) + await real_response.aread() return _record_responses(cassette, vcr_request, real_response) @@ -160,6 +140,7 @@ def _sync_vcr_send(cassette, real_send, *args, **kwargs): return response real_response = real_send(*args, **kwargs) + real_response.read() return _record_responses(cassette, vcr_request, real_response) From 480044154fa8ffec903677e306d7c1bd86672cdc Mon Sep 17 00:00:00 2001 From: Parker Hancock <633163+parkerhancock@users.noreply.github.com> Date: Fri, 8 Dec 2023 09:30:42 -0600 Subject: [PATCH 2/5] remove unnecssary comment --- vcr/stubs/httpx_stubs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vcr/stubs/httpx_stubs.py b/vcr/stubs/httpx_stubs.py index 453d3290..0bb3b4a4 100644 --- a/vcr/stubs/httpx_stubs.py +++ b/vcr/stubs/httpx_stubs.py @@ -38,7 +38,7 @@ def _to_serialized_response(httpx_response): "status_code": httpx_response.status_code, "http_version": httpx_response.http_version, "headers": _transform_headers(httpx_response), - "content": httpx_response.content#.decode("utf-8", "ignore"), + "content": httpx_response.content, } From 50c9001559bce538bd7aa679f3711bccd4ad5a7b Mon Sep 17 00:00:00 2001 From: Parker Hancock <633163+parkerhancock@users.noreply.github.com> Date: Fri, 8 Dec 2023 10:49:29 -0600 Subject: [PATCH 3/5] fix tests --- tests/integration/conftest.py | 16 ----- tests/integration/test_aiohttp.py | 66 ++++++++++---------- tests/integration/test_basic.py | 2 +- tests/integration/test_config.py | 24 +++---- tests/integration/test_disksaver.py | 14 ++--- tests/integration/test_httplib2.py | 6 +- tests/integration/test_httpx.py | 36 +++++------ tests/integration/test_proxy.py | 13 ++-- tests/integration/test_register_matcher.py | 16 ++--- tests/integration/test_register_persister.py | 2 +- tests/integration/test_tornado.py | 6 +- tests/integration/test_urllib2.py | 6 +- tests/integration/test_urllib3.py | 4 +- tox.ini | 2 +- 14 files changed, 101 insertions(+), 112 deletions(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index a01f3c3f..05908e51 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -4,22 +4,6 @@ import pytest -@pytest.fixture(params=["https", "http"]) -def scheme(request): - """Fixture that returns both http and https.""" - return request.param - - -@pytest.fixture -def mockbin(scheme): - return scheme + "://mockbin.org" - - -@pytest.fixture -def mockbin_request_url(mockbin): - return mockbin + "/request" - - @pytest.fixture def httpbin_ssl_context(): ssl_ca_location = os.environ["REQUESTS_CA_BUNDLE"] diff --git a/tests/integration/test_aiohttp.py b/tests/integration/test_aiohttp.py index 49cf9934..a740ef21 100644 --- a/tests/integration/test_aiohttp.py +++ b/tests/integration/test_aiohttp.py @@ -36,8 +36,8 @@ def post(url, output="text", **kwargs): @pytest.mark.online -def test_status(tmpdir, mockbin_request_url): - url = mockbin_request_url +def test_status(tmpdir, httpbin): + url = httpbin.url with vcr.use_cassette(str(tmpdir.join("status.yaml"))): response, _ = get(url) @@ -50,8 +50,8 @@ def test_status(tmpdir, mockbin_request_url): @pytest.mark.online @pytest.mark.parametrize("auth", [None, aiohttp.BasicAuth("vcrpy", "test")]) -def test_headers(tmpdir, auth, mockbin_request_url): - url = mockbin_request_url +def test_headers(tmpdir, auth, httpbin): + url = httpbin.url with vcr.use_cassette(str(tmpdir.join("headers.yaml"))): response, _ = get(url, auth=auth) @@ -67,8 +67,8 @@ def test_headers(tmpdir, auth, mockbin_request_url): @pytest.mark.online -def test_case_insensitive_headers(tmpdir, mockbin_request_url): - url = mockbin_request_url +def test_case_insensitive_headers(tmpdir, httpbin): + url = httpbin.url with vcr.use_cassette(str(tmpdir.join("whatever.yaml"))): _, _ = get(url) @@ -81,8 +81,8 @@ def test_case_insensitive_headers(tmpdir, mockbin_request_url): @pytest.mark.online -def test_text(tmpdir, mockbin_request_url): - url = mockbin_request_url +def test_text(tmpdir, httpbin): + url = httpbin.url with vcr.use_cassette(str(tmpdir.join("text.yaml"))): _, response_text = get(url) @@ -94,8 +94,8 @@ def test_text(tmpdir, mockbin_request_url): @pytest.mark.online -def test_json(tmpdir, mockbin_request_url): - url = mockbin_request_url +def test_json(tmpdir, httpbin): + url = httpbin.url + "/json" headers = {"Content-Type": "application/json"} with vcr.use_cassette(str(tmpdir.join("json.yaml"))): @@ -108,8 +108,8 @@ def test_json(tmpdir, mockbin_request_url): @pytest.mark.online -def test_binary(tmpdir, mockbin_request_url): - url = mockbin_request_url + "/image/png" +def test_binary(tmpdir, httpbin): + url = httpbin.url + "/image/png" with vcr.use_cassette(str(tmpdir.join("binary.yaml"))): _, response_binary = get(url, output="raw") @@ -120,8 +120,8 @@ def test_binary(tmpdir, mockbin_request_url): @pytest.mark.online -def test_stream(tmpdir, mockbin_request_url): - url = mockbin_request_url +def test_stream(tmpdir, httpbin): + url = httpbin.url with vcr.use_cassette(str(tmpdir.join("stream.yaml"))): _, body = get(url, output="raw") # Do not use stream here, as the stream is exhausted by vcr @@ -134,10 +134,10 @@ def test_stream(tmpdir, mockbin_request_url): @pytest.mark.online @pytest.mark.parametrize("body", ["data", "json"]) -def test_post(tmpdir, body, caplog, mockbin_request_url): +def test_post(tmpdir, body, caplog, httpbin): caplog.set_level(logging.INFO) data = {"key1": "value1", "key2": "value2"} - url = mockbin_request_url + url = httpbin.url with vcr.use_cassette(str(tmpdir.join("post.yaml"))): _, response_json = post(url, **{body: data}) @@ -159,14 +159,14 @@ def test_post(tmpdir, body, caplog, mockbin_request_url): @pytest.mark.online -def test_params(tmpdir, mockbin_request_url): - url = mockbin_request_url + "?d=d" +def test_params(tmpdir, httpbin): + url = httpbin.url + "/get?d=d" headers = {"Content-Type": "application/json"} params = {"a": 1, "b": 2, "c": "c"} with vcr.use_cassette(str(tmpdir.join("get.yaml"))) as cassette: _, response_json = get(url, output="json", params=params, headers=headers) - assert response_json["queryString"] == {"a": "1", "b": "2", "c": "c", "d": "d"} + assert response_json["args"] == {"a": "1", "b": "2", "c": "c", "d": "d"} with vcr.use_cassette(str(tmpdir.join("get.yaml"))) as cassette: _, cassette_response_json = get(url, output="json", params=params, headers=headers) @@ -175,8 +175,8 @@ def test_params(tmpdir, mockbin_request_url): @pytest.mark.online -def test_params_same_url_distinct_params(tmpdir, mockbin_request_url): - url = mockbin_request_url +def test_params_same_url_distinct_params(tmpdir, httpbin): + url = httpbin.url + "/json" headers = {"Content-Type": "application/json"} params = {"a": 1, "b": 2, "c": "c"} @@ -195,8 +195,8 @@ def test_params_same_url_distinct_params(tmpdir, mockbin_request_url): @pytest.mark.online -def test_params_on_url(tmpdir, mockbin_request_url): - url = mockbin_request_url + "?a=1&b=foo" +def test_params_on_url(tmpdir, httpbin): + url = httpbin.url + "/get?a=1&b=foo" headers = {"Content-Type": "application/json"} with vcr.use_cassette(str(tmpdir.join("get.yaml"))) as cassette: @@ -261,8 +261,8 @@ def test_aiohttp_test_client_json(aiohttp_client, tmpdir): @pytest.mark.online -def test_redirect(tmpdir, mockbin): - url = mockbin + "/redirect/302/2" +def test_redirect(tmpdir, httpbin): + url = httpbin.url + "/redirect/2" with vcr.use_cassette(str(tmpdir.join("redirect.yaml"))): response, _ = get(url) @@ -284,9 +284,9 @@ def test_redirect(tmpdir, mockbin): @pytest.mark.online -def test_not_modified(tmpdir, mockbin): +def test_not_modified(tmpdir, httpbin): """It doesn't try to redirect on 304""" - url = mockbin + "/status/304" + url = httpbin.url + "/status/304" with vcr.use_cassette(str(tmpdir.join("not_modified.yaml"))): response, _ = get(url) @@ -302,13 +302,13 @@ def test_not_modified(tmpdir, mockbin): @pytest.mark.online -def test_double_requests(tmpdir, mockbin_request_url): +def test_double_requests(tmpdir, httpbin): """We should capture, record, and replay all requests and response chains, even if there are duplicate ones. We should replay in the order we saw them. """ - url = mockbin_request_url + url = httpbin.url with vcr.use_cassette(str(tmpdir.join("text.yaml"))): _, response_text1 = get(url, output="text") @@ -426,18 +426,18 @@ async def run(loop): @pytest.mark.online -def test_not_allow_redirects(tmpdir, mockbin): - url = mockbin + "/redirect/308/5" +def test_not_allow_redirects(tmpdir, httpbin): + url = httpbin + "/redirect-to?url=.%2F&status_code=308" path = str(tmpdir.join("redirects.yaml")) with vcr.use_cassette(path): response, _ = get(url, allow_redirects=False) - assert response.url.path == "/redirect/308/5" + assert response.url.path == "/redirect-to" assert response.status == 308 with vcr.use_cassette(path) as cassette: response, _ = get(url, allow_redirects=False) - assert response.url.path == "/redirect/308/5" + assert response.url.path == "/redirect-to" assert response.status == 308 assert cassette.play_count == 1 diff --git a/tests/integration/test_basic.py b/tests/integration/test_basic.py index fba9fe22..bfad6153 100644 --- a/tests/integration/test_basic.py +++ b/tests/integration/test_basic.py @@ -39,7 +39,7 @@ def test_basic_json_use(tmpdir, httpbin): test_fixture = str(tmpdir.join("synopsis.json")) with vcr.use_cassette(test_fixture, serializer="json"): response = urlopen(httpbin.url).read() - assert b"difficult sometimes" in response + assert b"A simple HTTP Request & Response Service." in response def test_patched_content(tmpdir, httpbin): diff --git a/tests/integration/test_config.py b/tests/integration/test_config.py index 274dbd07..6be10439 100644 --- a/tests/integration/test_config.py +++ b/tests/integration/test_config.py @@ -8,12 +8,12 @@ @pytest.mark.online -def test_set_serializer_default_config(tmpdir, mockbin_request_url): +def test_set_serializer_default_config(tmpdir, httpbin): my_vcr = vcr.VCR(serializer="json") with my_vcr.use_cassette(str(tmpdir.join("test.json"))): assert my_vcr.serializer == "json" - urlopen(mockbin_request_url) + urlopen(httpbin.url) with open(str(tmpdir.join("test.json"))) as f: file_content = f.read() @@ -22,37 +22,37 @@ def test_set_serializer_default_config(tmpdir, mockbin_request_url): @pytest.mark.online -def test_default_set_cassette_library_dir(tmpdir, mockbin_request_url): +def test_default_set_cassette_library_dir(tmpdir, httpbin): my_vcr = vcr.VCR(cassette_library_dir=str(tmpdir.join("subdir"))) with my_vcr.use_cassette("test.json"): - urlopen(mockbin_request_url) + urlopen(httpbin.url) assert os.path.exists(str(tmpdir.join("subdir").join("test.json"))) @pytest.mark.online -def test_override_set_cassette_library_dir(tmpdir, mockbin_request_url): +def test_override_set_cassette_library_dir(tmpdir, httpbin): my_vcr = vcr.VCR(cassette_library_dir=str(tmpdir.join("subdir"))) cld = str(tmpdir.join("subdir2")) with my_vcr.use_cassette("test.json", cassette_library_dir=cld): - urlopen(mockbin_request_url) + urlopen(httpbin.url) assert os.path.exists(str(tmpdir.join("subdir2").join("test.json"))) assert not os.path.exists(str(tmpdir.join("subdir").join("test.json"))) @pytest.mark.online -def test_override_match_on(tmpdir, mockbin_request_url): +def test_override_match_on(tmpdir, httpbin): my_vcr = vcr.VCR(match_on=["method"]) with my_vcr.use_cassette(str(tmpdir.join("test.json"))): - urlopen(mockbin_request_url) + urlopen(httpbin.url) with my_vcr.use_cassette(str(tmpdir.join("test.json"))) as cass: - urlopen(mockbin_request_url) + urlopen(httpbin.url) assert len(cass) == 1 assert cass.play_count == 1 @@ -67,12 +67,12 @@ def test_missing_matcher(): @pytest.mark.online -def test_dont_record_on_exception(tmpdir, mockbin_request_url): +def test_dont_record_on_exception(tmpdir, httpbin): my_vcr = vcr.VCR(record_on_exception=False) @my_vcr.use_cassette(str(tmpdir.join("dontsave.yml"))) def some_test(): - assert b"Not in content" in urlopen(mockbin_request_url) + assert b"Not in content" in urlopen(httpbin.url) with pytest.raises(AssertionError): some_test() @@ -82,6 +82,6 @@ def some_test(): # Make sure context decorator has the same behavior with pytest.raises(AssertionError): with my_vcr.use_cassette(str(tmpdir.join("dontsave2.yml"))): - assert b"Not in content" in urlopen(mockbin_request_url).read() + assert b"Not in content" in urlopen(httpbin.url).read() assert not os.path.exists(str(tmpdir.join("dontsave2.yml"))) diff --git a/tests/integration/test_disksaver.py b/tests/integration/test_disksaver.py index f9127d91..985762eb 100644 --- a/tests/integration/test_disksaver.py +++ b/tests/integration/test_disksaver.py @@ -12,19 +12,19 @@ @pytest.mark.online -def test_disk_saver_nowrite(tmpdir, mockbin_request_url): +def test_disk_saver_nowrite(tmpdir, httpbin): """ Ensure that when you close a cassette without changing it it doesn't rewrite the file """ fname = str(tmpdir.join("synopsis.yaml")) with vcr.use_cassette(fname) as cass: - urlopen(mockbin_request_url).read() + urlopen(httpbin.url).read() assert cass.play_count == 0 last_mod = os.path.getmtime(fname) with vcr.use_cassette(fname) as cass: - urlopen(mockbin_request_url).read() + urlopen(httpbin.url).read() assert cass.play_count == 1 assert cass.dirty is False last_mod2 = os.path.getmtime(fname) @@ -33,14 +33,14 @@ def test_disk_saver_nowrite(tmpdir, mockbin_request_url): @pytest.mark.online -def test_disk_saver_write(tmpdir, mockbin_request_url): +def test_disk_saver_write(tmpdir, httpbin): """ Ensure that when you close a cassette after changing it it does rewrite the file """ fname = str(tmpdir.join("synopsis.yaml")) with vcr.use_cassette(fname) as cass: - urlopen(mockbin_request_url).read() + urlopen(httpbin.url).read() assert cass.play_count == 0 last_mod = os.path.getmtime(fname) @@ -49,8 +49,8 @@ def test_disk_saver_write(tmpdir, mockbin_request_url): time.sleep(1) with vcr.use_cassette(fname, record_mode=vcr.mode.ANY) as cass: - urlopen(mockbin_request_url).read() - urlopen(mockbin_request_url + "/get").read() + urlopen(httpbin.url).read() + urlopen(httpbin.url + "/get").read() assert cass.play_count == 1 assert cass.dirty last_mod2 = os.path.getmtime(fname) diff --git a/tests/integration/test_httplib2.py b/tests/integration/test_httplib2.py index 2cafb39a..7fecd720 100644 --- a/tests/integration/test_httplib2.py +++ b/tests/integration/test_httplib2.py @@ -56,14 +56,14 @@ def test_response_headers(tmpdir, httpbin_both): @pytest.mark.online -def test_effective_url(tmpdir): +def test_effective_url(tmpdir, httpbin): """Ensure that the effective_url is captured""" - url = "http://mockbin.org/redirect/301" + url = httpbin.url + "/redirect-to?url=.%2F&status_code=301" with vcr.use_cassette(str(tmpdir.join("headers.yaml"))): resp, _ = http().request(url) effective_url = resp["content-location"] - assert effective_url == "http://mockbin.org/redirect/301/0" + assert effective_url == httpbin.url + "/" with vcr.use_cassette(str(tmpdir.join("headers.yaml"))): resp, _ = http().request(url) diff --git a/tests/integration/test_httpx.py b/tests/integration/test_httpx.py index 45415a0e..cf1f6c21 100644 --- a/tests/integration/test_httpx.py +++ b/tests/integration/test_httpx.py @@ -9,14 +9,12 @@ from vcr.stubs.httpx_stubs import HTTPX_REDIRECT_PARAM # noqa: E402 + @pytest.fixture(params=["https", "http"]) def scheme(request): """Fixture that returns both http and https.""" return request.param -@pytest.fixture -def httpbin(scheme): - return scheme + "://httpbin.org" class BaseDoRequest: _client_class = None @@ -24,7 +22,7 @@ class BaseDoRequest: def __init__(self, *args, **kwargs): self._client_args = args self._client_kwargs = kwargs - self._client_kwargs['follow_redirects'] = self._client_kwargs.get('follow_redirects', True) + self._client_kwargs["follow_redirects"] = self._client_kwargs.get("follow_redirects", True) def _make_client(self): return self._client_class(*self._client_args, **self._client_kwargs) @@ -49,11 +47,12 @@ def client(self): def __call__(self, *args, **kwargs): return self.client.request(*args, timeout=60, **kwargs) - + def stream(self, *args, **kwargs): with self.client.stream(*args, **kwargs) as response: return b"".join(response.iter_bytes()) + class DoAsyncRequest(BaseDoRequest): _client_class = httpx.AsyncClient @@ -88,14 +87,14 @@ def __call__(self, *args, **kwargs): # Use one-time context and dispose of the loop/client afterwards with self: return self._loop.run_until_complete(self.client.request(*args, **kwargs)) - + async def _get_stream(self, *args, **kwargs): async with self.client.stream(*args, **kwargs) as response: content = b"" async for c in response.aiter_bytes(): content += c return content - + def stream(self, *args, **kwargs): if hasattr(self, "_loop"): return self._loop.run_until_complete(self._get_stream(*args, **kwargs)) @@ -104,6 +103,7 @@ def stream(self, *args, **kwargs): with self: return self._loop.run_until_complete(self._get_stream(*args, **kwargs)) + def pytest_generate_tests(metafunc): if "do_request" in metafunc.fixturenames: metafunc.parametrize("do_request", [DoAsyncRequest, DoSyncRequest]) @@ -116,7 +116,7 @@ def yml(tmpdir, request): @pytest.mark.online def test_status(tmpdir, httpbin, do_request): - url = httpbin + url = httpbin.url with vcr.use_cassette(str(tmpdir.join("status.yaml"))): response = do_request()("GET", url) @@ -129,7 +129,7 @@ def test_status(tmpdir, httpbin, do_request): @pytest.mark.online def test_case_insensitive_headers(tmpdir, httpbin, do_request): - url = httpbin + url = httpbin.url with vcr.use_cassette(str(tmpdir.join("whatever.yaml"))): do_request()("GET", url) @@ -143,7 +143,7 @@ def test_case_insensitive_headers(tmpdir, httpbin, do_request): @pytest.mark.online def test_content(tmpdir, httpbin, do_request): - url = httpbin + url = httpbin.url with vcr.use_cassette(str(tmpdir.join("cointent.yaml"))): response = do_request()("GET", url) @@ -156,7 +156,7 @@ def test_content(tmpdir, httpbin, do_request): @pytest.mark.online def test_json(tmpdir, httpbin, do_request): - url = httpbin + "/json" + url = httpbin.url + "/json" with vcr.use_cassette(str(tmpdir.join("json.yaml"))): response = do_request()("GET", url) @@ -169,7 +169,7 @@ def test_json(tmpdir, httpbin, do_request): @pytest.mark.online def test_params_same_url_distinct_params(tmpdir, httpbin, do_request): - url = httpbin + "/get" + url = httpbin.url + "/get" headers = {"Content-Type": "application/json"} params = {"a": 1, "b": False, "c": "c"} @@ -190,7 +190,7 @@ def test_params_same_url_distinct_params(tmpdir, httpbin, do_request): @pytest.mark.online def test_redirect(httpbin, yml, do_request): - url = httpbin + "/redirect-to" + url = httpbin.url + "/redirect-to" response = do_request()("GET", url) with vcr.use_cassette(yml): @@ -213,7 +213,7 @@ def test_redirect(httpbin, yml, do_request): @pytest.mark.online def test_work_with_gzipped_data(httpbin, do_request, yml): - url = httpbin + "/gzip?foo=bar" + url = httpbin.url + "/gzip?foo=bar" headers = {"accept-encoding": "deflate, gzip"} with vcr.use_cassette(yml): @@ -238,6 +238,7 @@ def test_simple_fetching(do_request, yml, url): assert str(cassette_response.request.url) == url assert cassette.play_count == 1 + @pytest.mark.online def test_cookies(tmpdir, httpbin, do_request): def client_cookies(client): @@ -246,7 +247,7 @@ def client_cookies(client): def response_cookies(response): return list(response.cookies) - url = httpbin + "/cookies/set" + url = httpbin.url + "/cookies/set" params = {"k1": "v1", "k2": "v2"} with do_request(params=params, follow_redirects=False) as client: @@ -273,19 +274,18 @@ def response_cookies(response): assert response_cookies(cassette_response) == ["k1", "k2"] assert client_cookies(new_client) == ["k1", "k2"] + @pytest.mark.online def test_stream(tmpdir, httpbin, do_request): - url = httpbin + "/stream-bytes/512" + url = httpbin.url + "/stream-bytes/512" testfile = str(tmpdir.join("stream.yml")) with vcr.use_cassette(testfile): response_content = do_request().stream("GET", url) assert len(response_content) == 512 - with vcr.use_cassette(testfile) as cassette: cassette_content = do_request().stream("GET", url) assert cassette_content == response_content assert len(cassette_content) == 512 assert cassette.play_count == 1 - \ No newline at end of file diff --git a/tests/integration/test_proxy.py b/tests/integration/test_proxy.py index e4376432..c450ab23 100644 --- a/tests/integration/test_proxy.py +++ b/tests/integration/test_proxy.py @@ -2,6 +2,7 @@ import http.server import multiprocessing +import threading import socketserver from urllib.request import urlopen @@ -29,7 +30,8 @@ def do_GET(self): # In Python 2 the response is an addinfourl instance. status = upstream_response.code headers = upstream_response.info().items() - self.send_response(status, upstream_response.msg) + self.log_request(status) + self.send_response_only(status, upstream_response.msg) for header in headers: self.send_header(*header) self.end_headers() @@ -39,10 +41,11 @@ def do_GET(self): @pytest.fixture(scope="session") def proxy_server(): httpd = socketserver.ThreadingTCPServer(("", 0), Proxy) - proxy_process = multiprocessing.Process(target=httpd.serve_forever) + proxy_process = threading.Thread(target=httpd.serve_forever) proxy_process.start() yield "http://{}:{}".format(*httpd.server_address) - proxy_process.terminate() + httpd.shutdown() + proxy_process.join() def test_use_proxy(tmpdir, httpbin, proxy_server): @@ -50,8 +53,10 @@ def test_use_proxy(tmpdir, httpbin, proxy_server): with vcr.use_cassette(str(tmpdir.join("proxy.yaml"))): response = requests.get(httpbin.url, proxies={"http": proxy_server}) - with vcr.use_cassette(str(tmpdir.join("proxy.yaml"))) as cassette: + with vcr.use_cassette(str(tmpdir.join("proxy.yaml")), mode="once") as cassette: cassette_response = requests.get(httpbin.url, proxies={"http": proxy_server}) + for key in set(cassette_response.headers.keys()) & set(response.headers.keys()): + assert cassette_response.headers[key] == response.headers[key] assert cassette_response.headers == response.headers assert cassette.play_count == 1 diff --git a/tests/integration/test_register_matcher.py b/tests/integration/test_register_matcher.py index b6e20989..9b9b5ad5 100644 --- a/tests/integration/test_register_matcher.py +++ b/tests/integration/test_register_matcher.py @@ -14,28 +14,28 @@ def false_matcher(r1, r2): @pytest.mark.online -def test_registered_true_matcher(tmpdir, mockbin_request_url): +def test_registered_true_matcher(tmpdir, httpbin): my_vcr = vcr.VCR() my_vcr.register_matcher("true", true_matcher) testfile = str(tmpdir.join("test.yml")) with my_vcr.use_cassette(testfile, match_on=["true"]): # These 2 different urls are stored as the same request - urlopen(mockbin_request_url) - urlopen(mockbin_request_url + "/get") + urlopen(httpbin.url) + urlopen(httpbin.url + "/get") with my_vcr.use_cassette(testfile, match_on=["true"]): # I can get the response twice even though I only asked for it once - urlopen(mockbin_request_url) - urlopen(mockbin_request_url) + urlopen(httpbin.url) + urlopen(httpbin.url) @pytest.mark.online -def test_registered_false_matcher(tmpdir, mockbin_request_url): +def test_registered_false_matcher(tmpdir, httpbin): my_vcr = vcr.VCR() my_vcr.register_matcher("false", false_matcher) testfile = str(tmpdir.join("test.yml")) with my_vcr.use_cassette(testfile, match_on=["false"]) as cass: # These 2 different urls are stored as different requests - urlopen(mockbin_request_url) - urlopen(mockbin_request_url + "/get") + urlopen(httpbin.url) + urlopen(httpbin.url + "/get") assert len(cass) == 2 diff --git a/tests/integration/test_register_persister.py b/tests/integration/test_register_persister.py index 31d54db8..e904197c 100644 --- a/tests/integration/test_register_persister.py +++ b/tests/integration/test_register_persister.py @@ -66,7 +66,7 @@ def test_load_cassette_with_custom_persister(tmpdir, httpbin): with my_vcr.use_cassette(test_fixture, serializer="json"): response = urlopen(httpbin.url).read() - assert b"difficult sometimes" in response + assert b"A simple HTTP Request & Response Service." in response def test_load_cassette_persister_exception_handling(tmpdir, httpbin): diff --git a/tests/integration/test_tornado.py b/tests/integration/test_tornado.py index e5c75690..2013088f 100644 --- a/tests/integration/test_tornado.py +++ b/tests/integration/test_tornado.py @@ -81,12 +81,12 @@ def test_body(get_client, tmpdir, scheme): @pytest.mark.gen_test -def test_effective_url(get_client, scheme, tmpdir): +def test_effective_url(get_client, tmpdir, httpbin): """Ensure that the effective_url is captured""" - url = scheme + "://mockbin.org/redirect/301?url=/html" + url = httpbin.url + "/redirect/1" with vcr.use_cassette(str(tmpdir.join("url.yaml"))): effective_url = (yield get(get_client(), url)).effective_url - assert effective_url == scheme + "://mockbin.org/redirect/301/0" + assert effective_url == httpbin.url + "/get" with vcr.use_cassette(str(tmpdir.join("url.yaml"))) as cass: assert effective_url == (yield get(get_client(), url)).effective_url diff --git a/tests/integration/test_urllib2.py b/tests/integration/test_urllib2.py index 9d56556b..2e50c3b7 100644 --- a/tests/integration/test_urllib2.py +++ b/tests/integration/test_urllib2.py @@ -57,13 +57,13 @@ def test_response_headers(httpbin_both, tmpdir): @mark.online -def test_effective_url(tmpdir): +def test_effective_url(tmpdir, httpbin): """Ensure that the effective_url is captured""" - url = "http://mockbin.org/redirect/301" + url = httpbin.url + "/redirect-to?url=.%2F&status_code=301" with vcr.use_cassette(str(tmpdir.join("headers.yaml"))): effective_url = urlopen_with_cafile(url).geturl() - assert effective_url == "http://mockbin.org/redirect/301/0" + assert effective_url == httpbin.url + "/" with vcr.use_cassette(str(tmpdir.join("headers.yaml"))): assert effective_url == urlopen_with_cafile(url).geturl() diff --git a/tests/integration/test_urllib3.py b/tests/integration/test_urllib3.py index 710ffdf8..fdaf620f 100644 --- a/tests/integration/test_urllib3.py +++ b/tests/integration/test_urllib3.py @@ -99,9 +99,9 @@ def test_post(tmpdir, httpbin_both, verify_pool_mgr): @pytest.mark.online -def test_redirects(tmpdir, verify_pool_mgr): +def test_redirects(tmpdir, verify_pool_mgr, httpbin): """Ensure that we can handle redirects""" - url = "http://mockbin.org/redirect/301" + url = httpbin.url + "/redirect/1" with vcr.use_cassette(str(tmpdir.join("verify_pool_mgr.yaml"))): content = verify_pool_mgr.request("GET", url).data diff --git a/tox.ini b/tox.ini index e1476903..62035131 100644 --- a/tox.ini +++ b/tox.ini @@ -6,7 +6,7 @@ envlist = {py38,py39,py310,py311,py312}-{requests-urllib3-1,httplib2,urllib3-1,tornado4,boto3,aiohttp,httpx}, {py310,py311,py312}-{requests-urllib3-2,urllib3-2}, {pypy3}-{requests-urllib3-1,httplib2,urllib3-1,tornado4,boto3}, - {py310}-httpx019, + #{py310}-httpx019, cov-report From cbf9b6dab37fad211d469e35daec335781d2761b Mon Sep 17 00:00:00 2001 From: Parker Hancock <633163+parkerhancock@users.noreply.github.com> Date: Fri, 8 Dec 2023 10:57:32 -0600 Subject: [PATCH 4/5] make linters happy --- tests/integration/test_httpx.py | 2 -- tests/integration/test_proxy.py | 3 +-- vcr/serializers/compat.py | 2 +- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/tests/integration/test_httpx.py b/tests/integration/test_httpx.py index cf1f6c21..4bedc8a2 100644 --- a/tests/integration/test_httpx.py +++ b/tests/integration/test_httpx.py @@ -1,4 +1,3 @@ -import os import pytest @@ -7,7 +6,6 @@ asyncio = pytest.importorskip("asyncio") httpx = pytest.importorskip("httpx") -from vcr.stubs.httpx_stubs import HTTPX_REDIRECT_PARAM # noqa: E402 @pytest.fixture(params=["https", "http"]) diff --git a/tests/integration/test_proxy.py b/tests/integration/test_proxy.py index c450ab23..7366d33e 100644 --- a/tests/integration/test_proxy.py +++ b/tests/integration/test_proxy.py @@ -1,9 +1,8 @@ """Test using a proxy.""" import http.server -import multiprocessing -import threading import socketserver +import threading from urllib.request import urlopen import pytest diff --git a/vcr/serializers/compat.py b/vcr/serializers/compat.py index 0ab358d2..65ba96fc 100644 --- a/vcr/serializers/compat.py +++ b/vcr/serializers/compat.py @@ -56,7 +56,7 @@ def convert_body_to_unicode(resp): If the request or responses body is bytes, decode it to a string (for python3 support) """ - if type(resp) is not dict: + if not isinstance(resp, dict): # Some of the tests just serialize and deserialize a string. return _convert_string_to_unicode(resp) else: From 5e6c20cbd3fd1db1ecf5b0d9ab8a4f96f718aa9e Mon Sep 17 00:00:00 2001 From: Parker Hancock <633163+parkerhancock@users.noreply.github.com> Date: Fri, 8 Dec 2023 13:15:14 -0600 Subject: [PATCH 5/5] more attempts to make the linters happy --- tests/integration/test_httpx.py | 2 -- vcr/stubs/httpx_stubs.py | 1 - 2 files changed, 3 deletions(-) diff --git a/tests/integration/test_httpx.py b/tests/integration/test_httpx.py index 4bedc8a2..22b3e693 100644 --- a/tests/integration/test_httpx.py +++ b/tests/integration/test_httpx.py @@ -1,4 +1,3 @@ - import pytest import vcr @@ -7,7 +6,6 @@ httpx = pytest.importorskip("httpx") - @pytest.fixture(params=["https", "http"]) def scheme(request): """Fixture that returns both http and https.""" diff --git a/vcr/stubs/httpx_stubs.py b/vcr/stubs/httpx_stubs.py index 0bb3b4a4..aa5e05a4 100644 --- a/vcr/stubs/httpx_stubs.py +++ b/vcr/stubs/httpx_stubs.py @@ -111,7 +111,6 @@ def _play_responses(cassette, request, vcr_request, client, kwargs): return response - async def _async_vcr_send(cassette, real_send, *args, **kwargs): vcr_request, response = _shared_vcr_send(cassette, real_send, *args, **kwargs) if response: