From 99c03847707c8f75b7b14f00d1986993f36cc4be Mon Sep 17 00:00:00 2001 From: Daniel Silva Date: Mon, 2 Jan 2023 03:52:52 +0000 Subject: [PATCH 1/3] feat: add an option to exclude unused interactions Introduce the `drop_unused_requests` option (False by default). If True, it will force the `Cassette` saving operation with only played old interactions and new ones if they exist. As a result, unused old requests are dropped. Add `_old_interactions`, `_played_interactions` and `_new_interactions()`. The `_old_interactions` are previously recorded interactions loaded from Cassette files. The `_played_interactions` is a set of old interactions that were marked as played. A new interaction is a tuple (request, response) in `self.data` that is not in `_old_interactions` list. --- vcr/cassette.py | 27 ++++++++++++++++++++++++++- vcr/config.py | 3 +++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/vcr/cassette.py b/vcr/cassette.py index 5822afac..107e5d73 100644 --- a/vcr/cassette.py +++ b/vcr/cassette.py @@ -197,6 +197,7 @@ def __init__( custom_patches=(), inject=False, allow_playback_repeats=False, + drop_unused_requests=False, ): self._persister = persister or FilesystemPersister self._path = path @@ -209,6 +210,7 @@ def __init__( self.record_mode = record_mode self.custom_patches = custom_patches self.allow_playback_repeats = allow_playback_repeats + self.drop_unused_requests = drop_unused_requests # self.data is the list of (req, resp) tuples self.data = [] @@ -216,6 +218,10 @@ def __init__( self.dirty = False self.rewound = False + # Subsets of self.data to store old and played interactions + self._old_interactions = [] + self._played_interactions = [] + @property def play_count(self): return sum(self.play_counts.values()) @@ -277,6 +283,7 @@ def play_response(self, request): for index, response in self._responses(request): if self.play_counts[index] == 0 or self.allow_playback_repeats: self.play_counts[index] += 1 + self._played_interactions.append((request, response)) return response # The cassette doesn't contain the request asked for. raise UnhandledHTTPRequestError( @@ -337,10 +344,27 @@ def find_requests_with_most_matches(self, request): return final_best_matches + def _new_interactions(self): + """List of new HTTP interactions (request/response tuples)""" + new_interactions = [] + for request, response in self.data: + if all(not requests_match(request, old_request, self._match_on) + for old_request, _ in self._old_interactions): + new_interactions.append((request, response)) + return new_interactions + def _as_dict(self): - return {"requests": self.requests, "responses": self.responses} + requests = self.requests + responses = self.responses + if self.drop_unused_requests: + interactions = self._played_interactions + self._new_interactions() + requests = [request for request, _ in interactions] + responses = [response for _, response in interactions] + return {"requests": requests, "responses": responses} def _save(self, force=False): + if (len(self._played_interactions) < len(self._old_interactions)): + force = True if force or self.dirty: self._persister.save_cassette(self._path, self._as_dict(), serializer=self._serializer) self.dirty = False @@ -350,6 +374,7 @@ def _load(self): requests, responses = self._persister.load_cassette(self._path, serializer=self._serializer) for request, response in zip(requests, responses): self.append(request, response) + self._old_interactions.append((request, response)) self.dirty = False self.rewound = True except ValueError: diff --git a/vcr/config.py b/vcr/config.py index a991c958..7139bd34 100644 --- a/vcr/config.py +++ b/vcr/config.py @@ -50,6 +50,7 @@ def __init__( func_path_generator=None, decode_compressed_response=False, record_on_exception=True, + drop_unused_requests=False, ): self.serializer = serializer self.match_on = match_on @@ -83,6 +84,7 @@ def __init__( self.decode_compressed_response = decode_compressed_response self.record_on_exception = record_on_exception self._custom_patches = tuple(custom_patches) + self.drop_unused_requests = drop_unused_requests def _get_serializer(self, serializer_name): try: @@ -153,6 +155,7 @@ def add_cassette_library_dir(path): "func_path_generator": func_path_generator, "allow_playback_repeats": kwargs.get("allow_playback_repeats", False), "record_on_exception": record_on_exception, + "drop_unused_requests": kwargs.get("drop_unused_requests", self.drop_unused_requests), } path = kwargs.get("path") if path: From 010fa268d130f4d95a8b93bc728ff734749c4c5c Mon Sep 17 00:00:00 2001 From: Daniel Silva Date: Wed, 4 Jan 2023 20:02:43 +0000 Subject: [PATCH 2/3] test: add tests to drop_unused_requests option --- tests/integration/test_config.py | 18 ++++++++++++++++++ tests/unit/test_cassettes.py | 23 +++++++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/tests/integration/test_config.py b/tests/integration/test_config.py index 013f8493..361e7a33 100644 --- a/tests/integration/test_config.py +++ b/tests/integration/test_config.py @@ -5,6 +5,7 @@ import pytest import vcr +from vcr.cassette import Cassette def test_set_serializer_default_config(tmpdir, httpbin): @@ -80,3 +81,20 @@ def some_test(): assert b"Not in content" in urlopen("http://httpbin.org/get").read() assert not os.path.exists(str(tmpdir.join("dontsave2.yml"))) + +def test_set_drop_unused_requests(tmpdir, httpbin): + my_vcr = vcr.VCR(drop_unused_requests=True) + file = str(tmpdir.join("test.yaml")) + + with my_vcr.use_cassette(file): + urlopen(httpbin.url) + urlopen(httpbin.url + "/get") + + cassette = Cassette.load(path=file) + assert len(cassette) == 2 + + with my_vcr.use_cassette(file): + urlopen(httpbin.url) + + cassette = Cassette.load(path=file) + assert len(cassette) == 1 diff --git a/tests/unit/test_cassettes.py b/tests/unit/test_cassettes.py index 41e3df53..0acd1164 100644 --- a/tests/unit/test_cassettes.py +++ b/tests/unit/test_cassettes.py @@ -11,6 +11,7 @@ from vcr.cassette import Cassette from vcr.errors import UnhandledHTTPRequestError from vcr.patch import force_reset +from vcr.request import Request from vcr.stubs import VCRHTTPSConnection @@ -394,3 +395,25 @@ def test_find_requests_with_most_matches_many_similar_requests(mock_get_matchers (1, ["method", "path"], [("query", "failed : query")]), (3, ["method", "path"], [("query", "failed : query")]), ] + + +def test_used_interactions(tmpdir): + interactions = [ + {"request": {"body": "", "uri": "foo1", "method": "GET", "headers": {}}, "response": "bar1"}, + {"request": {"body": "", "uri": "foo2", "method": "GET", "headers": {}}, "response": "bar2"}, + {"request": {"body": "", "uri": "foo3", "method": "GET", "headers": {}}, "response": "bar3"} + ] + file = tmpdir.join("test_cassette.yml") + file.write(yaml.dump({"interactions": [interactions[0], interactions[1]]})) + + cassette = Cassette.load(path=str(file)) + request = Request._from_dict(interactions[1]["request"]) + cassette.play_response(request) + assert len(cassette._played_interactions) < len(cassette._old_interactions) + + request = Request._from_dict(interactions[2]["request"]) + cassette.append(request, interactions[2]["response"]) + assert len(cassette._new_interactions()) == 1 + + used_interactions = cassette._played_interactions + cassette._new_interactions() + assert len(used_interactions) == 2 From 36c7465cf742e2766e4ee813140c276b2d759c8b Mon Sep 17 00:00:00 2001 From: Daniel Silva Date: Wed, 4 Jan 2023 21:59:58 +0000 Subject: [PATCH 3/3] docs: add drop_unused_requests option --- docs/advanced.rst | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/advanced.rst b/docs/advanced.rst index fb287fa3..b335aa83 100644 --- a/docs/advanced.rst +++ b/docs/advanced.rst @@ -426,3 +426,16 @@ If you want to save the cassette only when the test succeedes, set the Cassette # Since there was an exception, the cassette file hasn't been created. assert not os.path.exists('fixtures/vcr_cassettes/synopsis.yaml') + +Drop unused requests +-------------------- + +Even if any HTTP request is changed or removed from tests, previously recorded +interactions remain in the cassette file. If set the ``drop_unused_requests`` +option to ``True``, VCR will not save old HTTP interactions if they are not used. + +.. code:: python + + my_vcr = VCR(drop_unused_requests=True) + with my_vcr.use_cassette('fixtures/vcr_cassettes/synopsis.yaml'): + # your http here