From 83adac604f19c9309ae1f0a25401a574f14b7d3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Abril=20Rinc=C3=B3n=20Blanco?= Date: Thu, 12 Dec 2024 14:26:30 +0100 Subject: [PATCH 01/10] Keep sessions around between each ConanRequester Avoid creating a new session for each ConanRequester instance, this helps a lot with performance, as it avoids the overhead duplicate handshakes --- conans/client/rest/conan_requester.py | 31 +++++++++++++++------------ 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/conans/client/rest/conan_requester.py b/conans/client/rest/conan_requester.py index edb2ce98926..a7a4f982ba0 100644 --- a/conans/client/rest/conan_requester.py +++ b/conans/client/rest/conan_requester.py @@ -97,20 +97,11 @@ def add_auth(self, url, kwargs): class ConanRequester: + _http_requester_imp = None def __init__(self, config, cache_folder=None): - # TODO: Make all this lazy, to avoid fully configuring Requester, for every api call - # even if it doesn't use it - # FIXME: Trick for testing when requests is mocked - if hasattr(requests, "Session"): - self._http_requester = requests.Session() - adapter = HTTPAdapter(max_retries=self._get_retries(config)) - self._http_requester.mount("http://", adapter) - self._http_requester.mount("https://", adapter) - else: - self._http_requester = requests - self._url_creds = _SourceURLCredentials(cache_folder) + self._max_retries = config.get("core.net.http:max_retries", default=2, check_type=int) self._timeout = config.get("core.net.http:timeout", default=DEFAULT_TIMEOUT) self._no_proxy_match = config.get("core.net.http:no_proxy_match", check_type=list) self._proxies = config.get("core.net.http:proxies") @@ -123,9 +114,21 @@ def __init__(self, config, cache_folder=None): platform.machine()]) self._user_agent = "Conan/%s (%s)" % (__version__, platform_info) - @staticmethod - def _get_retries(config): - retry = config.get("core.net.http:max_retries", default=2, check_type=int) + @property + def _http_requester(self): + if ConanRequester._http_requester_imp is None: + # FIXME: Trick for testing when requests is mocked + if hasattr(requests, "Session"): + ConanRequester._http_requester_imp = requests.Session() + adapter = HTTPAdapter(max_retries=self._get_retries()) + ConanRequester._http_requester_imp.mount("http://", adapter) + ConanRequester._http_requester_imp.mount("https://", adapter) + else: + ConanRequester._http_requester_imp = requests + return ConanRequester._http_requester_imp + + def _get_retries(self): + retry = self._max_retries if retry == 0: return 0 retry_status_code_set = { From e26a354d84b24a3c4a175189f1f1a9d8a2697e17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Abril=20Rinc=C3=B3n=20Blanco?= Date: Thu, 12 Dec 2024 15:19:50 +0100 Subject: [PATCH 02/10] Patch new cached value on ConanRequester tests --- conan/test/utils/tools.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/conan/test/utils/tools.py b/conan/test/utils/tools.py index 4a0e9448b20..479e578690b 100644 --- a/conan/test/utils/tools.py +++ b/conan/test/utils/tools.py @@ -531,7 +531,8 @@ def chdir(self, newdir): def mocked_servers(self, requester=None): _req = requester or TestRequester(self.servers) with mock.patch("conans.client.rest.conan_requester.requests", _req): - yield + with mock.patch("conans.client.rest.conan_requester.ConanRequester._http_requester_imp", _req): + yield @contextmanager def mocked_io(self): From 0dba33d71918b1a64fc2713503cf624b1aff6c85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Abril=20Rinc=C3=B3n=20Blanco?= Date: Thu, 12 Dec 2024 22:19:48 +0100 Subject: [PATCH 03/10] Modify adapter as needed --- conans/client/rest/conan_requester.py | 9 ++++++--- .../integration/configuration/requester_test.py | 17 +++++++++-------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/conans/client/rest/conan_requester.py b/conans/client/rest/conan_requester.py index a7a4f982ba0..880dda8aa0e 100644 --- a/conans/client/rest/conan_requester.py +++ b/conans/client/rest/conan_requester.py @@ -98,10 +98,13 @@ def add_auth(self, url, kwargs): class ConanRequester: _http_requester_imp = None + _adapter = None def __init__(self, config, cache_folder=None): self._url_creds = _SourceURLCredentials(cache_folder) self._max_retries = config.get("core.net.http:max_retries", default=2, check_type=int) + if ConanRequester._http_requester_imp is not None and hasattr(requests, "Session"): + ConanRequester._adapter.max_retries = self._get_retries() self._timeout = config.get("core.net.http:timeout", default=DEFAULT_TIMEOUT) self._no_proxy_match = config.get("core.net.http:no_proxy_match", check_type=list) self._proxies = config.get("core.net.http:proxies") @@ -120,9 +123,9 @@ def _http_requester(self): # FIXME: Trick for testing when requests is mocked if hasattr(requests, "Session"): ConanRequester._http_requester_imp = requests.Session() - adapter = HTTPAdapter(max_retries=self._get_retries()) - ConanRequester._http_requester_imp.mount("http://", adapter) - ConanRequester._http_requester_imp.mount("https://", adapter) + ConanRequester._adapter = HTTPAdapter(max_retries=self._get_retries()) + ConanRequester._http_requester_imp.mount("http://", ConanRequester._adapter) + ConanRequester._http_requester_imp.mount("https://", ConanRequester._adapter) else: ConanRequester._http_requester_imp = requests return ConanRequester._http_requester_imp diff --git a/test/integration/configuration/requester_test.py b/test/integration/configuration/requester_test.py index 5acb473602f..81d538396bd 100644 --- a/test/integration/configuration/requester_test.py +++ b/test/integration/configuration/requester_test.py @@ -49,11 +49,12 @@ class ConanRequesterHeadersTests(unittest.TestCase): def test_user_agent(self): mock_http_requester = MagicMock() with mock.patch("conans.client.rest.conan_requester.requests", mock_http_requester): - requester = ConanRequester(ConfDefinition()) - requester.get(url="aaa") - headers = requester._http_requester.get.call_args[1]["headers"] - self.assertIn("Conan/%s" % __version__, headers["User-Agent"]) - - requester.get(url="aaa", headers={"User-Agent": "MyUserAgent"}) - headers = requester._http_requester.get.call_args[1]["headers"] - self.assertEqual("MyUserAgent", headers["User-Agent"]) + with mock.patch("conans.client.rest.conan_requester.ConanRequester._http_requester_imp", mock_http_requester): + requester = ConanRequester(ConfDefinition()) + requester.get(url="aaa") + headers = requester._http_requester.get.call_args[1]["headers"] + self.assertIn("Conan/%s" % __version__, headers["User-Agent"]) + + requester.get(url="aaa", headers={"User-Agent": "MyUserAgent"}) + headers = requester._http_requester.get.call_args[1]["headers"] + self.assertEqual("MyUserAgent", headers["User-Agent"]) From ab839d8b2347e053ebd50e7a2a07cc3b29d76a55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Abril=20Rinc=C3=B3n=20Blanco?= Date: Mon, 16 Dec 2024 19:35:00 +0100 Subject: [PATCH 04/10] Fix tests --- test/integration/remote/rest_api_test.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/integration/remote/rest_api_test.py b/test/integration/remote/rest_api_test.py index 56855079674..e17e9ac0cba 100644 --- a/test/integration/remote/rest_api_test.py +++ b/test/integration/remote/rest_api_test.py @@ -18,7 +18,7 @@ from conan.test.utils.mocks import LocalDBMock from conan.test.utils.server_launcher import TestServerLauncher from conan.test.utils.test_files import temp_folder -from conan.test.utils.tools import get_free_port +from conan.test.utils.tools import get_free_port, TestRequester from conans.util.files import md5, save @@ -43,6 +43,7 @@ def setUpClass(cls): save(filename, "") config = ConfDefinition() requester = ConanRequester(config) + ConanRequester._http_requester_imp = TestRequester({"default", cls.server},) localdb = LocalDBMock() mocked_user_input = UserInput(non_interactive=False) @@ -61,6 +62,7 @@ def setUpClass(cls): @classmethod def tearDownClass(cls): + ConanRequester._http_requester_imp = None cls.server.stop() def tearDown(self): From 127fc08c9db3eeca55656c8a2ddac7cd3e2b8725 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Abril=20Rinc=C3=B3n=20Blanco?= Date: Mon, 16 Dec 2024 19:35:56 +0100 Subject: [PATCH 05/10] Name --- test/integration/remote/rest_api_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/remote/rest_api_test.py b/test/integration/remote/rest_api_test.py index e17e9ac0cba..e705891a96e 100644 --- a/test/integration/remote/rest_api_test.py +++ b/test/integration/remote/rest_api_test.py @@ -43,7 +43,7 @@ def setUpClass(cls): save(filename, "") config = ConfDefinition() requester = ConanRequester(config) - ConanRequester._http_requester_imp = TestRequester({"default", cls.server},) + ConanRequester._http_requester_imp = TestRequester({"myremote", cls.server},) localdb = LocalDBMock() mocked_user_input = UserInput(non_interactive=False) From 065926d87f435a8ca1343e38a216f43e039911bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Abril=20Rinc=C3=B3n=20Blanco?= Date: Thu, 19 Dec 2024 22:02:19 +0100 Subject: [PATCH 06/10] Update conans/client/rest/conan_requester.py --- conans/client/rest/conan_requester.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/conans/client/rest/conan_requester.py b/conans/client/rest/conan_requester.py index 880dda8aa0e..65d3623194d 100644 --- a/conans/client/rest/conan_requester.py +++ b/conans/client/rest/conan_requester.py @@ -103,8 +103,6 @@ class ConanRequester: def __init__(self, config, cache_folder=None): self._url_creds = _SourceURLCredentials(cache_folder) self._max_retries = config.get("core.net.http:max_retries", default=2, check_type=int) - if ConanRequester._http_requester_imp is not None and hasattr(requests, "Session"): - ConanRequester._adapter.max_retries = self._get_retries() self._timeout = config.get("core.net.http:timeout", default=DEFAULT_TIMEOUT) self._no_proxy_match = config.get("core.net.http:no_proxy_match", check_type=list) self._proxies = config.get("core.net.http:proxies") From 7057bd039f58d765df594803636569dfe2832f64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Abril=20Rinc=C3=B3n=20Blanco?= Date: Mon, 23 Dec 2024 10:13:16 +0100 Subject: [PATCH 07/10] Refactor requester tests --- conan/api/conan_api.py | 9 ++++++--- conan/api/subapi/remotes.py | 3 +++ conan/internal/conan_app.py | 5 ++--- conan/test/utils/tools.py | 13 ++++++++++-- conans/client/rest/conan_requester.py | 19 ++++-------------- .../command/remote_verify_ssl_test.py | 20 +++++++++++++++++++ test/integration/command/remove_test.py | 16 +++++++-------- test/integration/conan_api/search_test.py | 8 ++++---- .../configuration/requester_test.py | 17 ++++++++-------- test/integration/remote/rest_api_test.py | 2 -- 10 files changed, 66 insertions(+), 46 deletions(-) diff --git a/conan/api/conan_api.py b/conan/api/conan_api.py index 0b280e5ed58..3d806d32f68 100644 --- a/conan/api/conan_api.py +++ b/conan/api/conan_api.py @@ -40,10 +40,14 @@ def __init__(self, cache_folder=None): # Migration system migrator = ClientMigrator(self.cache_folder, conan_version) migrator.migrate() + self.reinit() - self.command = CommandAPI(self) + + def reinit(self): + self.config = ConfigAPI(self) self.remotes = RemotesAPI(self) - # Search recipes by wildcard and packages filtering by configuracion + self.command = CommandAPI(self) + # Search recipes by wildcard and packages filtering by configuration self.search = SearchAPI(self) # Get latest refs and list refs of recipes and packages self.list = ListAPI(self) @@ -52,7 +56,6 @@ def __init__(self, cache_folder=None): self.graph = GraphAPI(self) self.export = ExportAPI(self) self.remove = RemoveAPI(self) - self.config = ConfigAPI(self) self.new = NewAPI(self) self.upload = UploadAPI(self) self.download = DownloadAPI(self) diff --git a/conan/api/subapi/remotes.py b/conan/api/subapi/remotes.py index 4377d5a02f4..66163441bee 100644 --- a/conan/api/subapi/remotes.py +++ b/conan/api/subapi/remotes.py @@ -7,6 +7,7 @@ from conan.api.output import ConanOutput from conan.internal.cache.home_paths import HomePaths from conan.internal.conan_app import ConanApp +from conans.client.rest.conan_requester import ConanRequester from conans.client.rest_client_local_recipe_index import add_local_recipes_index_remote, \ remove_local_recipes_index_remote from conan.internal.api.remotes.localdb import LocalDB @@ -30,6 +31,8 @@ def __init__(self, conan_api): # This method is private, the subapi is not instantiated by users self.conan_api = conan_api self._remotes_file = HomePaths(self.conan_api.cache_folder).remotes_path + # Wraps an http_requester to inject proxies, certs, etc + self._requester = ConanRequester(self.conan_api.config.global_conf, self.conan_api.cache_folder) def list(self, pattern=None, only_enabled=True): """ diff --git a/conan/internal/conan_app.py b/conan/internal/conan_app.py index 0cc07f107c5..730732f6157 100644 --- a/conan/internal/conan_app.py +++ b/conan/internal/conan_app.py @@ -44,9 +44,8 @@ def __init__(self, conan_api): self.cache = PkgCache(self.cache_folder, global_conf) home_paths = HomePaths(self.cache_folder) - - # Wraps an http_requester to inject proxies, certs, etc - self.requester = ConanRequester(global_conf, cache_folder) + # FIXME: ugly private access + self.requester = conan_api.remotes._requester # To handle remote connections # Wraps RestApiClient to add authentication support (same interface) self.localdb = LocalDB(cache_folder) diff --git a/conan/test/utils/tools.py b/conan/test/utils/tools.py index 479e578690b..bb489608c48 100644 --- a/conan/test/utils/tools.py +++ b/conan/test/utils/tools.py @@ -275,6 +275,16 @@ def _set_auth_headers(kwargs): kwargs["headers"] = {} kwargs["headers"].update(mock_request.headers) + def mount(self, *args, **kwargs): + pass + + def Session(self): + return self + + @property + def codes(self): + return requests.codes + class TestServer(object): def __init__(self, read_permissions=None, @@ -531,8 +541,7 @@ def chdir(self, newdir): def mocked_servers(self, requester=None): _req = requester or TestRequester(self.servers) with mock.patch("conans.client.rest.conan_requester.requests", _req): - with mock.patch("conans.client.rest.conan_requester.ConanRequester._http_requester_imp", _req): - yield + yield @contextmanager def mocked_io(self): diff --git a/conans/client/rest/conan_requester.py b/conans/client/rest/conan_requester.py index 65d3623194d..e020f24c94d 100644 --- a/conans/client/rest/conan_requester.py +++ b/conans/client/rest/conan_requester.py @@ -97,12 +97,14 @@ def add_auth(self, url, kwargs): class ConanRequester: - _http_requester_imp = None - _adapter = None def __init__(self, config, cache_folder=None): self._url_creds = _SourceURLCredentials(cache_folder) self._max_retries = config.get("core.net.http:max_retries", default=2, check_type=int) + self._http_requester = requests.Session() + _adapter = HTTPAdapter(max_retries=self._get_retries()) + self._http_requester.mount("http://", _adapter) + self._http_requester.mount("https://", _adapter) self._timeout = config.get("core.net.http:timeout", default=DEFAULT_TIMEOUT) self._no_proxy_match = config.get("core.net.http:no_proxy_match", check_type=list) self._proxies = config.get("core.net.http:proxies") @@ -115,19 +117,6 @@ def __init__(self, config, cache_folder=None): platform.machine()]) self._user_agent = "Conan/%s (%s)" % (__version__, platform_info) - @property - def _http_requester(self): - if ConanRequester._http_requester_imp is None: - # FIXME: Trick for testing when requests is mocked - if hasattr(requests, "Session"): - ConanRequester._http_requester_imp = requests.Session() - ConanRequester._adapter = HTTPAdapter(max_retries=self._get_retries()) - ConanRequester._http_requester_imp.mount("http://", ConanRequester._adapter) - ConanRequester._http_requester_imp.mount("https://", ConanRequester._adapter) - else: - ConanRequester._http_requester_imp = requests - return ConanRequester._http_requester_imp - def _get_retries(self): retry = self._max_retries if retry == 0: diff --git a/test/integration/command/remote_verify_ssl_test.py b/test/integration/command/remote_verify_ssl_test.py index 8ad93e8842c..0d872d53cc9 100644 --- a/test/integration/command/remote_verify_ssl_test.py +++ b/test/integration/command/remote_verify_ssl_test.py @@ -1,5 +1,6 @@ import unittest +import requests from requests.models import Response from conan.test.utils.tools import TestClient @@ -19,6 +20,15 @@ def get(self, url, *args, **kwargs): assert "cacert.pem" in kwargs["verify"], "TEST FAILURE: cacert.pem not in verify kwarg" return resp + def Session(self): + return self + + @property + def codes(self): + return requests.codes + + def mount(self, *args, **kwargs): + pass class RequesterMockFalse(object): @@ -29,6 +39,16 @@ def get(self, url, *args, **kwargs): assert kwargs["verify"] is False, "TEST FAILURE: verify arg is not False" return resp + def Session(self): + return self + + @property + def codes(self): + return requests.codes + + def mount(self, *args, **kwargs): + pass + class VerifySSLTest(unittest.TestCase): diff --git a/test/integration/command/remove_test.py b/test/integration/command/remove_test.py index c8b7bfee474..9fc8589e68b 100644 --- a/test/integration/command/remove_test.py +++ b/test/integration/command/remove_test.py @@ -352,17 +352,17 @@ def test_package_query_no_package_ref(populated_client): def _get_all_recipes(client, with_remote): - api = ConanAPI(client.cache_folder) - remote = api.remotes.get("default") if with_remote else None with client.mocked_servers(): + api = ConanAPI(client.cache_folder) + remote = api.remotes.get("default") if with_remote else None return set([r.repr_notime() for r in api.search.recipes("*", remote=remote)]) def _get_all_packages(client, ref, with_remote): ref = RecipeReference.loads(ref) - api = ConanAPI(client.cache_folder) - remote = api.remotes.get("default") if with_remote else None with client.mocked_servers(): + api = ConanAPI(client.cache_folder) + remote = api.remotes.get("default") if with_remote else None try: return set([r.repr_notime() for r in api.list.packages_configurations(ref, remote=remote)]) except NotFoundException: @@ -371,9 +371,9 @@ def _get_all_packages(client, ref, with_remote): def _get_revisions_recipes(client, ref, with_remote): ref = RecipeReference.loads(ref) - api = ConanAPI(client.cache_folder) - remote = api.remotes.get("default") if with_remote else None with client.mocked_servers(): + api = ConanAPI(client.cache_folder) + remote = api.remotes.get("default") if with_remote else None try: return set([r.repr_notime() for r in api.list.recipe_revisions(ref, remote=remote)]) except NotFoundException: @@ -382,9 +382,9 @@ def _get_revisions_recipes(client, ref, with_remote): def _get_revisions_packages(client, pref, with_remote): pref = PkgReference.loads(pref) - api = ConanAPI(client.cache_folder) - remote = api.remotes.get("default") if with_remote else None with client.mocked_servers(): + api = ConanAPI(client.cache_folder) + remote = api.remotes.get("default") if with_remote else None try: return set([r.repr_notime() for r in api.list.package_revisions(pref, remote=remote)]) except NotFoundException: diff --git a/test/integration/conan_api/search_test.py b/test/integration/conan_api/search_test.py index 2d8c8c83dfc..1aaa07f0a20 100644 --- a/test/integration/conan_api/search_test.py +++ b/test/integration/conan_api/search_test.py @@ -24,11 +24,11 @@ def test_search_recipes(remote_name): client.run("upload * -r=default -c") - # Search all the recipes locally and in the remote - api = ConanAPI(client.cache_folder) - remote = api.remotes.get(remote_name) if remote_name else None - with client.mocked_servers(): + # Search all the recipes locally and in the remote + api = ConanAPI(client.cache_folder) + remote = api.remotes.get(remote_name) if remote_name else None + sot = api.search.recipes(query="f*", remote=remote) assert sot == [RecipeReference.loads("felipe/1.0"), RecipeReference.loads("felipe/2.0"), diff --git a/test/integration/configuration/requester_test.py b/test/integration/configuration/requester_test.py index 81d538396bd..5acb473602f 100644 --- a/test/integration/configuration/requester_test.py +++ b/test/integration/configuration/requester_test.py @@ -49,12 +49,11 @@ class ConanRequesterHeadersTests(unittest.TestCase): def test_user_agent(self): mock_http_requester = MagicMock() with mock.patch("conans.client.rest.conan_requester.requests", mock_http_requester): - with mock.patch("conans.client.rest.conan_requester.ConanRequester._http_requester_imp", mock_http_requester): - requester = ConanRequester(ConfDefinition()) - requester.get(url="aaa") - headers = requester._http_requester.get.call_args[1]["headers"] - self.assertIn("Conan/%s" % __version__, headers["User-Agent"]) - - requester.get(url="aaa", headers={"User-Agent": "MyUserAgent"}) - headers = requester._http_requester.get.call_args[1]["headers"] - self.assertEqual("MyUserAgent", headers["User-Agent"]) + requester = ConanRequester(ConfDefinition()) + requester.get(url="aaa") + headers = requester._http_requester.get.call_args[1]["headers"] + self.assertIn("Conan/%s" % __version__, headers["User-Agent"]) + + requester.get(url="aaa", headers={"User-Agent": "MyUserAgent"}) + headers = requester._http_requester.get.call_args[1]["headers"] + self.assertEqual("MyUserAgent", headers["User-Agent"]) diff --git a/test/integration/remote/rest_api_test.py b/test/integration/remote/rest_api_test.py index e705891a96e..8a220a5b2bc 100644 --- a/test/integration/remote/rest_api_test.py +++ b/test/integration/remote/rest_api_test.py @@ -43,7 +43,6 @@ def setUpClass(cls): save(filename, "") config = ConfDefinition() requester = ConanRequester(config) - ConanRequester._http_requester_imp = TestRequester({"myremote", cls.server},) localdb = LocalDBMock() mocked_user_input = UserInput(non_interactive=False) @@ -62,7 +61,6 @@ def setUpClass(cls): @classmethod def tearDownClass(cls): - ConanRequester._http_requester_imp = None cls.server.stop() def tearDown(self): From 44f90cb911c14d5e60b28d6fdf17b573af952323 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Abril=20Rinc=C3=B3n=20Blanco?= Date: Mon, 23 Dec 2024 10:19:36 +0100 Subject: [PATCH 08/10] Reinit is not part of this PR --- conan/api/conan_api.py | 3 --- test/integration/remote/rest_api_test.py | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/conan/api/conan_api.py b/conan/api/conan_api.py index 3d806d32f68..d63845c4269 100644 --- a/conan/api/conan_api.py +++ b/conan/api/conan_api.py @@ -40,10 +40,7 @@ def __init__(self, cache_folder=None): # Migration system migrator = ClientMigrator(self.cache_folder, conan_version) migrator.migrate() - self.reinit() - - def reinit(self): self.config = ConfigAPI(self) self.remotes = RemotesAPI(self) self.command = CommandAPI(self) diff --git a/test/integration/remote/rest_api_test.py b/test/integration/remote/rest_api_test.py index 8a220a5b2bc..56855079674 100644 --- a/test/integration/remote/rest_api_test.py +++ b/test/integration/remote/rest_api_test.py @@ -18,7 +18,7 @@ from conan.test.utils.mocks import LocalDBMock from conan.test.utils.server_launcher import TestServerLauncher from conan.test.utils.test_files import temp_folder -from conan.test.utils.tools import get_free_port, TestRequester +from conan.test.utils.tools import get_free_port from conans.util.files import md5, save From e9ac51a6bac650f1b35e689bba25718e79687d39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Abril=20Rinc=C3=B3n=20Blanco?= Date: Mon, 23 Dec 2024 12:24:56 +0100 Subject: [PATCH 09/10] Refactor request usage out of app --- conan/api/subapi/config.py | 9 ++++++--- conan/api/subapi/remotes.py | 4 ++++ conan/api/subapi/upload.py | 5 ++--- conan/internal/api/config/config_installer.py | 4 +--- conan/internal/conan_app.py | 9 +++------ 5 files changed, 16 insertions(+), 15 deletions(-) diff --git a/conan/api/subapi/config.py b/conan/api/subapi/config.py index e1e5d9d3a93..04155f2ecfe 100644 --- a/conan/api/subapi/config.py +++ b/conan/api/subapi/config.py @@ -36,8 +36,9 @@ def install(self, path_or_url, verify_ssl, config_type=None, args=None, source_folder=None, target_folder=None): # TODO: We probably want to split this into git-folder-http cases? from conan.internal.api.config.config_installer import configuration_install - app = ConanApp(self.conan_api) - configuration_install(app, path_or_url, verify_ssl, config_type=config_type, args=args, + cache_folder = self.conan_api.cache_folder + requester = self.conan_api.remotes.requester + configuration_install(cache_folder, requester, path_or_url, verify_ssl, config_type=config_type, args=args, source_folder=source_folder, target_folder=target_folder) def install_pkg(self, ref, lockfile=None, force=False, remotes=None, profile=None): @@ -90,7 +91,9 @@ def install_pkg(self, ref, lockfile=None, force=False, remotes=None, profile=Non return pkg.pref # Already installed, we can skip repeating the install from conan.internal.api.config.config_installer import configuration_install - configuration_install(app, uri=pkg.conanfile.package_folder, verify_ssl=False, + cache_folder = self.conan_api.cache_folder + requester = self.conan_api.remotes.requester + configuration_install(cache_folder, requester, uri=pkg.conanfile.package_folder, verify_ssl=False, config_type="dir", ignore=["conaninfo.txt", "conanmanifest.txt"]) # We save the current package full reference in the file for future # And for ``package_id`` computation diff --git a/conan/api/subapi/remotes.py b/conan/api/subapi/remotes.py index 68ee8b9e40b..bac4f7e1bf0 100644 --- a/conan/api/subapi/remotes.py +++ b/conan/api/subapi/remotes.py @@ -256,6 +256,10 @@ def user_auth(self, remote: Remote, with_user=False, force=False): user, token, _ = localdb.get_login(remote.url) return user + @property + def requester(self): + return self._requester + def _load(remotes_file): if not os.path.exists(remotes_file): diff --git a/conan/api/subapi/upload.py b/conan/api/subapi/upload.py index 0522096e697..4f038cc9fb0 100644 --- a/conan/api/subapi/upload.py +++ b/conan/api/subapi/upload.py @@ -113,9 +113,8 @@ def upload_backup_sources(self, files): output.info("No backup sources files to upload") return files - app = ConanApp(self.conan_api) - # TODO: verify might need a config to force it to False - uploader = FileUploader(app.requester, verify=True, config=config, source_credentials=True) + requester = self.conan_api.remotes.requester + uploader = FileUploader(requester, verify=True, config=config, source_credentials=True) # TODO: For Artifactory, we can list all files once and check from there instead # of 1 request per file, but this is more general for file in files: diff --git a/conan/internal/api/config/config_installer.py b/conan/internal/api/config/config_installer.py index d31569e90d3..e97426b293a 100644 --- a/conan/internal/api/config/config_installer.py +++ b/conan/internal/api/config/config_installer.py @@ -193,10 +193,8 @@ def _is_compressed_file(filename): return any(filename.endswith(e) for e in tgz_exts) -def configuration_install(app, uri, verify_ssl, config_type=None, +def configuration_install(cache_folder, requester, uri, verify_ssl, config_type=None, args=None, source_folder=None, target_folder=None, ignore=None): - requester = app.requester - cache_folder = app.cache_folder config = _ConfigOrigin(uri, config_type, verify_ssl, args, source_folder, target_folder) try: if config.type == "git": diff --git a/conan/internal/conan_app.py b/conan/internal/conan_app.py index 0a67ee40113..1d9a8540f81 100644 --- a/conan/internal/conan_app.py +++ b/conan/internal/conan_app.py @@ -46,12 +46,9 @@ def __init__(self, conan_api): cache_folder = conan_api.home_folder self.cache_folder = cache_folder self.cache = PkgCache(self.cache_folder, global_conf) - # FIXME: ugly private access - self.requester = conan_api.remotes._requester - # To handle remote connections # Wraps RestApiClient to add authentication support (same interface) localdb = LocalDB(cache_folder) - auth_manager = ConanApiAuthManager(self.requester, cache_folder, localdb, global_conf) + auth_manager = ConanApiAuthManager(conan_api.remotes.requester, cache_folder, localdb, global_conf) # Handle remote connections self.remote_manager = RemoteManager(self.cache, auth_manager, cache_folder) global_editables = conan_api.local.editable_packages @@ -70,6 +67,6 @@ def __init__(self, conan_api): self.pyreq_loader = PyRequireLoader(self, self.global_conf) cmd_wrap = CmdWrapper(HomePaths(self.cache_folder).wrapper_path) - conanfile_helpers = ConanFileHelpers(self.requester, cmd_wrap, self.global_conf, self.cache, - self.cache_folder) + conanfile_helpers = ConanFileHelpers(conan_api.remotes.requester, cmd_wrap, self.global_conf, + self.cache, self.cache_folder) self.loader = ConanFileLoader(self.pyreq_loader, conanfile_helpers) From 8600bc9606b2710a0c8e4c5d3c739e69cbf9b848 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Abril=20Rinc=C3=B3n=20Blanco?= Date: Thu, 9 Jan 2025 14:05:08 +0100 Subject: [PATCH 10/10] Make max_retries a local variable --- conans/client/rest/conan_requester.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/conans/client/rest/conan_requester.py b/conans/client/rest/conan_requester.py index c11dda72a36..ffc0fab7c33 100644 --- a/conans/client/rest/conan_requester.py +++ b/conans/client/rest/conan_requester.py @@ -101,9 +101,9 @@ class ConanRequester: def __init__(self, config, cache_folder=None): self._url_creds = _SourceURLCredentials(cache_folder) - self._max_retries = config.get("core.net.http:max_retries", default=2, check_type=int) + _max_retries = config.get("core.net.http:max_retries", default=2, check_type=int) self._http_requester = requests.Session() - _adapter = HTTPAdapter(max_retries=self._get_retries()) + _adapter = HTTPAdapter(max_retries=self._get_retries(_max_retries)) self._http_requester.mount("http://", _adapter) self._http_requester.mount("https://", _adapter) self._timeout = config.get("core.net.http:timeout", default=DEFAULT_TIMEOUT) @@ -118,8 +118,9 @@ def __init__(self, config, cache_folder=None): platform.machine()]) self._user_agent = "Conan/%s (%s)" % (__version__, platform_info) - def _get_retries(self): - retry = self._max_retries + @staticmethod + def _get_retries(max_retries): + retry = max_retries if retry == 0: return 0 retry_status_code_set = {