From c661e41964fb66ca1b8a5bc68d6d8d42d8e65ef1 Mon Sep 17 00:00:00 2001 From: Maciej Urbanski Date: Sat, 24 Feb 2024 17:11:28 +0100 Subject: [PATCH 1/2] fix upload threads setting --- b2/_internal/console_tool.py | 5 +-- changelog.d/+upload_threads.fixed.md | 1 + pyproject.toml | 2 +- test/unit/console_tool/test_download_file.py | 16 ++++++++ test/unit/console_tool/test_upload_file.py | 31 ++++++++++++++++ test/unit/test_console_tool.py | 39 +------------------- 6 files changed, 51 insertions(+), 43 deletions(-) create mode 100644 changelog.d/+upload_threads.fixed.md diff --git a/b2/_internal/console_tool.py b/b2/_internal/console_tool.py index ba51cda8e..073fb1a95 100644 --- a/b2/_internal/console_tool.py +++ b/b2/_internal/console_tool.py @@ -734,9 +734,8 @@ def _get_threads_from_args(self, args) -> int: def _set_threads_from_args(self, args): threads = self._get_threads_from_args(args) - # FIXME: This is using deprecated API. It should be be replaced when moving to b2sdk apiver 3. - # There is `max_download_workers` param in B2Api constructor for this. self.api.services.download_manager.set_thread_pool_size(threads) + self.api.services.upload_manager.set_thread_pool_size(threads) class _TqdmCloser: @@ -2811,8 +2810,6 @@ def _run(self, args): upload_threads = args.uploadThreads download_threads = args.downloadThreads - # FIXME: This is using deprecated API. It should be replaced when moving to b2sdk apiver 3. - # There are `max_X_workers` params in B2Api constructor for this. self.api.services.upload_manager.set_thread_pool_size(upload_threads) self.api.services.download_manager.set_thread_pool_size(download_threads) diff --git a/changelog.d/+upload_threads.fixed.md b/changelog.d/+upload_threads.fixed.md new file mode 100644 index 000000000..2187578fc --- /dev/null +++ b/changelog.d/+upload_threads.fixed.md @@ -0,0 +1 @@ +Fix `--threads` option being silently ignored in upload commands. diff --git a/pyproject.toml b/pyproject.toml index 1aeb20707..81e92c83f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ classifiers = [ dependencies = [ "argcomplete>=2,<4", "arrow>=1.0.2,<2.0.0", - "b2sdk>=1.31.0,<2", + "b2sdk>=1.32.0,<2", "docutils>=0.18.1", "idna~=3.4; platform_system == 'Java'", "importlib-metadata>=3.3; python_version < '3.8'", diff --git a/test/unit/console_tool/test_download_file.py b/test/unit/console_tool/test_download_file.py index a2a82f80c..58098fa2b 100644 --- a/test/unit/console_tool/test_download_file.py +++ b/test/unit/console_tool/test_download_file.py @@ -190,3 +190,19 @@ def test_cat__b2id_uri(b2_cli, bucket, uploaded_stdout_txt, tmp_path, capfd): """Test download_file_by_name stdout alias support""" b2_cli.run(['cat', '--noProgress', "b2id://9999"],) assert capfd.readouterr().out == uploaded_stdout_txt['content'] + + +def test__download_file__threads(b2_cli, local_file, uploaded_file, tmp_path): + num_threads = 13 + output_path = tmp_path / 'output.txt' + + b2_cli.run( + [ + 'download-file', '--noProgress', '--threads', + str(num_threads), 'b2://my-bucket/file1.txt', + str(output_path) + ] + ) + + assert output_path.read_text() == uploaded_file['content'] + assert b2_cli.b2_api.services.download_manager.get_thread_pool_size() == num_threads diff --git a/test/unit/console_tool/test_upload_file.py b/test/unit/console_tool/test_upload_file.py index 095b1a6bc..9a9795b04 100644 --- a/test/unit/console_tool/test_upload_file.py +++ b/test/unit/console_tool/test_upload_file.py @@ -125,3 +125,34 @@ def test_upload_file__stdin(b2_cli, bucket, tmpdir, mock_stdin): remove_version=True, expected_part_of_stdout=expected_stdout, ) + + +def test_upload_file__threads_setting(b2_cli, bucket, tmp_path): + """Test upload_file supports setting number of threads""" + num_threads = 66 + filename = 'file1.txt' + content = 'hello world' + local_file1 = tmp_path / 'file1.txt' + local_file1.write_text(content) + + expected_json = { + "action": "upload", + "contentSha1": "2aae6c35c94fcfb415dbe95f408b9ce91ee846ed", + "fileInfo": { + "src_last_modified_millis": f"{local_file1.stat().st_mtime_ns // 1000000}" + }, + "fileName": filename, + "size": len(content), + } + + b2_cli.run( + [ + 'upload-file', '--noProgress', 'my-bucket', '--threads', + str(num_threads), + str(local_file1), 'file1.txt' + ], + expected_json_in_stdout=expected_json, + remove_version=True, + ) + + assert b2_cli.b2_api.services.upload_manager.get_thread_pool_size() == num_threads diff --git a/test/unit/test_console_tool.py b/test/unit/test_console_tool.py index 96fa13986..8a2f8bfa7 100644 --- a/test/unit/test_console_tool.py +++ b/test/unit/test_console_tool.py @@ -242,7 +242,7 @@ def _run_command( self.assertNotIn(unexpected_part_of_stdout, actual_stdout) if expected_stderr is not None: self.assertEqual(expected_stderr, actual_stderr, 'stderr') - self.assertEqual(expected_status, actual_status, 'exit status code') + assert expected_status == actual_status, 'exit status code' return actual_status, actual_stdout, actual_stderr @classmethod @@ -1094,43 +1094,6 @@ def test_files_encrypted(self): expected_json_in_stdout=expected_json, ) - def _test_download_threads(self, download_by, num_threads): - self._authorize_account() - self._create_my_bucket() - - with TempDir() as temp_dir: - local_file = self._make_local_file(temp_dir, 'file.txt') - self._run_command( - ['upload-file', '--noProgress', 'my-bucket', local_file, 'file.txt'], - remove_version=True, - ) - - command = [ - 'download-file-by-%s' % download_by, '--noProgress', '--threads', - str(num_threads) - ] - command += ['9999'] if download_by == 'id' else ['my-bucket', 'file.txt'] - local_download = os.path.join(temp_dir, 'download.txt') - command += [local_download] - self._run_command( - command, - expected_stderr= - f'WARNING: download-file-by-{download_by} command is deprecated. Use download-file instead.\n' - ) - self.assertEqual(b'hello world', self._read_file(local_download)) - - def test_download_by_id_1_thread(self): - self._test_download_threads(download_by='id', num_threads=1) - - def test_download_by_id_10_threads(self): - self._test_download_threads(download_by='id', num_threads=10) - - def test_download_by_name_1_thread(self): - self._test_download_threads(download_by='name', num_threads=1) - - def test_download_by_name_10_threads(self): - self._test_download_threads(download_by='name', num_threads=10) - def _test_download_to_directory(self, download_by: str): self._authorize_account() self._create_my_bucket() From c7df206eb4512a04c9f5d23ed2a152f76124d99d Mon Sep 17 00:00:00 2001 From: Maciej Urbanski Date: Mon, 26 Feb 2024 13:06:40 +0100 Subject: [PATCH 2/2] lockfile update --- pdm.lock | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/pdm.lock b/pdm.lock index 469d10239..2a3dee65a 100644 --- a/pdm.lock +++ b/pdm.lock @@ -5,7 +5,7 @@ groups = ["default", "bundle", "doc", "format", "full", "license", "lint", "release", "test"] strategy = ["cross_platform", "inherit_metadata"] lock_version = "4.4.1" -content_hash = "sha256:4b5efeebe5af9b5151fcb7c33e86b4553b20dfdfc087cc03d9f03bade558a57a" +content_hash = "sha256:5c7675e01cd16ace6bc865250a99cf865ceca49d083356207cb0aa319f0b6977" [[package]] name = "alabaster" @@ -100,7 +100,7 @@ files = [ [[package]] name = "b2sdk" -version = "1.31.0" +version = "1.32.0" requires_python = ">=3.7" summary = "Backblaze B2 SDK" groups = ["default"] @@ -114,8 +114,8 @@ dependencies = [ "typing-extensions>=4.7.1; python_version < \"3.12\"", ] files = [ - {file = "b2sdk-1.31.0-py3-none-any.whl", hash = "sha256:75509102915ec0f7cfe42a08c40a9e6a2737ca50700b4b4cacf5fb1c115bfb76"}, - {file = "b2sdk-1.31.0.tar.gz", hash = "sha256:3e0e4a78e273ee4cafd622a849c1c20ba75900f4f66a252b0cc7174792c21d2d"}, + {file = "b2sdk-1.32.0-py3-none-any.whl", hash = "sha256:84b6f7f01b6fc74f982944d40260df695f1fdeae9fcee0b49afd053a7ad75501"}, + {file = "b2sdk-1.32.0.tar.gz", hash = "sha256:46321bccdd9a295af52cb05d98b5df9771bddf8dcdf68afe3a9fdf7dda156b32"}, ] [[package]] @@ -822,14 +822,14 @@ files = [ [[package]] name = "pipdeptree" -version = "2.14.0" +version = "2.15.1" requires_python = ">=3.8" summary = "Command line utility to show dependency tree of packages." groups = ["license"] marker = "python_version >= \"3.9\"" files = [ - {file = "pipdeptree-2.14.0-py3-none-any.whl", hash = "sha256:34666ca78f65414e1936492ebbabf3201460c85af5c31ab3b240ef5105f94f89"}, - {file = "pipdeptree-2.14.0.tar.gz", hash = "sha256:3296195250e00d37638f2cce70495e3345645b4bbecc1c38ac39339f1511d9b5"}, + {file = "pipdeptree-2.15.1-py3-none-any.whl", hash = "sha256:0abe6b5b9f86a79ea023a633d7526de14b64b848bbdf980c610d08d679c8329d"}, + {file = "pipdeptree-2.15.1.tar.gz", hash = "sha256:e4ae44fc9d0c747125316981804d7786e9d97bf59c128249b8ea272a7cbdecc4"}, ] [[package]] @@ -862,7 +862,7 @@ files = [ [[package]] name = "prettytable" -version = "3.9.0" +version = "3.10.0" requires_python = ">=3.8" summary = "A simple Python library for easily displaying tabular data in a visually appealing ASCII table format" groups = ["license"] @@ -871,8 +871,8 @@ dependencies = [ "wcwidth", ] files = [ - {file = "prettytable-3.9.0-py3-none-any.whl", hash = "sha256:a71292ab7769a5de274b146b276ce938786f56c31cf7cea88b6f3775d82fe8c8"}, - {file = "prettytable-3.9.0.tar.gz", hash = "sha256:f4ed94803c23073a90620b201965e5dc0bccf1760b7a7eaf3158cab8aaffdf34"}, + {file = "prettytable-3.10.0-py3-none-any.whl", hash = "sha256:6536efaf0757fdaa7d22e78b3aac3b69ea1b7200538c2c6995d649365bddab92"}, + {file = "prettytable-3.10.0.tar.gz", hash = "sha256:9665594d137fb08a1117518c25551e0ede1687197cf353a4fdc78d27e1073568"}, ] [[package]] @@ -1450,7 +1450,7 @@ files = [ [[package]] name = "sphinxcontrib-plantuml" -version = "0.27" +version = "0.28" summary = "Sphinx \"plantuml\" extension" groups = ["doc"] marker = "python_version >= \"3.9\"" @@ -1458,7 +1458,7 @@ dependencies = [ "Sphinx>=1.6", ] files = [ - {file = "sphinxcontrib-plantuml-0.27.tar.gz", hash = "sha256:8975778eed9c142a42ecdc4ecd83132dec0690c035cae788c123750ead526703"}, + {file = "sphinxcontrib-plantuml-0.28.tar.gz", hash = "sha256:ce9362ad3d4bbcc4c30dfdfc308aacad2c07b344635316681a711269b8e00bea"}, ] [[package]]