From 7bf06f7c487da957fbce878504e1a3c85218167b Mon Sep 17 00:00:00 2001 From: Zach Howell Date: Fri, 22 Nov 2024 09:55:02 -0800 Subject: [PATCH] Use -m flag to run gsutil cp in parallel for longer copies Got this helpful message when running a copy command that timed out: ``` NOTE: You are performing a sequence of gsutil operations that may run significantly faster if you instead use gsutil -m cp ... Please see the -m section under "gsutil help options" for further information about when gsutil -m can be advantageous. ``` PiperOrigin-RevId: 699205081 --- perfkitbenchmarker/providers/gcp/gcs.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/perfkitbenchmarker/providers/gcp/gcs.py b/perfkitbenchmarker/providers/gcp/gcs.py index ce72fb5ad..d67e16085 100644 --- a/perfkitbenchmarker/providers/gcp/gcs.py +++ b/perfkitbenchmarker/providers/gcp/gcs.py @@ -114,6 +114,9 @@ def Copy(self, src_url, dst_url, recursive=False, timeout=None): timeout: The timeout for the copy command. """ cmd = ['gsutil', 'cp'] + if recursive or timeout is not None: + # -m runs in parallel, which is faster. + cmd = ['gsutil', '-m', 'cp'] if recursive: cmd += ['-r'] cmd += [src_url, dst_url]