From 394868e9dad401d834e69d7c8aa05a3a08b674f0 Mon Sep 17 00:00:00 2001 From: Evgeniy Zayats Date: Tue, 22 Oct 2024 18:38:41 -0400 Subject: [PATCH] env: lock binaries downloads Signed-off-by: Evgeniy Zayats --- neofs-testlib/neofs_testlib/env/env.py | 99 ++++++++++++++------------ pytest_tests/lib/helpers/common.py | 1 + 2 files changed, 53 insertions(+), 47 deletions(-) diff --git a/neofs-testlib/neofs_testlib/env/env.py b/neofs-testlib/neofs_testlib/env/env.py index 38d4f23d8..b1c36801d 100644 --- a/neofs-testlib/neofs_testlib/env/env.py +++ b/neofs-testlib/neofs_testlib/env/env.py @@ -27,7 +27,7 @@ import jinja2 import requests import yaml -from helpers.common import ALLOCATED_PORTS_FILE, ALLOCATED_PORTS_LOCK_FILE, get_assets_dir_path +from helpers.common import ALLOCATED_PORTS_FILE, ALLOCATED_PORTS_LOCK_FILE, BINARY_DOWNLOADS_LOCK_FILE, get_assets_dir_path from neofs_testlib.cli import NeofsAdm, NeofsCli, NeofsLens, NeoGo from neofs_testlib.shell import LocalShell @@ -371,52 +371,57 @@ def log_versions_to_allure(self): @allure.step("Download binaries") def download_binaries(self): - logger.info("Going to download missing binaries and contracts, if needed") - deploy_threads = [] - - binaries = [ - (self.neofs_adm_path, "neofs_adm"), - (self.neofs_cli_path, "neofs_cli"), - (self.neofs_lens_path, "neofs_lens"), - (self.neo_go_path, "neo_go"), - (self.neofs_ir_path, "neofs_ir"), - (self.neofs_node_path, "neofs_node"), - (self.neofs_s3_authmate_path, "neofs_s3_authmate"), - (self.neofs_s3_gw_path, "neofs_s3_gw"), - (self.neofs_rest_gw_path, "neofs_rest_gw"), - (self.neofs_contract_dir, "neofs_contract"), - (self.warp_path, "warp"), - ] - - for binary in binaries: - binary_path, binary_name = binary - if not os.path.isfile(binary_path) and not os.path.isdir(binary_path): - neofs_binary_params = self.neofs_env_config["binaries"][binary_name] - allure_step_name = "Downloading " - allure_step_name += f" {neofs_binary_params['repo']}/" - allure_step_name += f"{neofs_binary_params['version']}/" - allure_step_name += f"{neofs_binary_params['file']}" - with allure.step(allure_step_name): - deploy_threads.append( - threading.Thread( - target=NeoFSEnv.download_binary, - args=( - neofs_binary_params["repo"], - neofs_binary_params["version"], - neofs_binary_params["file"], - binary_path, - ), - ) - ) - else: - logger.info(f"'{binary_path}' already exists, will not be downloaded") - - if len(deploy_threads) > 0: - for t in deploy_threads: - t.start() - logger.info("Wait until all binaries are downloaded") - for t in deploy_threads: - t.join() + with open(BINARY_DOWNLOADS_LOCK_FILE, "w") as lock_file: + fcntl.flock(lock_file, fcntl.LOCK_EX) + try: + logger.info("Going to download missing binaries and contracts, if needed") + deploy_threads = [] + + binaries = [ + (self.neofs_adm_path, "neofs_adm"), + (self.neofs_cli_path, "neofs_cli"), + (self.neofs_lens_path, "neofs_lens"), + (self.neo_go_path, "neo_go"), + (self.neofs_ir_path, "neofs_ir"), + (self.neofs_node_path, "neofs_node"), + (self.neofs_s3_authmate_path, "neofs_s3_authmate"), + (self.neofs_s3_gw_path, "neofs_s3_gw"), + (self.neofs_rest_gw_path, "neofs_rest_gw"), + (self.neofs_contract_dir, "neofs_contract"), + (self.warp_path, "warp"), + ] + + for binary in binaries: + binary_path, binary_name = binary + if not os.path.isfile(binary_path) and not os.path.isdir(binary_path): + neofs_binary_params = self.neofs_env_config["binaries"][binary_name] + allure_step_name = "Downloading " + allure_step_name += f" {neofs_binary_params['repo']}/" + allure_step_name += f"{neofs_binary_params['version']}/" + allure_step_name += f"{neofs_binary_params['file']}" + with allure.step(allure_step_name): + deploy_threads.append( + threading.Thread( + target=NeoFSEnv.download_binary, + args=( + neofs_binary_params["repo"], + neofs_binary_params["version"], + neofs_binary_params["file"], + binary_path, + ), + ) + ) + else: + logger.info(f"'{binary_path}' already exists, will not be downloaded") + + if len(deploy_threads) > 0: + for t in deploy_threads: + t.start() + logger.info("Wait until all binaries are downloaded") + for t in deploy_threads: + t.join() + finally: + fcntl.flock(lock_file, fcntl.LOCK_UN) def _is_binary_compatible(self, expected_platform: str = None, expected_arch: str = None) -> bool: if expected_platform is None or expected_arch is None: diff --git a/pytest_tests/lib/helpers/common.py b/pytest_tests/lib/helpers/common.py index 9744e7cbf..27834fff6 100644 --- a/pytest_tests/lib/helpers/common.py +++ b/pytest_tests/lib/helpers/common.py @@ -61,6 +61,7 @@ ALLOCATED_PORTS_LOCK_FILE = "/tmp/allocated_ports.lock" ALLOCATED_PORTS_FILE = "/tmp/allocated_ports.txt" +BINARY_DOWNLOADS_LOCK_FILE = "/tmp/binary_downloads.lock" # Generate wallet configs # TODO: we should move all info about wallet configs to fixtures