From 1d8a7336fa3b02cd0c3ca302205ae5dadacba1bc Mon Sep 17 00:00:00 2001 From: Yi Yao Date: Mon, 23 Sep 2024 19:59:57 -0700 Subject: [PATCH 1/9] Fix typo in benchmark.yaml Signed-off-by: CharleneHu-42 --- evals/benchmark/benchmark.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/evals/benchmark/benchmark.yaml b/evals/benchmark/benchmark.yaml index 37e9e61f..453532e1 100644 --- a/evals/benchmark/benchmark.yaml +++ b/evals/benchmark/benchmark.yaml @@ -19,9 +19,9 @@ test_suite_config: # Overall configuration settings for the test suite load_shape: # Tenant concurrency pattern name: constant # poisson or constant(locust default load shape) params: # Loadshape-specific parameters - constant: # Poisson load shape specific parameters, activate only if load_shape is poisson + constant: # Constant load shape specific parameters, activate only if load_shape.name is constant concurrent_level: 4 # If user_queries is specified, concurrent_level is target number of requests per user. If not, it is the number of simulated users - poisson: # Poisson load shape specific parameters, activate only if load_shape is poisson + poisson: # Poisson load shape specific parameters, activate only if load_shape.name is poisson arrival-rate: 1.0 # Request arrival rate test_cases: From c5bf97410971824f4c23184fdac1e9f74a8ad53c Mon Sep 17 00:00:00 2001 From: Yi Yao Date: Tue, 24 Sep 2024 23:33:31 -0700 Subject: [PATCH 2/9] Add load shape related information into outputs. Signed-off-by: CharleneHu-42 --- evals/benchmark/stresscli/commands/load_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/evals/benchmark/stresscli/commands/load_test.py b/evals/benchmark/stresscli/commands/load_test.py index 4c015454..64851d0d 100644 --- a/evals/benchmark/stresscli/commands/load_test.py +++ b/evals/benchmark/stresscli/commands/load_test.py @@ -142,10 +142,10 @@ def run_locust_test(kubeconfig, global_settings, run_settings, output_folder, in load_shape_conf = run_settings.get("load-shape", global_settings.get("load-shape", locust_defaults["load-shape"])) try: load_shape = load_shape_conf["name"] + runspec["load-shape"] = load_shape_conf except KeyError: load_shape = DEFAULT_LOADSHAPE - runspec["load-shape"] = load_shape if load_shape == DEFAULT_LOADSHAPE: # constant load is Locust's default load shape, do nothing. console_logger.info("Use default load shape.") @@ -214,7 +214,7 @@ def run_locust_test(kubeconfig, global_settings, run_settings, output_folder, in "--run-time", runspec["runtime"], "--load-shape", - runspec["load-shape"], + load_shape, "--dataset", runspec["dataset"], "--seed", From ecf9d8785cd0143d36046b15bdd59407fdbf1ebb Mon Sep 17 00:00:00 2001 From: CharleneHu-42 Date: Fri, 11 Oct 2024 17:12:13 +0800 Subject: [PATCH 3/9] add new constant loader & fix poisson loader issue Signed-off-by: CharleneHu-42 --- evals/benchmark/README.md | 7 +-- evals/benchmark/benchmark.yaml | 3 +- .../benchmark/stresscli/commands/load_test.py | 29 ++++++----- evals/benchmark/stresscli/locust/aistress.py | 12 +++-- .../stresscli/locust/constant_load_shape.py | 51 +++++++++++++++++++ .../stresscli/locust/poisson_load_shape.py | 2 +- 6 files changed, 81 insertions(+), 23 deletions(-) create mode 100644 evals/benchmark/stresscli/locust/constant_load_shape.py diff --git a/evals/benchmark/README.md b/evals/benchmark/README.md index 726513e9..1c43921f 100644 --- a/evals/benchmark/README.md +++ b/evals/benchmark/README.md @@ -66,10 +66,11 @@ test_suite_config: load_shape: # Tenant concurrency pattern name: constant # poisson or constant(locust default load shape) params: # Loadshape-specific parameters - constant: # Poisson load shape specific parameters, activate only if load_shape is poisson + constant: # Constant load shape specific parameters, activate only if load_shape.name is constant concurrent_level: 4 # If user_queries is specified, concurrent_level is target number of requests per user. If not, it is the number of simulated users - poisson: # Poisson load shape specific parameters, activate only if load_shape is poisson - arrival-rate: 1.0 # Request arrival rate + # arrival_rate: 1.0 # Request arrival rate. If set, concurrent_level will be overrided, constant load will be generated based on arrival-rate + poisson: # Poisson load shape specific parameters, activate only if load_shape.name is poisson + arrival_rate: 1.0 # Request arrival rate warm_ups: 0 # Number of test requests for warm-ups run_time: 60m # Total runtime for the test suite seed: # The seed for all RNGs diff --git a/evals/benchmark/benchmark.yaml b/evals/benchmark/benchmark.yaml index 453532e1..3b464b1c 100644 --- a/evals/benchmark/benchmark.yaml +++ b/evals/benchmark/benchmark.yaml @@ -21,8 +21,9 @@ test_suite_config: # Overall configuration settings for the test suite params: # Loadshape-specific parameters constant: # Constant load shape specific parameters, activate only if load_shape.name is constant concurrent_level: 4 # If user_queries is specified, concurrent_level is target number of requests per user. If not, it is the number of simulated users + # arrival_rate: 1.0 # Request arrival rate. If set, concurrent_level will be overrided, constant load will be generated based on arrival-rate poisson: # Poisson load shape specific parameters, activate only if load_shape.name is poisson - arrival-rate: 1.0 # Request arrival rate + arrival_rate: 1.0 # Request arrival rate test_cases: chatqna: diff --git a/evals/benchmark/stresscli/commands/load_test.py b/evals/benchmark/stresscli/commands/load_test.py index 64851d0d..748347c0 100644 --- a/evals/benchmark/stresscli/commands/load_test.py +++ b/evals/benchmark/stresscli/commands/load_test.py @@ -146,7 +146,13 @@ def run_locust_test(kubeconfig, global_settings, run_settings, output_folder, in except KeyError: load_shape = DEFAULT_LOADSHAPE - if load_shape == DEFAULT_LOADSHAPE: + load_shape_params = None + try: + load_shape_params = load_shape_conf["params"][load_shape] + except KeyError: + console_logger.info(f"The specified load shape not found: {load_shape}") + + if load_shape == DEFAULT_LOADSHAPE and (load_shape_params is None or "arrival_rate" not in load_shape_params): # constant load is Locust's default load shape, do nothing. console_logger.info("Use default load shape.") else: @@ -155,7 +161,10 @@ def run_locust_test(kubeconfig, global_settings, run_settings, output_folder, in if os.path.isfile(f_custom_load_shape): # Add the locust file of the custom load shape into classpath runspec["locustfile"] += f",{f_custom_load_shape}" - console_logger.info("Use custom load shape: {load_shape}") + if load_shape == DEFAULT_LOADSHAPE: + console_logger.info("Use default load shape based on request arrival rate") + else: + console_logger.info("Use custom load shape: {load_shape}") else: console_logger.error( f"Unsupported load shape: {load_shape}." @@ -180,15 +189,9 @@ def run_locust_test(kubeconfig, global_settings, run_settings, output_folder, in spawn_rate = 100 if runspec["users"] > 100 else runspec["users"] - load_shape_params = None - try: - load_shape_params = load_shape_conf["params"][load_shape] - except KeyError: - console_logger.info(f"The specified load shape not found: {load_shape}") - # Dynamically allocate Locust processes to fit different loads processes = 2 - if load_shape == "constant": + if load_shape == "constant" and (load_shape_params is None or "arrival_rate" not in load_shape_params): if runspec["max_requests"] > 0: processes = 10 if runspec["max_requests"] > 2000 else 5 if runspec["max_requests"] > 1000 else processes else: @@ -197,11 +200,11 @@ def run_locust_test(kubeconfig, global_settings, run_settings, output_folder, in concurrent_level = int(load_shape_params["concurrent_level"]) processes = 10 if concurrent_level > 400 else 5 if concurrent_level > 200 else processes elif load_shape == "poisson": - if load_shape_params and "arrival-rate" in load_shape_params: - processes = max(2, math.ceil(int(load_shape_params["arrival-rate"]) / 5)) + if load_shape_params and "arrival_rate" in load_shape_params: + processes = max(2, math.ceil(int(load_shape_params["arrival_rate"]) / 5)) else: - if load_shape_params and "arrival-rate" in load_shape_params: - processes = max(2, math.ceil(int(load_shape_params["arrival-rate"]) / 5)) + if load_shape_params and "arrival_rate" in load_shape_params: + processes = max(2, math.ceil(int(load_shape_params["arrival_rate"]) / 5)) elif runspec["max_requests"] > 0: processes = 10 if runspec["max_requests"] > 2000 else 5 if runspec["max_requests"] > 1000 else processes diff --git a/evals/benchmark/stresscli/locust/aistress.py b/evals/benchmark/stresscli/locust/aistress.py index 52c030ba..89cc9538 100644 --- a/evals/benchmark/stresscli/locust/aistress.py +++ b/evals/benchmark/stresscli/locust/aistress.py @@ -87,9 +87,9 @@ def __init__(self, *args, **kwargs): def bench_main(self): max_request = self.environment.parsed_options.max_request if max_request >= 0 and AiStressUser.request >= max_request: - # For poisson load shape, a user only sends single request before it stops. + # For custom load shape based on arrival_rate, new users spawned after exceeding max_request is reached will be stopped. # TODO: user should not care about load shape - if self.environment.parsed_options.load_shape == "poisson": + if "arrival_rate" in self.environment.parsed_options: self.stop(force=True) time.sleep(1) @@ -186,10 +186,10 @@ def bench_main(self): self.environment.runner.stats.log_request("POST", url, time.perf_counter() - start_ts, 0) self.environment.runner.stats.log_error("POST", url, "Locust Request error") - # For poisson load shape, a user only sends single request before it stops. + # For custom load shape based on arrival_rate, a user only sends single request before it sleeps. # TODO: user should not care about load shape - if self.environment.parsed_options.load_shape == "poisson": - self.stop(force=True) + if "arrival_rate" in self.environment.parsed_options: + time.sleep(365 * 60 * 60) # def on_stop(self) -> None: @@ -255,6 +255,8 @@ def checker(environment): max_request = environment.parsed_options.max_request if max_request >= 0 and environment.runner.stats.num_requests >= max_request: logging.info(f"Exceed the max-request number:{environment.runner.stats.num_requests}, Exit...") + # Remove stop_timeout after test quit to avoid Locust user stop exception with custom load shape + environment.runner.environment.stop_timeout = 0.0 # while environment.runner.user_count > 0: time.sleep(5) environment.runner.quit() diff --git a/evals/benchmark/stresscli/locust/constant_load_shape.py b/evals/benchmark/stresscli/locust/constant_load_shape.py new file mode 100644 index 00000000..fc90d1d9 --- /dev/null +++ b/evals/benchmark/stresscli/locust/constant_load_shape.py @@ -0,0 +1,51 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +import logging + +import locust +from locust import LoadTestShape, events + +logger = logging.getLogger(__name__) + + +@events.init_command_line_parser.add_listener +def _(parser): + parser.add_argument( + "--arrival_rate", + type=float, + default=1.0, + ) + + +class ConstantRPSLoadShape(LoadTestShape): + use_common_options = True + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.arrival_rate = locust.argument_parser.parse_options().arrival_rate + self.last_tick = 0 + + def tick(self): + if self.last_tick == 0: + logger.info("Constant load shape arrival rate: {arrival_rate}".format(arrival_rate=self.arrival_rate)) + + run_time = self.get_run_time() + if run_time < self.runner.environment.parsed_options.run_time: + self.last_tick = run_time + + new_users = int(self.arrival_rate) + current_users = self.get_current_user_count() + user_count = current_users + new_users + logger.debug( + "Current users: {current_users}, New users: {new_users}, Target users: {target_users}".format( + current_users=current_users, new_users=new_users, target_users=user_count + ) + ) + # Avoid illegal spawn_rate value of 0 + spawn_rate = max(0.01, new_users) + return (user_count, spawn_rate) + + self.runner.environment.stop_timeout = 0 + return None + diff --git a/evals/benchmark/stresscli/locust/poisson_load_shape.py b/evals/benchmark/stresscli/locust/poisson_load_shape.py index 35a8b64c..80fc0e99 100644 --- a/evals/benchmark/stresscli/locust/poisson_load_shape.py +++ b/evals/benchmark/stresscli/locust/poisson_load_shape.py @@ -13,7 +13,7 @@ @events.init_command_line_parser.add_listener def _(parser): parser.add_argument( - "--arrival-rate", + "--arrival_rate", type=float, default=1.0, ) From ec5a4c582fdb752d25d760f09035694f70d665c0 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 11 Oct 2024 09:19:46 +0000 Subject: [PATCH 4/9] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci Signed-off-by: CharleneHu-42 --- evals/benchmark/stresscli/locust/constant_load_shape.py | 1 - 1 file changed, 1 deletion(-) diff --git a/evals/benchmark/stresscli/locust/constant_load_shape.py b/evals/benchmark/stresscli/locust/constant_load_shape.py index fc90d1d9..ec6239d6 100644 --- a/evals/benchmark/stresscli/locust/constant_load_shape.py +++ b/evals/benchmark/stresscli/locust/constant_load_shape.py @@ -48,4 +48,3 @@ def tick(self): self.runner.environment.stop_timeout = 0 return None - From 54ff0ce0571481a5be8d85af0ce8e7148041599a Mon Sep 17 00:00:00 2001 From: CharleneHu-42 Date: Fri, 11 Oct 2024 17:25:22 +0800 Subject: [PATCH 5/9] fix typo Signed-off-by: CharleneHu-42 --- evals/benchmark/README.md | 16 ++++++++++------ evals/benchmark/benchmark.yaml | 2 +- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/evals/benchmark/README.md b/evals/benchmark/README.md index 1c43921f..c5ffb401 100644 --- a/evals/benchmark/README.md +++ b/evals/benchmark/README.md @@ -12,11 +12,15 @@ This Tool provides a microservice benchmarking framework that uses YAML configur ## Table of Contents -- [Installation](#installation) -- [Usage](#usage) -- [Configuration](#configuration) - - [Test Suite Configuration](#test-suite-configuration) - - [Test Cases](#test-cases) +- [OPEA Benchmark Tool](#opea-benchmark-tool) + - [Features](#features) + - [Table of Contents](#table-of-contents) + - [Installation](#installation) + - [Prerequisites](#prerequisites) + - [Usage](#usage) + - [Configuration](#configuration) + - [Test Suite Configuration](#test-suite-configuration) + - [Test Cases](#test-cases) ## Installation @@ -68,7 +72,7 @@ test_suite_config: params: # Loadshape-specific parameters constant: # Constant load shape specific parameters, activate only if load_shape.name is constant concurrent_level: 4 # If user_queries is specified, concurrent_level is target number of requests per user. If not, it is the number of simulated users - # arrival_rate: 1.0 # Request arrival rate. If set, concurrent_level will be overrided, constant load will be generated based on arrival-rate + # arrival_rate: 1.0 # Request arrival rate. If set, concurrent_level will be overridden, constant load will be generated based on arrival-rate poisson: # Poisson load shape specific parameters, activate only if load_shape.name is poisson arrival_rate: 1.0 # Request arrival rate warm_ups: 0 # Number of test requests for warm-ups diff --git a/evals/benchmark/benchmark.yaml b/evals/benchmark/benchmark.yaml index 3b464b1c..5dbbc67f 100644 --- a/evals/benchmark/benchmark.yaml +++ b/evals/benchmark/benchmark.yaml @@ -21,7 +21,7 @@ test_suite_config: # Overall configuration settings for the test suite params: # Loadshape-specific parameters constant: # Constant load shape specific parameters, activate only if load_shape.name is constant concurrent_level: 4 # If user_queries is specified, concurrent_level is target number of requests per user. If not, it is the number of simulated users - # arrival_rate: 1.0 # Request arrival rate. If set, concurrent_level will be overrided, constant load will be generated based on arrival-rate + # arrival_rate: 1.0 # Request arrival rate. If set, concurrent_level will be overridden, constant load will be generated based on arrival-rate poisson: # Poisson load shape specific parameters, activate only if load_shape.name is poisson arrival_rate: 1.0 # Request arrival rate From 6e9c0a6486ef49da109ce2d974bcc22402b702eb Mon Sep 17 00:00:00 2001 From: CharleneHu-42 Date: Mon, 14 Oct 2024 17:33:20 +0800 Subject: [PATCH 6/9] unset stop_timeout for locust workers to avoid exception Signed-off-by: CharleneHu-42 --- evals/benchmark/stresscli/locust/aistress.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/evals/benchmark/stresscli/locust/aistress.py b/evals/benchmark/stresscli/locust/aistress.py index 89cc9538..795974bf 100644 --- a/evals/benchmark/stresscli/locust/aistress.py +++ b/evals/benchmark/stresscli/locust/aistress.py @@ -220,6 +220,7 @@ def on_locust_init(environment, **_kwargs): environment.runner.register_message("worker_reqsent", on_reqsent) if not isinstance(environment.runner, MasterRunner): environment.runner.register_message("all_reqcnt", on_reqcount) + environment.runner.register_message("test_quit", on_quit) @events.quitting.add_listener @@ -249,6 +250,11 @@ def on_reqcount(msg, **kwargs): AiStressUser.request = msg.data +def on_quit(environment, msg, **kwargs): + logging.debug(f"Test quitting, set stop_timeout to 0...") + environment.runner.environment.stop_timeout = 0 + + def checker(environment): while environment.runner.state not in [STATE_STOPPING, STATE_STOPPED, STATE_CLEANUP]: time.sleep(1) @@ -256,7 +262,8 @@ def checker(environment): if max_request >= 0 and environment.runner.stats.num_requests >= max_request: logging.info(f"Exceed the max-request number:{environment.runner.stats.num_requests}, Exit...") # Remove stop_timeout after test quit to avoid Locust user stop exception with custom load shape - environment.runner.environment.stop_timeout = 0.0 + environment.runner.send_message("test_quit", None) + environment.runner.environment.stop_timeout = 0 # while environment.runner.user_count > 0: time.sleep(5) environment.runner.quit() From 84a20dc8bdd5e087dde94647c9b9535e5f4ecdcd Mon Sep 17 00:00:00 2001 From: CharleneHu-42 Date: Thu, 17 Oct 2024 17:02:40 +0800 Subject: [PATCH 7/9] fix readme Signed-off-by: CharleneHu-42 --- evals/benchmark/README.md | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/evals/benchmark/README.md b/evals/benchmark/README.md index c5ffb401..ac2f3870 100644 --- a/evals/benchmark/README.md +++ b/evals/benchmark/README.md @@ -12,15 +12,11 @@ This Tool provides a microservice benchmarking framework that uses YAML configur ## Table of Contents -- [OPEA Benchmark Tool](#opea-benchmark-tool) - - [Features](#features) - - [Table of Contents](#table-of-contents) - - [Installation](#installation) - - [Prerequisites](#prerequisites) - - [Usage](#usage) - - [Configuration](#configuration) - - [Test Suite Configuration](#test-suite-configuration) - - [Test Cases](#test-cases) +- [Installation](#installation) +- [Usage](#usage) +- [Configuration](#configuration) + - [Test Suite Configuration](#test-suite-configuration) + - [Test Cases](#test-cases) ## Installation From c3c579197193fdbd8f220a42131b1480606b14f1 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 17 Oct 2024 09:06:08 +0000 Subject: [PATCH 8/9] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- evals/benchmark/stresscli/locust/aistress.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/evals/benchmark/stresscli/locust/aistress.py b/evals/benchmark/stresscli/locust/aistress.py index 795974bf..4dbc220a 100644 --- a/evals/benchmark/stresscli/locust/aistress.py +++ b/evals/benchmark/stresscli/locust/aistress.py @@ -251,7 +251,7 @@ def on_reqcount(msg, **kwargs): def on_quit(environment, msg, **kwargs): - logging.debug(f"Test quitting, set stop_timeout to 0...") + logging.debug("Test quitting, set stop_timeout to 0...") environment.runner.environment.stop_timeout = 0 From bc2db4a1f15a2b6fb10998486ac43877f3278b60 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 18 Oct 2024 01:12:01 +0000 Subject: [PATCH 9/9] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- evals/benchmark/benchmark.py | 2 +- evals/benchmark/utils.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/evals/benchmark/benchmark.py b/evals/benchmark/benchmark.py index 8ce76c83..ccb73a3c 100644 --- a/evals/benchmark/benchmark.py +++ b/evals/benchmark/benchmark.py @@ -248,7 +248,7 @@ def run_service_test(example, service_type, service, test_suite_config): deployment_type, test_suite_config.get("service_ip"), test_suite_config.get("service_port"), - test_suite_config.get("namespace") + test_suite_config.get("namespace"), ) base_url = f"http://{svc_ip}:{port}" diff --git a/evals/benchmark/utils.py b/evals/benchmark/utils.py index 2256f0c3..d66212a0 100644 --- a/evals/benchmark/utils.py +++ b/evals/benchmark/utils.py @@ -29,8 +29,10 @@ def write_json(data, filename): logging.error(f"Failed to write {filename}: {e}") return False + from kubernetes import client, config + def get_service_cluster_ip(service_name, namespace="default"): # Load the Kubernetes configuration config.load_kube_config() # or use config.load_incluster_config() if running inside a Kubernetes pod