From 70e77e9fc8864a19f20bea60295b5d74e432e46d Mon Sep 17 00:00:00 2001 From: Marcin Copik Date: Fri, 6 May 2022 20:21:52 +0200 Subject: [PATCH 1/3] [system] Remove old option from the installation script --- install.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/install.py b/install.py index f98c2a29..0be56542 100755 --- a/install.py +++ b/install.py @@ -66,9 +66,8 @@ def execute(cmd): if args.local: print("Install Python dependencies for local") execute(". {}/bin/activate && pip3 install -r requirements.local.txt".format(env_dir)) - if not args.dont_rebuild_docker_images: - print("Initialize Docker image for local storage.") - execute("docker pull minio/minio:latest") + print("Initialize Docker image for local storage.") + execute("docker pull minio/minio:latest") print("Initialize git submodules") execute("git submodule update --init --recursive") From 3cdbdc9d0786efc6644111742e3f62af7eff911d Mon Sep 17 00:00:00 2001 From: Marcin Copik Date: Fri, 6 May 2022 20:50:10 +0200 Subject: [PATCH 2/3] [aws] Implement waiting for changes in function states --- sebs/aws/aws.py | 37 ++++++++++++++++++++++++++++--------- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/sebs/aws/aws.py b/sebs/aws/aws.py index 7442bb97..6c34af90 100644 --- a/sebs/aws/aws.py +++ b/sebs/aws/aws.py @@ -217,8 +217,7 @@ def create_function(self, code_package: Benchmark, func_name: str) -> "LambdaFun Timeout=timeout, Code=code_config, ) - # url = self.create_http_trigger(func_name, None, None) - # print(url) + lambda_function = LambdaFunction( func_name, code_package.benchmark, @@ -230,6 +229,8 @@ def create_function(self, code_package: Benchmark, func_name: str) -> "LambdaFun code_bucket, ) + self.wait_function_active(lambda_function) + # Add LibraryTrigger to a new function from sebs.aws.triggers import LibraryTrigger @@ -280,15 +281,15 @@ def update_function(self, function: Function, code_package: Benchmark): self.client.update_function_code( FunctionName=name, S3Bucket=bucket, S3Key=code_package_name ) - self.logging.info( - f"Updated code of {name} function. " - "Sleep 5 seconds before updating configuration to avoid cloud errors." - ) - time.sleep(5) + self.wait_function_updated(function) + self.logging.info(f"Updated code of {name} function. ") # and update config self.client.update_function_configuration( FunctionName=name, Timeout=function.config.timeout, MemorySize=function.config.memory ) + self.wait_function_updated(function) + self.logging.info(f"Updated configuration of {name} function. ") + self.wait_function_updated(function) self.logging.info("Published new function code") def update_function_configuration(self, function: Function, benchmark: Benchmark): @@ -298,6 +299,8 @@ def update_function_configuration(self, function: Function, benchmark: Benchmark Timeout=function.config.timeout, MemorySize=function.config.memory, ) + self.wait_function_updated(function) + self.logging.info(f"Updated configuration of {function.name} function. ") @staticmethod def default_function_name(code_package: Benchmark) -> str: @@ -512,6 +515,22 @@ def enforce_cold_start(self, functions: List[Function], code_package: Benchmark) self.cold_start_counter += 1 for func in functions: self._enforce_cold_start(func) - import time + self.logging.info("Sent function updates enforcing cold starts.") + for func in functions: + lambda_function = cast(LambdaFunction, func) + self.wait_function_updated(lambda_function) + self.logging.info("Finished function updates enforcing cold starts.") + + def wait_function_active(self, func: LambdaFunction): + + self.logging.info("Waiting for Lambda function to be created...") + waiter = self.client.get_waiter("function_active_v2") + waiter.wait(FunctionName=func.name) + self.logging.info("Lambda function has been created.") + + def wait_function_updated(self, func: LambdaFunction): - time.sleep(5) + self.logging.info("Waiting for Lambda function to be updated...") + waiter = self.client.get_waiter("function_updated_v2") + waiter.wait(FunctionName=func.name) + self.logging.info("Lambda function has been updated.") From dc9fd80cee15899c574082301de5ec5008a88910 Mon Sep 17 00:00:00 2001 From: Marcin Copik Date: Fri, 6 May 2022 20:58:09 +0200 Subject: [PATCH 3/3] [experiments] Improve output --- sebs/experiments/perf_cost.py | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/sebs/experiments/perf_cost.py b/sebs/experiments/perf_cost.py index 3fc81482..36cde660 100644 --- a/sebs/experiments/perf_cost.py +++ b/sebs/experiments/perf_cost.py @@ -87,7 +87,7 @@ def run(self): def compute_statistics(self, times: List[float]): mean, median, std, cv = basic_stats(times) - self.logging.info(f"Mean {mean}, median {median}, std {std}, CV {cv}") + self.logging.info(f"Mean {mean} [ms], median {median} [ms], std {std}, CV {cv}") for alpha in [0.95, 0.99]: ci_interval = ci_tstudents(alpha, times) interval_width = ci_interval[1] - ci_interval[0] @@ -173,15 +173,11 @@ def _run_configuration( ret = res.get() if first_iteration: continue - if (run_type == PerfCost.RunType.COLD and not ret.stats.cold_start) or ( - run_type == PerfCost.RunType.WARM and ret.stats.cold_start - ): - self.logging.info( - f"Invocation {ret.request_id} " - f"cold: {ret.stats.cold_start} " - f"on experiment {run_type.str()}!" - ) + if run_type == PerfCost.RunType.COLD and not ret.stats.cold_start: + self.logging.info(f"Invocation {ret.request_id} is not cold!") incorrect.append(ret) + elif run_type == PerfCost.RunType.WARM and ret.stats.cold_start: + self.logging.info(f"Invocation {ret.request_id} is cold!") else: result.add_invocation(self._function, ret) colds_count += ret.stats.cold_start @@ -190,14 +186,15 @@ def _run_configuration( except Exception as e: error_count += 1 error_executions.append(str(e)) - self.logging.info( - f"Processed {samples_gathered} samples out of {repetitions}," - f"{error_count} errors" - ) samples_generated += invocations if first_iteration: self.logging.info( - f"Processed {samples_gathered} warm-up samples, ignore results." + f"Processed {samples_gathered} warm-up samples, ignoring these results." + ) + else: + self.logging.info( + f"Processed {samples_gathered} samples out of {repetitions}," + f" {error_count} errors" ) first_iteration = False