Merge branch 'dev' of github.com:spcl/serverless-benchmarks into dev

install.py

@@ -66,9 +66,8 @@ def execute(cmd):
 if args.local:
     print("Install Python dependencies for local")
     execute(". {}/bin/activate && pip3 install -r requirements.local.txt".format(env_dir))
-    if not args.dont_rebuild_docker_images:
-        print("Initialize Docker image for local storage.")
-        execute("docker pull minio/minio:latest")
+    print("Initialize Docker image for local storage.")
+    execute("docker pull minio/minio:latest")
 
 print("Initialize git submodules")
 execute("git submodule update --init --recursive")

sebs/aws/aws.py

@@ -217,8 +217,7 @@ def create_function(self, code_package: Benchmark, func_name: str) -> "LambdaFun
                 Timeout=timeout,
                 Code=code_config,
             )
-            # url = self.create_http_trigger(func_name, None, None)
-            # print(url)
+
             lambda_function = LambdaFunction(
                 func_name,
                 code_package.benchmark,
@@ -230,6 +229,8 @@ def create_function(self, code_package: Benchmark, func_name: str) -> "LambdaFun
                 code_bucket,
             )
 
+            self.wait_function_active(lambda_function)
+
         # Add LibraryTrigger to a new function
         from sebs.aws.triggers import LibraryTrigger
 
@@ -280,15 +281,15 @@ def update_function(self, function: Function, code_package: Benchmark):
             self.client.update_function_code(
                 FunctionName=name, S3Bucket=bucket, S3Key=code_package_name
             )
-        self.logging.info(
-            f"Updated code of {name} function. "
-            "Sleep 5 seconds before updating configuration to avoid cloud errors."
-        )
-        time.sleep(5)
+        self.wait_function_updated(function)
+        self.logging.info(f"Updated code of {name} function. ")
         # and update config
         self.client.update_function_configuration(
             FunctionName=name, Timeout=function.config.timeout, MemorySize=function.config.memory
         )
+        self.wait_function_updated(function)
+        self.logging.info(f"Updated configuration of {name} function. ")
+        self.wait_function_updated(function)
         self.logging.info("Published new function code")
 
     def update_function_configuration(self, function: Function, benchmark: Benchmark):
@@ -298,6 +299,8 @@ def update_function_configuration(self, function: Function, benchmark: Benchmark
             Timeout=function.config.timeout,
             MemorySize=function.config.memory,
         )
+        self.wait_function_updated(function)
+        self.logging.info(f"Updated configuration of {function.name} function. ")
 
     @staticmethod
     def default_function_name(code_package: Benchmark) -> str:
@@ -512,6 +515,22 @@ def enforce_cold_start(self, functions: List[Function], code_package: Benchmark)
         self.cold_start_counter += 1
         for func in functions:
             self._enforce_cold_start(func)
-        import time
+        self.logging.info("Sent function updates enforcing cold starts.")
+        for func in functions:
+            lambda_function = cast(LambdaFunction, func)
+            self.wait_function_updated(lambda_function)
+        self.logging.info("Finished function updates enforcing cold starts.")
+
+    def wait_function_active(self, func: LambdaFunction):
+
+        self.logging.info("Waiting for Lambda function to be created...")
+        waiter = self.client.get_waiter("function_active_v2")
+        waiter.wait(FunctionName=func.name)
+        self.logging.info("Lambda function has been created.")
+
+    def wait_function_updated(self, func: LambdaFunction):
 
-        time.sleep(5)
+        self.logging.info("Waiting for Lambda function to be updated...")
+        waiter = self.client.get_waiter("function_updated_v2")
+        waiter.wait(FunctionName=func.name)
+        self.logging.info("Lambda function has been updated.")

sebs/experiments/perf_cost.py

@@ -87,7 +87,7 @@ def run(self):
     def compute_statistics(self, times: List[float]):
 
         mean, median, std, cv = basic_stats(times)
-        self.logging.info(f"Mean {mean}, median {median}, std {std}, CV {cv}")
+        self.logging.info(f"Mean {mean} [ms], median {median} [ms], std {std}, CV {cv}")
         for alpha in [0.95, 0.99]:
             ci_interval = ci_tstudents(alpha, times)
             interval_width = ci_interval[1] - ci_interval[0]
@@ -173,15 +173,11 @@ def _run_configuration(
                             ret = res.get()
                             if first_iteration:
                                 continue
-                            if (run_type == PerfCost.RunType.COLD and not ret.stats.cold_start) or (
-                                run_type == PerfCost.RunType.WARM and ret.stats.cold_start
-                            ):
-                                self.logging.info(
-                                    f"Invocation {ret.request_id} "
-                                    f"cold: {ret.stats.cold_start} "
-                                    f"on experiment {run_type.str()}!"
-                                )
+                            if run_type == PerfCost.RunType.COLD and not ret.stats.cold_start:
+                                self.logging.info(f"Invocation {ret.request_id} is not cold!")
                                 incorrect.append(ret)
+                            elif run_type == PerfCost.RunType.WARM and ret.stats.cold_start:
+                                self.logging.info(f"Invocation {ret.request_id} is cold!")
                             else:
                                 result.add_invocation(self._function, ret)
                                 colds_count += ret.stats.cold_start
@@ -190,14 +186,15 @@ def _run_configuration(
                         except Exception as e:
                             error_count += 1
                             error_executions.append(str(e))
-                    self.logging.info(
-                        f"Processed {samples_gathered} samples out of {repetitions},"
-                        f"{error_count} errors"
-                    )
                     samples_generated += invocations
                     if first_iteration:
                         self.logging.info(
-                            f"Processed {samples_gathered} warm-up samples, ignore results."
+                            f"Processed {samples_gathered} warm-up samples, ignoring these results."
+                        )
+                    else:
+                        self.logging.info(
+                            f"Processed {samples_gathered} samples out of {repetitions},"
+                            f" {error_count} errors"
                         )
 
                     first_iteration = False