Merge pull request ClickHouse#53499 from ClickHouse/simplify-performa…

…nce-test

Simplify performance test

docker/test/performance-comparison/Dockerfile

@@ -1,18 +1,7 @@
 # docker build -t clickhouse/performance-comparison .
 
-# Using ubuntu:22.04 over 20.04 as all other images, since:
-# a) ubuntu 20.04 has too old parallel, and does not support --memsuspend
-# b) anyway for perf tests it should not be important (backward compatiblity
-# with older ubuntu had been checked lots of times in various tests)
-FROM ubuntu:22.04
-
-# ARG for quick switch to a given ubuntu mirror
-ARG apt_archive="http://archive.ubuntu.com"
-RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list
-
-ENV LANG=C.UTF-8
-ENV TZ=Europe/Amsterdam
-RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
+ARG FROM_TAG=latest
+FROM clickhouse/test-base:$FROM_TAG
 
 RUN apt-get update \
  && DEBIAN_FRONTEND=noninteractive apt-get install --yes --no-install-recommends \
@@ -56,10 +45,9 @@ COPY * /
 # node #0 should be less stable because of system interruptions. We bind
 # randomly to node 1 or 0 to gather some statistics on that. We have to bind
 # both servers and the tmpfs on which the database is stored. How to do it
-# through Yandex Sandbox API is unclear, but by default tmpfs uses
+# is unclear, but by default tmpfs uses
 # 'process allocation policy', not sure which process but hopefully the one that
-# writes to it, so just bind the downloader script as well. We could also try to
-# remount it with proper options in Sandbox task.
+# writes to it, so just bind the downloader script as well.
 # https://www.kernel.org/doc/Documentation/filesystems/tmpfs.txt
 # Double-escaped backslashes are a tribute to the engineering wonder of docker --
 # it gives '/bin/sh: 1: [bash,: not found' otherwise.

docker/test/performance-comparison/compare.sh

@@ -90,7 +90,7 @@ function configure
  set +m
 
  wait_for_server $LEFT_SERVER_PORT $left_pid
- echo Server for setup started
+ echo "Server for setup started"
 
  clickhouse-client --port $LEFT_SERVER_PORT --query "create database test" ||:
  clickhouse-client --port $LEFT_SERVER_PORT --query "rename table datasets.hits_v1 to test.hits" ||:
@@ -156,9 +156,9 @@ function restart
  wait_for_server $RIGHT_SERVER_PORT $right_pid
  echo right ok
 
- clickhouse-client --port $LEFT_SERVER_PORT --query "select * from system.tables where database != 'system'"
+ clickhouse-client --port $LEFT_SERVER_PORT --query "select * from system.tables where database NOT IN ('system', 'INFORMATION_SCHEMA', 'information_schema')"
  clickhouse-client --port $LEFT_SERVER_PORT --query "select * from system.build_options"
- clickhouse-client --port $RIGHT_SERVER_PORT --query "select * from system.tables where database != 'system'"
+ clickhouse-client --port $RIGHT_SERVER_PORT --query "select * from system.tables where database NOT IN ('system', 'INFORMATION_SCHEMA', 'information_schema')"
  clickhouse-client --port $RIGHT_SERVER_PORT --query "select * from system.build_options"
 
  # Check again that both servers we started are running -- this is important
@@ -352,14 +352,12 @@ function get_profiles
  wait
 
  clickhouse-client --port $LEFT_SERVER_PORT --query "select * from system.query_log where type in ('QueryFinish', 'ExceptionWhileProcessing') format TSVWithNamesAndTypes" > left-query-log.tsv ||: &
- clickhouse-client --port $LEFT_SERVER_PORT --query "select * from system.query_thread_log format TSVWithNamesAndTypes" > left-query-thread-log.tsv ||: &
  clickhouse-client --port $LEFT_SERVER_PORT --query "select * from system.trace_log format TSVWithNamesAndTypes" > left-trace-log.tsv ||: &
  clickhouse-client --port $LEFT_SERVER_PORT --query "select arrayJoin(trace) addr, concat(splitByChar('/', addressToLine(addr))[-1], '#', demangle(addressToSymbol(addr)) ) name from system.trace_log group by addr format TSVWithNamesAndTypes" > left-addresses.tsv ||: &
  clickhouse-client --port $LEFT_SERVER_PORT --query "select * from system.metric_log format TSVWithNamesAndTypes" > left-metric-log.tsv ||: &
  clickhouse-client --port $LEFT_SERVER_PORT --query "select * from system.asynchronous_metric_log format TSVWithNamesAndTypes" > left-async-metric-log.tsv ||: &
 
  clickhouse-client --port $RIGHT_SERVER_PORT --query "select * from system.query_log where type in ('QueryFinish', 'ExceptionWhileProcessing') format TSVWithNamesAndTypes" > right-query-log.tsv ||: &
- clickhouse-client --port $RIGHT_SERVER_PORT --query "select * from system.query_thread_log format TSVWithNamesAndTypes" > right-query-thread-log.tsv ||: &
  clickhouse-client --port $RIGHT_SERVER_PORT --query "select * from system.trace_log format TSVWithNamesAndTypes" > right-trace-log.tsv ||: &
  clickhouse-client --port $RIGHT_SERVER_PORT --query "select arrayJoin(trace) addr, concat(splitByChar('/', addressToLine(addr))[-1], '#', demangle(addressToSymbol(addr)) ) name from system.trace_log group by addr format TSVWithNamesAndTypes" > right-addresses.tsv ||: &
  clickhouse-client --port $RIGHT_SERVER_PORT --query "select * from system.metric_log format TSVWithNamesAndTypes" > right-metric-log.tsv ||: &

docker/test/performance-comparison/config/config.d/zzz-perf-comparison-tweaks-config.xml

@@ -19,31 +19,6 @@
  <opentelemetry_span_log remove="remove"/>
  <session_log remove="remove"/>
 
- <!-- performance tests does not uses real block devices,
- instead they stores everything in memory.
-
- And so, to avoid extra memory reference switch *_log to Memory engine. -->
- <query_log>
- <engine>ENGINE = Memory</engine>
- <partition_by remove="remove"/>
- </query_log>
- <query_thread_log>
- <engine>ENGINE = Memory</engine>
- <partition_by remove="remove"/>
- </query_thread_log>
- <trace_log>
- <engine>ENGINE = Memory</engine>
- <partition_by remove="remove"/>
- </trace_log>
- <metric_log>
- <engine>ENGINE = Memory</engine>
- <partition_by remove="remove"/>
- </metric_log>
- <asynchronous_metric_log>
- <engine>ENGINE = Memory</engine>
- <partition_by remove="remove"/>
- </asynchronous_metric_log>
-
  <uncompressed_cache_size>1000000000</uncompressed_cache_size>
 
  <asynchronous_metrics_update_period_s>10</asynchronous_metrics_update_period_s>

docker/test/performance-comparison/download.sh

@@ -31,8 +31,6 @@ function download
  # Test all of them.
  declare -a urls_to_try=(
  "$S3_URL/PRs/$left_pr/$left_sha/$BUILD_NAME/performance.tar.zst"
- "$S3_URL/$left_pr/$left_sha/$BUILD_NAME/performance.tar.zst"
- "$S3_URL/$left_pr/$left_sha/$BUILD_NAME/performance.tgz"
  )
 
  for path in "${urls_to_try[@]}"

docker/test/performance-comparison/entrypoint.sh

@@ -130,7 +130,7 @@ then
  git -C right/ch diff --name-only "$base" pr -- :!tests/performance :!docker/test/performance-comparison | tee other-changed-files.txt
 fi
 
-# Set python output encoding so that we can print queries with Russian letters.
+# Set python output encoding so that we can print queries with non-ASCII letters.
 export PYTHONIOENCODING=utf-8
 
 # By default, use the main comparison script from the tested package, so that we
@@ -151,11 +151,7 @@ export PATH
 export REF_PR
 export REF_SHA
 
-# Try to collect some core dumps. I've seen two patterns in Sandbox:
-# 1) |/home/zomb-sandbox/venv/bin/python /home/zomb-sandbox/client/sandbox/bin/coredumper.py %e %p %g %u %s %P %c
-# Not sure what this script does (puts them to sandbox resources, logs some messages?),
-# and it's not accessible from inside docker anyway.
-# 2) something like %e.%p.core.dmp. The dump should end up in the workspace directory.
+# Try to collect some core dumps.
 # At least we remove the ulimit and then try to pack some common file names into output.
 ulimit -c unlimited
 cat /proc/sys/kernel/core_pattern

tests/ci/performance_comparison_check.py

@@ -1,6 +1,5 @@
 #!/usr/bin/env python3
 
-
 import os
 import logging
 import sys
@@ -20,11 +19,15 @@
 from pr_info import PRInfo
 from s3_helper import S3Helper
 from tee_popen import TeePopen
+from clickhouse_helper import get_instance_type
+from stopwatch import Stopwatch
 
 IMAGE_NAME = "clickhouse/performance-comparison"
 
 
 def get_run_command(
+ check_start_time,
+ check_name,
  workspace,
  result_path,
  repo_tests_path,
@@ -33,12 +36,24 @@ def get_run_command(
  additional_env,
  image,
 ):
+ instance_type = get_instance_type()
+
+ envs = [
+ f"-e CHECK_START_TIME='{check_start_time}'",
+ f"-e CHECK_NAME='{check_name}'",
+ f"-e INSTANCE_TYPE='{instance_type}'",
+ f"-e PR_TO_TEST={pr_to_test}",
+ f"-e SHA_TO_TEST={sha_to_test}",
+ ]
+
+ env_str = " ".join(envs)
+
  return (
  f"docker run --privileged --volume={workspace}:/workspace "
  f"--volume={result_path}:/output "
  f"--volume={repo_tests_path}:/usr/share/clickhouse-test "
  f"--cap-add syslog --cap-add sys_admin --cap-add sys_rawio "
- f"-e PR_TO_TEST={pr_to_test} -e SHA_TO_TEST={sha_to_test} {additional_env} "
+ f"{env_str} {additional_env} "
  f"{image}"
  )
 
@@ -62,6 +77,9 @@ def __exit__(self, exc_type, exc_val, exc_tb):
 
 if __name__ == "__main__":
  logging.basicConfig(level=logging.INFO)
+
+ stopwatch = Stopwatch()
+
  temp_path = os.getenv("TEMP_PATH", os.path.abspath("."))
  repo_path = os.getenv("REPO_COPY", os.path.abspath("../../"))
  repo_tests_path = os.path.join(repo_path, "tests")
@@ -157,6 +175,8 @@ def __exit__(self, exc_type, exc_val, exc_tb):
  docker_env += "".join([f" -e {name}" for name in env_extra])
 
  run_command = get_run_command(
+ stopwatch.start_time_str,
+ check_name,
  result_path,
  result_path,
  repo_tests_path,
@@ -168,6 +188,7 @@ def __exit__(self, exc_type, exc_val, exc_tb):
  logging.info("Going to run command %s", run_command)
 
  run_log_path = os.path.join(temp_path, "run.log")
+ compare_log_path = os.path.join(result_path, "compare.log")
 
  popen_env = os.environ.copy()
  popen_env.update(env_extra)
@@ -181,7 +202,7 @@ def __exit__(self, exc_type, exc_val, exc_tb):
  subprocess.check_call(f"sudo chown -R ubuntu:ubuntu {temp_path}", shell=True)
 
  paths = {
- "compare.log": os.path.join(result_path, "compare.log"),
+ "compare.log": compare_log_path,
  "output.7z": os.path.join(result_path, "output.7z"),
  "report.html": os.path.join(result_path, "report.html"),
  "all-queries.html": os.path.join(result_path, "all-queries.html"),