Merge branch 'ClickHouse:master' into master

docker/test/upgrade/run.sh

@@ -76,7 +76,8 @@ sudo mv /etc/clickhouse-server/config.d/keeper_port.xml.tmp /etc/clickhouse-serv
 # But we still need default disk because some tables loaded only into it
 sudo cat /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml \
  | sed "s|<main><disk>s3</disk></main>|<main><disk>s3</disk></main><default><disk>default</disk></default>|" \
- > /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml.tmp mv /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml.tmp /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml
+ > /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml.tmp
+mv /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml.tmp /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml
 sudo chown clickhouse /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml
 sudo chgrp clickhouse /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml
 

docs/en/engines/table-engines/integrations/odbc.md

@@ -54,7 +54,7 @@ $ sudo mysql
 
 ``` sql
 mysql> CREATE USER 'clickhouse'@'localhost' IDENTIFIED BY 'clickhouse';
-mysql> GRANT ALL PRIVILEGES ON *.* TO 'clickhouse'@'clickhouse' WITH GRANT OPTION;
+mysql> GRANT ALL PRIVILEGES ON *.* TO 'clickhouse'@'localhost' WITH GRANT OPTION;
 ```
 
 Then configure the connection in `/etc/odbc.ini`.
@@ -66,7 +66,7 @@ DRIVER = /usr/local/lib/libmyodbc5w.so
 SERVER = 127.0.0.1
 PORT = 3306
 DATABASE = test
-USERNAME = clickhouse
+USER = clickhouse
 PASSWORD = clickhouse
 ```
 
@@ -83,6 +83,9 @@ $ isql -v mysqlconn
 Table in MySQL:
 
 ``` text
+mysql> CREATE DATABASE test;
+Query OK, 1 row affected (0,01 sec)
+
 mysql> CREATE TABLE `test`.`test` (
  -> `int_id` INT NOT NULL AUTO_INCREMENT,
  -> `int_nullable` INT NULL DEFAULT NULL,
@@ -91,10 +94,10 @@ mysql> CREATE TABLE `test`.`test` (
  -> PRIMARY KEY (`int_id`));
 Query OK, 0 rows affected (0,09 sec)
 
-mysql> insert into test (`int_id`, `float`) VALUES (1,2);
+mysql> insert into test.test (`int_id`, `float`) VALUES (1,2);
 Query OK, 1 row affected (0,00 sec)
 
-mysql> select * from test;
+mysql> select * from test.test;
 +------+----------+-----+----------+
 | int_id | int_nullable | float | float_nullable |
 +------+----------+-----+----------+

programs/local/LocalServer.cpp

@@ -8,7 +8,9 @@
 #include <Poco/Logger.h>
 #include <Poco/NullChannel.h>
 #include <Poco/SimpleFileChannel.h>
+#include <Databases/DatabaseFilesystem.h>
 #include <Databases/DatabaseMemory.h>
+#include <Databases/DatabasesOverlay.h>
 #include <Storages/System/attachSystemTables.h>
 #include <Storages/System/attachInformationSchemaTables.h>
 #include <Interpreters/DatabaseCatalog.h>
@@ -50,6 +52,8 @@
 #include <base/argsToConfig.h>
 #include <filesystem>
 
+#include "config.h"
+
 #if defined(FUZZING_MODE)
  #include <Functions/getFuzzerData.h>
 #endif
@@ -170,6 +174,13 @@ static DatabasePtr createMemoryDatabaseIfNotExists(ContextPtr context, const Str
  return system_database;
 }
 
+static DatabasePtr createClickHouseLocalDatabaseOverlay(const String & name_, ContextPtr context_)
+{
+ auto databaseCombiner = std::make_shared<DatabasesOverlay>(name_, context_);
+ databaseCombiner->registerNextDatabase(std::make_shared<DatabaseFilesystem>(name_, "", context_));
+ databaseCombiner->registerNextDatabase(std::make_shared<DatabaseMemory>(name_, context_));
+ return databaseCombiner;
+}
 
 /// If path is specified and not empty, will try to setup server environment and load existing metadata
 void LocalServer::tryInitPath()
@@ -669,7 +680,7 @@ void LocalServer::processConfig()
  * if such tables will not be dropped, clickhouse-server will not be able to load them due to security reasons.
  */
  std::string default_database = config().getString("default_database", "_local");
- DatabaseCatalog::instance().attachDatabase(default_database, std::make_shared<DatabaseMemory>(default_database, global_context));
+ DatabaseCatalog::instance().attachDatabase(default_database, createClickHouseLocalDatabaseOverlay(default_database, global_context));
  global_context->setCurrentDatabase(default_database);
  applyCmdOptions(global_context);
 

programs/server/dashboard.html

@@ -12,7 +12,8 @@
  --chart-background: white;
  --shadow-color: rgba(0, 0, 0, 0.25);
  --input-shadow-color: rgba(0, 255, 0, 1);
- --error-color: white;
+ --error-color: red;
+ --auth-error-color: white;
  --legend-background: rgba(255, 255, 255, 0.75);
  --title-color: #666;
  --text-color: black;
@@ -258,7 +259,7 @@
  width: 60%;
  padding: .5rem;
 
- color: var(--error-color);
+ color: var(--auth-error-color);
 
  display: flex;
  flex-flow: row nowrap;
@@ -906,9 +907,9 @@
 
  if (error) {
  const errorMatch = errorMessages.find(({ regex }) => error.match(regex))
- if (errorMatch) {
-  const match = error.match(errorMatch.regex)
-  const message = errorMatch.messageFunc(match)
+ const match = error.match(errorMatch.regex)
+ const message = errorMatch.messageFunc(match)
+ if (message) {
  const authError = new Error(message)
  throw authError
  }
@@ -930,7 +931,7 @@
  let title_div = chart.querySelector('.title');
  if (error) {
  error_div.firstChild.data = error;
- title_div.style.display = 'none';
+ title_div.style.display = 'none'; 
  error_div.style.display = 'block';
  return false;
  } else {
@@ -1019,13 +1020,15 @@
  firstLoad = false;
  } else {
  enableReloadButton();
+ enableRunButton();
  }
- if (!results.includes(false)) {
+ if (results.includes(true)) {
  const element = document.querySelector('.inputs');
  element.classList.remove('unconnected');
  const add = document.querySelector('#add');
  add.style.display = 'block';
- } else {
+ } 
+ else {
  const charts = document.querySelector('#charts')
  charts.style.height = '0px';
  }
@@ -1050,18 +1053,33 @@
  reloadButton.classList.add('disabled')
 }
 
+function disableRunButton() {
+ const runButton = document.getElementById('run')
+ runButton.value = 'Reloading...'
+ runButton.disabled = true
+ runButton.classList.add('disabled')
+}
+
 function enableReloadButton() {
  const reloadButton = document.getElementById('reload')
  reloadButton.value = 'Reload'
  reloadButton.disabled = false
  reloadButton.classList.remove('disabled')
 }
 
+function enableRunButton() {
+ const runButton = document.getElementById('run')
+ runButton.value = 'Ok'
+ runButton.disabled = false
+ runButton.classList.remove('disabled')
+}
+
 function reloadAll() {
  updateParams();
  drawAll();
  saveState();
- disableReloadButton()
+ disableReloadButton();
+ disableRunButton();
 }
 
 document.getElementById('params').onsubmit = function(event) {

src/Client/ClientBase.cpp

@@ -2297,7 +2297,9 @@ void ClientBase::runInteractive()
  catch (const ErrnoException & e)
  {
  if (e.getErrno() != EEXIST)
- throw;
+ {
+ std::cerr << getCurrentExceptionMessage(false) << '\n';
+ }
  }
  }
 

src/Client/ConnectionParameters.cpp

@@ -60,7 +60,15 @@ ConnectionParameters::ConnectionParameters(const Poco::Util::AbstractConfigurati
  quota_key = config.getString("quota_key", "");
 
  /// By default compression is disabled if address looks like localhost.
- compression = config.getBool("compression", !isLocalAddress(DNSResolver::instance().resolveHost(host)))
+
+ /// Avoid DNS request if the host is "localhost".
+ /// If ClickHouse is run under QEMU-user with a binary for a different architecture,
+ /// and there are all listed startup dependency shared libraries available, but not the runtime dependencies of glibc,
+ /// the glibc cannot open "plugins" for DNS resolving, and the DNS resolution does not work.
+ /// At the same time, I want clickhouse-local to always work, regardless.
+ /// TODO: get rid of glibc, or replace getaddrinfo to c-ares.
+
+ compression = config.getBool("compression", host != "localhost" && !isLocalAddress(DNSResolver::instance().resolveHost(host)))
  ? Protocol::Compression::Enable : Protocol::Compression::Disable;
 
  timeouts = ConnectionTimeouts(

src/Common/OptimizedRegularExpression.cpp

@@ -540,7 +540,7 @@ bool OptimizedRegularExpressionImpl<thread_safe>::match(const char * subject, si
  }
  }
 
- return re2->Match(StringPieceType(subject, subject_size), 0, subject_size, RegexType::UNANCHORED, nullptr, 0);
+ return re2->Match({subject, subject_size}, 0, subject_size, RegexType::UNANCHORED, nullptr, 0);
  }
 }
 
@@ -585,9 +585,9 @@ bool OptimizedRegularExpressionImpl<thread_safe>::match(const char * subject, si
  return false;
  }
 
- StringPieceType piece;
+ std::string_view piece;
 
- if (!RegexType::PartialMatch(StringPieceType(subject, subject_size), *re2, &piece))
+ if (!RegexType::PartialMatch({subject, subject_size}, *re2, &piece))
  return false;
  else
  {
@@ -652,10 +652,10 @@ unsigned OptimizedRegularExpressionImpl<thread_safe>::match(const char * subject
  return 0;
  }
 
- DB::PODArrayWithStackMemory<StringPieceType, 128> pieces(limit);
+ DB::PODArrayWithStackMemory<std::string_view, 128> pieces(limit);
 
  if (!re2->Match(
- StringPieceType(subject, subject_size),
+ {subject, subject_size},
  0,
  subject_size,
  RegexType::UNANCHORED,

src/Common/OptimizedRegularExpression.h

@@ -52,7 +52,6 @@ class OptimizedRegularExpressionImpl
  using MatchVec = std::vector<Match>;
 
  using RegexType = std::conditional_t<thread_safe, re2::RE2, re2_st::RE2>;
- using StringPieceType = std::conditional_t<thread_safe, re2::StringPiece, re2_st::StringPiece>;
 
  OptimizedRegularExpressionImpl(const std::string & regexp_, int options = 0); /// NOLINT
  /// StringSearcher store pointers to required_substring, it must be updated on move.

src/Common/SensitiveDataMasker.cpp

@@ -5,7 +5,6 @@
 #include <atomic>
 
 #include <re2/re2.h>
-#include <re2/stringpiece.h>
 
 #include <Poco/Util/AbstractConfiguration.h>
 
@@ -44,7 +43,7 @@ class SensitiveDataMasker::MaskingRule
  const std::string regexp_string;
 
  const RE2 regexp;
- const re2::StringPiece replacement;
+ const std::string_view replacement;
 
 #ifndef NDEBUG
  mutable std::atomic<std::uint64_t> matches_count = 0;

src/Common/checkStackSize.cpp

@@ -27,7 +27,7 @@ static thread_local size_t max_stack_size = 0;
  * @param out_address - if not nullptr, here the address of the stack will be written.
  * @return stack size
  */
-size_t getStackSize(void ** out_address)
+static size_t getStackSize(void ** out_address)
 {
  using namespace DB;
 
@@ -54,7 +54,15 @@ size_t getStackSize(void ** out_address)
  throwFromErrno("Cannot pthread_attr_get_np", ErrorCodes::CANNOT_PTHREAD_ATTR);
 # else
  if (0 != pthread_getattr_np(pthread_self(), &attr))
- throwFromErrno("Cannot pthread_getattr_np", ErrorCodes::CANNOT_PTHREAD_ATTR);
+ {
+ if (errno == ENOENT)
+ {
+ /// Most likely procfs is not mounted.
+ return 0;
+ }
+ else
+ throwFromErrno("Cannot pthread_getattr_np", ErrorCodes::CANNOT_PTHREAD_ATTR);
+ }
 # endif
 
  SCOPE_EXIT({ pthread_attr_destroy(&attr); });
@@ -83,6 +91,10 @@ __attribute__((__weak__)) void checkStackSize()
  if (!stack_address)
  max_stack_size = getStackSize(&stack_address);
 
+ /// The check is impossible.
+ if (!max_stack_size)
+ return;
+
  const void * frame_address = __builtin_frame_address(0);
  uintptr_t int_frame_address = reinterpret_cast<uintptr_t>(frame_address);
  uintptr_t int_stack_address = reinterpret_cast<uintptr_t>(stack_address);

src/Common/parseGlobs.cpp

@@ -3,7 +3,6 @@
 #include <IO/ReadBufferFromString.h>
 #include <IO/Operators.h>
 #include <re2/re2.h>
-#include <re2/stringpiece.h>
 #include <algorithm>
 #include <sstream>
 #include <iomanip>
@@ -33,14 +32,14 @@ std::string makeRegexpPatternFromGlobs(const std::string & initial_str_with_glob
  std::string escaped_with_globs = buf_for_escaping.str();
 
  static const re2::RE2 enum_or_range(R"({([\d]+\.\.[\d]+|[^{}*,]+,[^{}*]*[^{}*,])})"); /// regexp for {expr1,expr2,expr3} or {M..N}, where M and N - non-negative integers, expr's should be without "{", "}", "*" and ","
- re2::StringPiece input(escaped_with_globs);
- re2::StringPiece matched;
+ std::string_view input(escaped_with_globs);
+ std::string_view matched;
  std::ostringstream oss_for_replacing; // STYLE_CHECK_ALLOW_STD_STRING_STREAM
  oss_for_replacing.exceptions(std::ios::failbit);
  size_t current_index = 0;
  while (RE2::FindAndConsume(&input, enum_or_range, &matched))
  {
- std::string buffer{matched};
+ std::string buffer(matched);
  oss_for_replacing << escaped_with_globs.substr(current_index, matched.data() - escaped_with_globs.data() - current_index - 1) << '(';
 
  if (buffer.find(',') == std::string::npos)