Merge branch 'master' into docs-remove-async-ml-event-time

docker/test/integration/runner/Dockerfile

@@ -47,11 +47,13 @@ ENV TZ=Etc/UTC
 RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
 
 ENV DOCKER_CHANNEL stable
+# Unpin the docker version after the release 24.0.3 is released
+# https://github.com/moby/moby/issues/45770#issuecomment-1618255130
 RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - \
  && add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -c -s) ${DOCKER_CHANNEL}" \
  && apt-get update \
  && env DEBIAN_FRONTEND=noninteractive apt-get install --yes \
- docker-ce \
+ docker-ce='5:23.*' \
  && rm -rf \
  /var/lib/apt/lists/* \
  /var/cache/debconf \

docker/test/upgrade/run.sh

@@ -76,7 +76,8 @@ sudo mv /etc/clickhouse-server/config.d/keeper_port.xml.tmp /etc/clickhouse-serv
 # But we still need default disk because some tables loaded only into it
 sudo cat /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml \
  | sed "s|<main><disk>s3</disk></main>|<main><disk>s3</disk></main><default><disk>default</disk></default>|" \
- > /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml.tmp mv /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml.tmp /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml
+ > /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml.tmp
+mv /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml.tmp /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml
 sudo chown clickhouse /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml
 sudo chgrp clickhouse /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml
 

docs/en/engines/table-engines/integrations/odbc.md

@@ -54,7 +54,7 @@ $ sudo mysql
 
 ``` sql
 mysql> CREATE USER 'clickhouse'@'localhost' IDENTIFIED BY 'clickhouse';
-mysql> GRANT ALL PRIVILEGES ON *.* TO 'clickhouse'@'clickhouse' WITH GRANT OPTION;
+mysql> GRANT ALL PRIVILEGES ON *.* TO 'clickhouse'@'localhost' WITH GRANT OPTION;
 ```
 
 Then configure the connection in `/etc/odbc.ini`.
@@ -66,7 +66,7 @@ DRIVER = /usr/local/lib/libmyodbc5w.so
 SERVER = 127.0.0.1
 PORT = 3306
 DATABASE = test
-USERNAME = clickhouse
+USER = clickhouse
 PASSWORD = clickhouse
 ```
 
@@ -83,6 +83,9 @@ $ isql -v mysqlconn
 Table in MySQL:
 
 ``` text
+mysql> CREATE DATABASE test;
+Query OK, 1 row affected (0,01 sec)
+
 mysql> CREATE TABLE `test`.`test` (
  -> `int_id` INT NOT NULL AUTO_INCREMENT,
  -> `int_nullable` INT NULL DEFAULT NULL,
@@ -91,10 +94,10 @@ mysql> CREATE TABLE `test`.`test` (
  -> PRIMARY KEY (`int_id`));
 Query OK, 0 rows affected (0,09 sec)
 
-mysql> insert into test (`int_id`, `float`) VALUES (1,2);
+mysql> insert into test.test (`int_id`, `float`) VALUES (1,2);
 Query OK, 1 row affected (0,00 sec)
 
-mysql> select * from test;
+mysql> select * from test.test;
 +------+----------+-----+----------+
 | int_id | int_nullable | float | float_nullable |
 +------+----------+-----+----------+

docs/en/operations/settings/settings.md

@@ -3201,6 +3201,40 @@ ENGINE = Log
 └──────────────────────────────────────────────────────────────────────────┘
 ```
 
+## default_temporary_table_engine {#default_temporary_table_engine}
+
+Same as [default_table_engine](#default_table_engine) but for temporary tables.
+
+Default value: `Memory`.
+
+In this example, any new temporary table that does not specify an `Engine` will use the `Log` table engine:
+
+Query:
+
+```sql
+SET default_temporary_table_engine = 'Log';
+
+CREATE TEMPORARY TABLE my_table (
+ x UInt32,
+ y UInt32
+);
+
+SHOW CREATE TEMPORARY TABLE my_table;
+```
+
+Result:
+
+```response
+┌─statement────────────────────────────────────────────────────────────────┐
+│ CREATE TEMPORARY TABLE default.my_table
+(
+ `x` UInt32,
+ `y` UInt32
+)
+ENGINE = Log
+└──────────────────────────────────────────────────────────────────────────┘
+```
+
 ## data_type_default_nullable {#data_type_default_nullable}
 
 Allows data types without explicit modifiers [NULL or NOT NULL](../../sql-reference/statements/create/table.md/#null-modifiers) in column definition will be [Nullable](../../sql-reference/data-types/nullable.md/#data_type-nullable).

docs/en/operations/system-tables/index.md

@@ -13,6 +13,7 @@ System tables provide information about:
 
 - Server states, processes, and environment.
 - Server’s internal processes.
+- Options used when the ClickHouse binary was built.
 
 System tables:
 

docs/en/sql-reference/statements/show.md

@@ -283,7 +283,7 @@ The optional keyword `EXTENDED` currently has no effect, it only exists for MySQ
 
 `SHOW INDEX` produces a result table with the following structure:
 - table - The name of the table (String)
-- non_unique - 0 if the index can contain duplicates, 1 otherwise (UInt8)
+- non_unique - 0 if the index cannot contain duplicates, 1 otherwise (UInt8)
 - key_name - The name of the index, `PRIMARY` if the index is a primary key index (String)
 - seq_in_index - Currently unused
 - column_name - Currently unused

docs/zh/sql-reference/functions/functions-for-nulls.md

@@ -192,7 +192,7 @@ SELECT coalesce(mail, phone, CAST(icq,'Nullable(String)')) FROM aBook
 **返回值**
 
 - 如果`x`不为`NULL`，返回非`Nullable`类型的原始值。
-- 如果`x`为`NULL`，返回对应非`Nullable`类型的默认值。
+- 如果`x`为`NULL`，则返回任意值。
 
 **示例**
 

programs/local/LocalServer.cpp

@@ -8,7 +8,9 @@
 #include <Poco/Logger.h>
 #include <Poco/NullChannel.h>
 #include <Poco/SimpleFileChannel.h>
+#include <Databases/DatabaseFilesystem.h>
 #include <Databases/DatabaseMemory.h>
+#include <Databases/DatabasesOverlay.h>
 #include <Storages/System/attachSystemTables.h>
 #include <Storages/System/attachInformationSchemaTables.h>
 #include <Interpreters/DatabaseCatalog.h>
@@ -50,6 +52,8 @@
 #include <base/argsToConfig.h>
 #include <filesystem>
 
+#include "config.h"
+
 #if defined(FUZZING_MODE)
  #include <Functions/getFuzzerData.h>
 #endif
@@ -170,6 +174,13 @@ static DatabasePtr createMemoryDatabaseIfNotExists(ContextPtr context, const Str
  return system_database;
 }
 
+static DatabasePtr createClickHouseLocalDatabaseOverlay(const String & name_, ContextPtr context_)
+{
+ auto databaseCombiner = std::make_shared<DatabasesOverlay>(name_, context_);
+ databaseCombiner->registerNextDatabase(std::make_shared<DatabaseFilesystem>(name_, "", context_));
+ databaseCombiner->registerNextDatabase(std::make_shared<DatabaseMemory>(name_, context_));
+ return databaseCombiner;
+}
 
 /// If path is specified and not empty, will try to setup server environment and load existing metadata
 void LocalServer::tryInitPath()
@@ -669,7 +680,7 @@ void LocalServer::processConfig()
  * if such tables will not be dropped, clickhouse-server will not be able to load them due to security reasons.
  */
  std::string default_database = config().getString("default_database", "_local");
- DatabaseCatalog::instance().attachDatabase(default_database, std::make_shared<DatabaseMemory>(default_database, global_context));
+ DatabaseCatalog::instance().attachDatabase(default_database, createClickHouseLocalDatabaseOverlay(default_database, global_context));
  global_context->setCurrentDatabase(default_database);
  applyCmdOptions(global_context);
 

programs/server/dashboard.html

@@ -12,7 +12,8 @@
  --chart-background: white;
  --shadow-color: rgba(0, 0, 0, 0.25);
  --input-shadow-color: rgba(0, 255, 0, 1);
- --error-color: white;
+ --error-color: red;
+ --auth-error-color: white;
  --legend-background: rgba(255, 255, 255, 0.75);
  --title-color: #666;
  --text-color: black;
@@ -258,7 +259,7 @@
  width: 60%;
  padding: .5rem;
 
- color: var(--error-color);
+ color: var(--auth-error-color);
 
  display: flex;
  flex-flow: row nowrap;
@@ -906,9 +907,9 @@
 
  if (error) {
  const errorMatch = errorMessages.find(({ regex }) => error.match(regex))
- if (errorMatch) {
-  const match = error.match(errorMatch.regex)
-  const message = errorMatch.messageFunc(match)
+ const match = error.match(errorMatch.regex)
+ const message = errorMatch.messageFunc(match)
+ if (message) {
  const authError = new Error(message)
  throw authError
  }
@@ -930,7 +931,7 @@
  let title_div = chart.querySelector('.title');
  if (error) {
  error_div.firstChild.data = error;
- title_div.style.display = 'none';
+ title_div.style.display = 'none'; 
  error_div.style.display = 'block';
  return false;
  } else {
@@ -1019,13 +1020,15 @@
  firstLoad = false;
  } else {
  enableReloadButton();
+ enableRunButton();
  }
- if (!results.includes(false)) {
+ if (results.includes(true)) {
  const element = document.querySelector('.inputs');
  element.classList.remove('unconnected');
  const add = document.querySelector('#add');
  add.style.display = 'block';
- } else {
+ } 
+ else {
  const charts = document.querySelector('#charts')
  charts.style.height = '0px';
  }
@@ -1050,18 +1053,33 @@
  reloadButton.classList.add('disabled')
 }
 
+function disableRunButton() {
+ const runButton = document.getElementById('run')
+ runButton.value = 'Reloading...'
+ runButton.disabled = true
+ runButton.classList.add('disabled')
+}
+
 function enableReloadButton() {
  const reloadButton = document.getElementById('reload')
  reloadButton.value = 'Reload'
  reloadButton.disabled = false
  reloadButton.classList.remove('disabled')
 }
 
+function enableRunButton() {
+ const runButton = document.getElementById('run')
+ runButton.value = 'Ok'
+ runButton.disabled = false
+ runButton.classList.remove('disabled')
+}
+
 function reloadAll() {
  updateParams();
  drawAll();
  saveState();
- disableReloadButton()
+ disableReloadButton();
+ disableRunButton();
 }
 
 document.getElementById('params').onsubmit = function(event) {

src/Access/Common/AccessRightsElement.cpp

@@ -155,7 +155,7 @@ namespace
 
 
 AccessRightsElement::AccessRightsElement(AccessFlags access_flags_, std::string_view database_)
- : access_flags(access_flags_), database(database_), any_database(false)
+ : access_flags(access_flags_), database(database_), parameter(database_), any_database(false), any_parameter(false)
 {
 }
 

src/Access/Common/AccessType.h

@@ -70,7 +70,7 @@ enum class AccessType
  M(ALTER_FREEZE_PARTITION, "FREEZE PARTITION, UNFREEZE", TABLE, ALTER_TABLE) \
  \
  M(ALTER_DATABASE_SETTINGS, "ALTER DATABASE SETTING, ALTER MODIFY DATABASE SETTING, MODIFY DATABASE SETTING", DATABASE, ALTER_DATABASE) /* allows to execute ALTER MODIFY SETTING */\
- M(ALTER_NAMED_COLLECTION, "", NAMED_COLLECTION, NAMED_COLLECTION_CONTROL) /* allows to execute ALTER NAMED COLLECTION */\
+ M(ALTER_NAMED_COLLECTION, "", NAMED_COLLECTION, NAMED_COLLECTION_ADMIN) /* allows to execute ALTER NAMED COLLECTION */\
  \
  M(ALTER_TABLE, "", GROUP, ALTER) \
  M(ALTER_DATABASE, "", GROUP, ALTER) \
@@ -92,7 +92,7 @@ enum class AccessType
  M(CREATE_ARBITRARY_TEMPORARY_TABLE, "", GLOBAL, CREATE) /* allows to create and manipulate temporary tables
  with arbitrary table engine */\
  M(CREATE_FUNCTION, "", GLOBAL, CREATE) /* allows to execute CREATE FUNCTION */ \
- M(CREATE_NAMED_COLLECTION, "", NAMED_COLLECTION, NAMED_COLLECTION_CONTROL) /* allows to execute CREATE NAMED COLLECTION */ \
+ M(CREATE_NAMED_COLLECTION, "", NAMED_COLLECTION, NAMED_COLLECTION_ADMIN) /* allows to execute CREATE NAMED COLLECTION */ \
  M(CREATE, "", GROUP, ALL) /* allows to execute {CREATE|ATTACH} */ \
  \
  M(DROP_DATABASE, "", DATABASE, DROP) /* allows to execute {DROP|DETACH} DATABASE */\
@@ -101,7 +101,7 @@ enum class AccessType
  implicitly enabled by the grant DROP_TABLE */\
  M(DROP_DICTIONARY, "", DICTIONARY, DROP) /* allows to execute {DROP|DETACH} DICTIONARY */\
  M(DROP_FUNCTION, "", GLOBAL, DROP) /* allows to execute DROP FUNCTION */\
- M(DROP_NAMED_COLLECTION, "", NAMED_COLLECTION, NAMED_COLLECTION_CONTROL) /* allows to execute DROP NAMED COLLECTION */\
+ M(DROP_NAMED_COLLECTION, "", NAMED_COLLECTION, NAMED_COLLECTION_ADMIN) /* allows to execute DROP NAMED COLLECTION */\
  M(DROP, "", GROUP, ALL) /* allows to execute {DROP|DETACH} */\
  \
  M(UNDROP_TABLE, "", TABLE, ALL) /* allows to execute {UNDROP} TABLE */\
@@ -140,9 +140,10 @@ enum class AccessType
  M(SHOW_SETTINGS_PROFILES, "SHOW PROFILES, SHOW CREATE SETTINGS PROFILE, SHOW CREATE PROFILE", GLOBAL, SHOW_ACCESS) \
  M(SHOW_ACCESS, "", GROUP, ACCESS_MANAGEMENT) \
  M(ACCESS_MANAGEMENT, "", GROUP, ALL) \
- M(SHOW_NAMED_COLLECTIONS, "SHOW NAMED COLLECTIONS", NAMED_COLLECTION, NAMED_COLLECTION_CONTROL) \
- M(SHOW_NAMED_COLLECTIONS_SECRETS, "SHOW NAMED COLLECTIONS SECRETS", NAMED_COLLECTION, NAMED_COLLECTION_CONTROL) \
- M(NAMED_COLLECTION_CONTROL, "", NAMED_COLLECTION, ALL) \
+ M(SHOW_NAMED_COLLECTIONS, "SHOW NAMED COLLECTIONS", NAMED_COLLECTION, NAMED_COLLECTION_ADMIN) \
+ M(SHOW_NAMED_COLLECTIONS_SECRETS, "SHOW NAMED COLLECTIONS SECRETS", NAMED_COLLECTION, NAMED_COLLECTION_ADMIN) \
+ M(NAMED_COLLECTION, "NAMED COLLECTION USAGE, USE NAMED COLLECTION", NAMED_COLLECTION, NAMED_COLLECTION_ADMIN) \
+ M(NAMED_COLLECTION_ADMIN, "NAMED COLLECTION CONTROL", NAMED_COLLECTION, ALL) \
  \
  M(SYSTEM_SHUTDOWN, "SYSTEM KILL, SHUTDOWN", GLOBAL, SYSTEM) \
  M(SYSTEM_DROP_DNS_CACHE, "SYSTEM DROP DNS, DROP DNS CACHE, DROP DNS", GLOBAL, SYSTEM_DROP_CACHE) \

src/Access/UsersConfigAccessStorage.cpp

@@ -328,7 +328,7 @@ namespace
 
  if (!named_collection_control)
  {
- user->access.revoke(AccessType::NAMED_COLLECTION_CONTROL);
+ user->access.revoke(AccessType::NAMED_COLLECTION_ADMIN);
  }
 
  if (!show_named_collections_secrets)

src/Access/tests/gtest_access_rights_ops.cpp

@@ -53,7 +53,7 @@ TEST(AccessRights, Union)
  "SHOW ROW POLICIES, SYSTEM MERGES, SYSTEM TTL MERGES, SYSTEM FETCHES, "
  "SYSTEM MOVES, SYSTEM SENDS, SYSTEM REPLICATION QUEUES, "
  "SYSTEM DROP REPLICA, SYSTEM SYNC REPLICA, SYSTEM RESTART REPLICA, "
- "SYSTEM RESTORE REPLICA, SYSTEM WAIT LOADING PARTS, SYSTEM SYNC DATABASE REPLICA, SYSTEM FLUSH DISTRIBUTED, dictGet ON db1.*, GRANT NAMED COLLECTION CONTROL ON db1");
+ "SYSTEM RESTORE REPLICA, SYSTEM WAIT LOADING PARTS, SYSTEM SYNC DATABASE REPLICA, SYSTEM FLUSH DISTRIBUTED, dictGet ON db1.*, GRANT NAMED COLLECTION ADMIN ON db1");
 }
 
 

src/Client/ClientBase.cpp

@@ -2297,7 +2297,9 @@ void ClientBase::runInteractive()
  catch (const ErrnoException & e)
  {
  if (e.getErrno() != EEXIST)
- throw;
+ {
+ std::cerr << getCurrentExceptionMessage(false) << '\n';
+ }
  }
  }
 

src/Client/ConnectionParameters.cpp

@@ -60,7 +60,15 @@ ConnectionParameters::ConnectionParameters(const Poco::Util::AbstractConfigurati
  quota_key = config.getString("quota_key", "");
 
  /// By default compression is disabled if address looks like localhost.
- compression = config.getBool("compression", !isLocalAddress(DNSResolver::instance().resolveHost(host)))
+
+ /// Avoid DNS request if the host is "localhost".
+ /// If ClickHouse is run under QEMU-user with a binary for a different architecture,
+ /// and there are all listed startup dependency shared libraries available, but not the runtime dependencies of glibc,
+ /// the glibc cannot open "plugins" for DNS resolving, and the DNS resolution does not work.
+ /// At the same time, I want clickhouse-local to always work, regardless.
+ /// TODO: get rid of glibc, or replace getaddrinfo to c-ares.
+
+ compression = config.getBool("compression", host != "localhost" && !isLocalAddress(DNSResolver::instance().resolveHost(host)))
  ? Protocol::Compression::Enable : Protocol::Compression::Disable;
 
  timeouts = ConnectionTimeouts(

src/Client/Suggest.cpp

@@ -101,9 +101,8 @@ static String getLoadSuggestionQuery(Int32 suggestion_limit, bool basic_suggesti
  add_column("name", "columns", true, suggestion_limit);
  }
 
- /// FIXME: Forbid this query using new analyzer because of bug https://github.com/ClickHouse/ClickHouse/issues/50669
- /// We should remove this restriction after resolving this bug.
- query = "SELECT DISTINCT arrayJoin(extractAll(name, '[\\\\w_]{2,}')) AS res FROM (" + query + ") WHERE notEmpty(res) SETTINGS allow_experimental_analyzer=0";
+ /// FIXME: This query does not work with the new analyzer because of bug https://github.com/ClickHouse/ClickHouse/issues/50669
+ query = "SELECT DISTINCT arrayJoin(extractAll(name, '[\\\\w_]{2,}')) AS res FROM (" + query + ") WHERE notEmpty(res)";
  return query;
 }