Merge branch 'master' of github.com:ClickHouse/ClickHouse into clang-…

…18-ci

README.md

@@ -40,7 +40,6 @@ Every month we get together with the community (users, contributors, customers,
 
 Keep an eye out for upcoming meetups and events around the world. Somewhere else you want us to be? Please feel free to reach out to tyler `<at>` clickhouse `<dot>` com. You can also peruse [ClickHouse Events](https://clickhouse.com/company/news-events) for a list of all upcoming trainings, meetups, speaking engagements, etc.
 
-* [ClickHouse Meetup in Bengaluru](https://www.meetup.com/clickhouse-bangalore-user-group/events/300405581/) - May 4
 * [ClickHouse Happy Hour @ Tom's Watch Bar - Los Angeles](https://www.meetup.com/clickhouse-los-angeles-user-group/events/300740584/) - May 22
 * [ClickHouse & Confluent Meetup in Dubai](https://www.meetup.com/clickhouse-dubai-meetup-group/events/299629189/) - May 28
 * [ClickHouse Meetup in Stockholm](https://www.meetup.com/clickhouse-stockholm-user-group/events/299752651/) - Jun 3
@@ -49,6 +48,7 @@ Keep an eye out for upcoming meetups and events around the world. Somewhere else
 * [ClickHouse Meetup in Amsterdam](https://www.meetup.com/clickhouse-netherlands-user-group/events/300781068/) - Jun 27
 * [ClickHouse Meetup in Paris](https://www.meetup.com/clickhouse-france-user-group/events/300783448/) - Jul 9
 * [ClickHouse Meetup @ Ramp - New York City](https://www.meetup.com/clickhouse-new-york-user-group/events/300595845/) - Jul 9
+* [ClickHouse Meetup @ Klaviyo - Boston](https://www.meetup.com/clickhouse-boston-user-group/events/300907870) - Jul 11
 
 ## Recent Recordings
 * **Recent Meetup Videos**: [Meetup Playlist](https://www.youtube.com/playlist?list=PL0Z2YDlm0b3iNDUzpY1S3L_iV4nARda_U) Whenever possible recordings of the ClickHouse Community Meetups are edited and presented as individual talks. Current featuring "Modern SQL in 2023", "Fast, Concurrent, and Consistent Asynchronous INSERTS in ClickHouse", and "Full-Text Indices: Design and Experiments"

docker/server/Dockerfile.ubuntu

@@ -1,11 +1,14 @@
 FROM ubuntu:20.04
 
 # see https://github.com/moby/moby/issues/4032#issuecomment-192327844
+# It could be removed after we move on a version 23:04+
 ARG DEBIAN_FRONTEND=noninteractive
 
 # ARG for quick switch to a given ubuntu mirror
 ARG apt_archive="http://archive.ubuntu.com"
 
+# We shouldn't use `apt upgrade` to not change the upstream image. It's updated biweekly
+
 # user/group precreated explicitly with fixed uid/gid on purpose.
 # It is especially important for rootless containers: in that case entrypoint
 # can't do chown and owners of mounted volumes should be configured externally.
@@ -16,20 +19,21 @@ RUN sed -i "s|http://archive.ubuntu.com|${apt_archive}|g" /etc/apt/sources.list
     && groupadd -r clickhouse --gid=101 \
     && useradd -r -g clickhouse --uid=101 --home-dir=/var/lib/clickhouse --shell=/bin/bash clickhouse \
     && apt-get update \
-    && apt-get upgrade -yq \
     && apt-get install --yes --no-install-recommends \
         ca-certificates \
         locales \
         tzdata \
         wget \
-    && apt-get clean \
     && rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/*
 
 ARG REPO_CHANNEL="stable"
 ARG REPOSITORY="deb [signed-by=/usr/share/keyrings/clickhouse-keyring.gpg] https://packages.clickhouse.com/deb ${REPO_CHANNEL} main"
 ARG VERSION="24.4.1.2088"
 ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static"
 
+#docker-official-library:off
+# The part between `docker-official-library` tags is related to our builds
+
 # set non-empty deb_location_url url to create a docker image
 # from debs created by CI build, for example:
 # docker build . --network host --build-arg version="21.4.1.6282" --build-arg deb_location_url="https://..." -t ...
@@ -80,19 +84,22 @@ RUN if [ -n "${single_binary_location_url}" ]; then \
         && rm -rf /tmp/* ; \
     fi
 
+# The rest is the same in the official docker and in our build system
+#docker-official-library:on
+
 # A fallback to installation from ClickHouse repository
 RUN if ! clickhouse local -q "SELECT ''" > /dev/null 2>&1; then \
         apt-get update \
         && apt-get install --yes --no-install-recommends \
             apt-transport-https \
-            ca-certificates \
             dirmngr \
             gnupg2 \
         && mkdir -p /etc/apt/sources.list.d \
         && GNUPGHOME=$(mktemp -d) \
-        && GNUPGHOME="$GNUPGHOME" gpg --no-default-keyring \
+        && GNUPGHOME="$GNUPGHOME" gpg --batch --no-default-keyring \
             --keyring /usr/share/keyrings/clickhouse-keyring.gpg \
-            --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 8919F6BD2B48D754 \
+            --keyserver hkp://keyserver.ubuntu.com:80 \
+            --recv-keys 3a9ea1193a97b548be1457d48919f6bd2b48d754 \
         && rm -rf "$GNUPGHOME" \
         && chmod +r /usr/share/keyrings/clickhouse-keyring.gpg \
         && echo "${REPOSITORY}" > /etc/apt/sources.list.d/clickhouse.list \
@@ -127,7 +134,6 @@ RUN mkdir /docker-entrypoint-initdb.d
 
 COPY docker_related_config.xml /etc/clickhouse-server/config.d/
 COPY entrypoint.sh /entrypoint.sh
-RUN chmod +x /entrypoint.sh
 
 EXPOSE 9000 8123 9009
 VOLUME /var/lib/clickhouse

docker/server/README.md

@@ -4,33 +4,34 @@
 
 ClickHouse is an open-source column-oriented DBMS (columnar database management system) for online analytical processing (OLAP) that allows users to generate analytical reports using SQL queries in real-time.
 
-ClickHouse works 100-1000x faster than traditional database management systems, and processes hundreds of millions to over a billion rows and tens of gigabytes of data per server per second.  With a widespread user base around the globe, the technology has received praise for its reliability, ease of use, and fault tolerance.
+ClickHouse works 100-1000x faster than traditional database management systems, and processes hundreds of millions to over a billion rows and tens of gigabytes of data per server per second. With a widespread user base around the globe, the technology has received praise for its reliability, ease of use, and fault tolerance.
 
 For more information and documentation see https://clickhouse.com/.
 
 ## Versions
 
-- The `latest` tag points to the latest release of the latest stable branch.
-- Branch tags like `22.2` point to the latest release of the corresponding branch.
-- Full version tags like `22.2.3.5` point to the corresponding release.
-- The tag `head` is built from the latest commit to the default branch.
-- Each tag has optional `-alpine` suffix to reflect that it's built on top of `alpine`.
+-	The `latest` tag points to the latest release of the latest stable branch.
+-	Branch tags like `22.2` point to the latest release of the corresponding branch.
+-	Full version tags like `22.2.3.5` point to the corresponding release.
+-	The tag `head` is built from the latest commit to the default branch.
+-	Each tag has optional `-alpine` suffix to reflect that it's built on top of `alpine`.
 
 ### Compatibility
 
-- The amd64 image requires support for [SSE3 instructions](https://en.wikipedia.org/wiki/SSE3). Virtually all x86 CPUs after 2005 support SSE3.
-- The arm64 image requires support for the [ARMv8.2-A architecture](https://en.wikipedia.org/wiki/AArch64#ARMv8.2-A) and additionally the Load-Acquire RCpc register. The register is optional in version ARMv8.2-A and mandatory in [ARMv8.3-A](https://en.wikipedia.org/wiki/AArch64#ARMv8.3-A). Supported in Graviton >=2, Azure and GCP instances. Examples for unsupported devices are Raspberry Pi 4 (ARMv8.0-A) and Jetson AGX Xavier/Orin (ARMv8.2-A).
+-	The amd64 image requires support for [SSE3 instructions](https://en.wikipedia.org/wiki/SSE3). Virtually all x86 CPUs after 2005 support SSE3.
+-	The arm64 image requires support for the [ARMv8.2-A architecture](https://en.wikipedia.org/wiki/AArch64#ARMv8.2-A) and additionally the Load-Acquire RCpc register. The register is optional in version ARMv8.2-A and mandatory in [ARMv8.3-A](https://en.wikipedia.org/wiki/AArch64#ARMv8.3-A). Supported in Graviton >=2, Azure and GCP instances. Examples for unsupported devices are Raspberry Pi 4 (ARMv8.0-A) and Jetson AGX Xavier/Orin (ARMv8.2-A).
 
 ## How to use this image
 
 ### start server instance
+
 ```bash
 docker run -d --name some-clickhouse-server --ulimit nofile=262144:262144 clickhouse/clickhouse-server
 ```
 
 By default, ClickHouse will be accessible only via the Docker network. See the [networking section below](#networking).
 
-By default, starting above server instance will be run as the `default` user without  password.
+By default, starting above server instance will be run as the `default` user without password.
 
 ### connect to it from a native client
 
@@ -66,9 +67,7 @@ docker run -d -p 18123:8123 -p19000:9000 --name some-clickhouse-server --ulimit
 echo 'SELECT version()' | curl 'http://localhost:18123/' --data-binary @-
 ```
 
-```
-22.6.3.35
-```
+`22.6.3.35`
 
 or by allowing the container to use [host ports directly](https://docs.docker.com/network/host/) using `--network=host` (also allows achieving better network performance):
 
@@ -77,16 +76,14 @@ docker run -d --network=host --name some-clickhouse-server --ulimit nofile=26214
 echo 'SELECT version()' | curl 'http://localhost:8123/' --data-binary @-
 ```
 
-```
-22.6.3.35
-```
+`22.6.3.35`
 
 ### Volumes
 
 Typically you may want to mount the following folders inside your container to achieve persistency:
 
-* `/var/lib/clickhouse/` - main folder where ClickHouse stores the data
-* `/var/log/clickhouse-server/` - logs
+-	`/var/lib/clickhouse/` - main folder where ClickHouse stores the data
+-	`/var/log/clickhouse-server/` - logs
 
 ```bash
 docker run -d \
@@ -97,9 +94,9 @@ docker run -d \
 
 You may also want to mount:
 
-* `/etc/clickhouse-server/config.d/*.xml` - files with server configuration adjustments
-* `/etc/clickhouse-server/users.d/*.xml` - files with user settings adjustments
-* `/docker-entrypoint-initdb.d/` - folder with database initialization scripts (see below).
+-	`/etc/clickhouse-server/config.d/*.xml` - files with server configuration adjustments
+-	`/etc/clickhouse-server/users.d/*.xml` - files with user settings adjustments
+-	`/docker-entrypoint-initdb.d/` - folder with database initialization scripts (see below).
 
 ### Linux capabilities
 
@@ -150,7 +147,7 @@ docker run --rm -e CLICKHOUSE_DB=my_database -e CLICKHOUSE_USER=username -e CLIC
 
 ## How to extend this image
 
-To perform additional initialization in an image derived from this one, add one or more `*.sql`, `*.sql.gz`, or `*.sh` scripts under `/docker-entrypoint-initdb.d`. After the entrypoint calls `initdb`, it will run any `*.sql` files, run any executable `*.sh` scripts, and source any non-executable `*.sh` scripts found in that directory to do further initialization before starting the service.
+To perform additional initialization in an image derived from this one, add one or more `*.sql`, `*.sql.gz`, or `*.sh` scripts under `/docker-entrypoint-initdb.d`. After the entrypoint calls `initdb`, it will run any `*.sql` files, run any executable `*.sh` scripts, and source any non-executable `*.sh` scripts found in that directory to do further initialization before starting the service.  
 Also, you can provide environment variables `CLICKHOUSE_USER` & `CLICKHOUSE_PASSWORD` that will be used for clickhouse-client during initialization.
 
 For example, to add an additional user and database, add the following to `/docker-entrypoint-initdb.d/init-db.sh`:

docs/en/engines/database-engines/materialized-mysql.md

@@ -51,6 +51,9 @@ ENGINE = MaterializedMySQL('host:port', ['database' | database], 'user', 'passwo
 ### allows_query_when_mysql_lost
 `allows_query_when_mysql_lost` — Allows to query a materialized table when MySQL is lost. Default: `0` (`false`).
 
+### allow_startup_database_without_connection_to_mysql
+`allow_startup_database_without_connection_to_mysql` — Allow to create and attach database without available connection to MySQL. Default: `0` (`false`).
+
 ### materialized_mysql_tables_list
 
 `materialized_mysql_tables_list` — a comma-separated list of mysql database tables, which will be replicated by MaterializedMySQL database engine. Default value: empty list — means whole tables will be replicated.

docs/en/sql-reference/functions/string-functions.md

@@ -1058,7 +1058,7 @@ convertCharset(s, from, to)
 
 ## base58Encode
 
-Encodes a String using [Base58](https://tools.ietf.org/id/draft-msporny-base58-01.html) in the "Bitcoin" alphabet.
+Encodes a String using [Base58](https://datatracker.ietf.org/doc/html/draft-msporny-base58) in the "Bitcoin" alphabet.
 
 **Syntax**
 
@@ -1092,7 +1092,7 @@ Result:
 
 ## base58Decode
 
-Accepts a String and decodes it using [Base58](https://tools.ietf.org/id/draft-msporny-base58-01.html) encoding scheme using "Bitcoin" alphabet.
+Accepts a String and decodes it using [Base58](https://datatracker.ietf.org/doc/html/draft-msporny-base58) encoding scheme using "Bitcoin" alphabet.
 
 **Syntax**
 

docs/ru/sql-reference/functions/string-functions.md

@@ -493,7 +493,7 @@ SELECT concat(key1, key2), sum(value) FROM key_val GROUP BY (key1, key2);
 
 ## base58Encode(plaintext), base58Decode(encoded_text) {#base58}
 
-Принимает на вход строку или колонку строк и кодирует/раскодирует их с помощью схемы кодирования [Base58](https://tools.ietf.org/id/draft-msporny-base58-01.html) с использованием стандартного алфавита Bitcoin.
+Принимает на вход строку или колонку строк и кодирует/раскодирует их с помощью схемы кодирования [Base58](https://datatracker.ietf.org/doc/html/draft-msporny-base58) с использованием стандартного алфавита Bitcoin.
 
 **Синтаксис**
 

programs/server/dashboard.html

@@ -538,9 +538,57 @@
 
 /// Palette generation for charts
 function generatePalette(numColors) {
+    // oklch() does not work in firefox<=125 inside <canvas> element so we convert it back to rgb for now.
+    // Based on https://github.com/color-js/color.js/blob/main/src/spaces/oklch.js
+    const multiplyMatrices = (A, B) => {
+        return [
+            A[0]*B[0] + A[1]*B[1] + A[2]*B[2],
+            A[3]*B[0] + A[4]*B[1] + A[5]*B[2],
+            A[6]*B[0] + A[7]*B[1] + A[8]*B[2]
+        ];
+    }
+
+    const oklch2oklab = ([l, c, h]) => [
+        l,
+        isNaN(h) ? 0 : c * Math.cos(h * Math.PI / 180),
+        isNaN(h) ? 0 : c * Math.sin(h * Math.PI / 180)
+    ]
+
+    const srgbLinear2rgb = rgb => rgb.map(c =>
+        Math.abs(c) > 0.0031308 ?
+            (c < 0 ? -1 : 1) * (1.055 * (Math.abs(c) ** (1 / 2.4)) - 0.055) :
+            12.92 * c
+    )
+
+    const oklab2xyz = lab => {
+        const LMSg = multiplyMatrices([
+            1,  0.3963377773761749,  0.2158037573099136,
+            1, -0.1055613458156586, -0.0638541728258133,
+            1, -0.0894841775298119, -1.2914855480194092,
+        ], lab)
+        const LMS = LMSg.map(val => val ** 3)
+        return multiplyMatrices([
+            1.2268798758459243,  -0.5578149944602171,  0.2813910456659647,
+            -0.0405757452148008,  1.1122868032803170, -0.0717110580655164,
+            -0.0763729366746601, -0.4214933324022432,  1.5869240198367816
+        ], LMS)
+    }
+
+    const xyz2rgbLinear = xyz => {
+        return multiplyMatrices([
+            3.2409699419045226,  -1.537383177570094,   -0.4986107602930034,
+           -0.9692436362808796,   1.8759675015077202,   0.04155505740717559,
+            0.05563007969699366, -0.20397695888897652,  1.0569715142428786
+        ], xyz)
+    }
+
+    const oklch2rgb = lch =>  srgbLinear2rgb(xyz2rgbLinear(oklab2xyz(oklch2oklab(lch))))
+
     palette = [];
     for (let i = 0; i < numColors; i++) {
-        palette.push(`oklch(${theme != 'dark' ? 0.75 : 0.5}, 0.15, ${360 * i / numColors})`);
+        //palette.push(`oklch(${theme != 'dark' ? 0.75 : 0.5}, 0.15, ${360 * i / numColors})`);
+        let rgb = oklch2rgb([theme != 'dark' ? 0.75 : 0.5, 0.15, 360 * i / numColors]);
+        palette.push(`rgb(${rgb[0] * 255}, ${rgb[1] * 255}, ${rgb[2] * 255})`);
     }
     return palette;
 }

src/AggregateFunctions/AggregateFunctionSumMap.cpp

@@ -711,7 +711,7 @@ auto parseArguments(const std::string & name, const DataTypes & arguments)
 
     const auto * array_type = checkAndGetDataType<DataTypeArray>(args[0].get());
     if (!array_type)
-        throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "First argument for function {} must be an array, not {}",
+        throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Argument #1 for function {} must be an array, not {}",
             name, args[0]->getName());
 
     DataTypePtr keys_type = array_type->getNestedType();
@@ -722,8 +722,8 @@ auto parseArguments(const std::string & name, const DataTypes & arguments)
     {
         array_type = checkAndGetDataType<DataTypeArray>(args[i].get());
         if (!array_type)
-            throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Argument #{} for function {} must be an array.",
-                i, name);
+            throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Argument #{} for function {} must be an array, not {}",
+                i + 1, name, args[i]->getName());
         values_types.push_back(array_type->getNestedType());
     }
 

src/Columns/ColumnTuple.cpp

@@ -164,7 +164,7 @@ bool ColumnTuple::tryInsert(const Field & x)
         if (!columns[i]->tryInsert(tuple[i]))
         {
             for (size_t j = 0; j != i; ++j)
-                columns[i]->popBack(1);
+                columns[j]->popBack(1);
 
             return false;
         }

src/Databases/MySQL/DatabaseMaterializedMySQL.cpp

@@ -84,7 +84,8 @@ LoadTaskPtr DatabaseMaterializedMySQL::startupDatabaseAsync(AsyncLoader & async_
         [this, mode] (AsyncLoader &, const LoadJobPtr &)
         {
             LOG_TRACE(log, "Starting MaterializeMySQL database");
-            if (mode < LoadingStrictnessLevel::FORCE_ATTACH)
+            if (!settings->allow_startup_database_without_connection_to_mysql
+                && mode < LoadingStrictnessLevel::FORCE_ATTACH)
                 materialize_thread.assertMySQLAvailable();
 
             materialize_thread.startSynchronization();

src/Databases/MySQL/MaterializedMySQLSettings.h

@@ -22,6 +22,7 @@ class ASTStorage;
     M(UInt64, max_milliseconds_to_wait_in_binlog_queue, 10000, "Max milliseconds to wait when max bytes exceeded in a binlog queue.", 0) \
     M(UInt64, max_bytes_in_binlog_dispatcher_buffer, DBMS_DEFAULT_BUFFER_SIZE, "Max bytes in the binlog dispatcher's buffer before it is flushed to attached binlogs.", 0) \
     M(UInt64, max_flush_milliseconds_in_binlog_dispatcher, 1000, "Max milliseconds in the binlog dispatcher's buffer to wait before it is flushed to attached binlogs.", 0) \
+    M(Bool, allow_startup_database_without_connection_to_mysql, false, "Allow to create and attach database without available connection to MySQL.", 0) \
 
     DECLARE_SETTINGS_TRAITS(MaterializedMySQLSettingsTraits, LIST_OF_MATERIALIZE_MODE_SETTINGS)
 

src/Storages/HDFS/StorageHDFS.cpp

@@ -1101,6 +1101,10 @@ SinkToStoragePtr StorageHDFS::write(const ASTPtr & query, const StorageMetadataP
 
     if (is_partitioned_implementation)
     {
+        String path = current_uri.substr(current_uri.find('/', current_uri.find("//") + 2));
+        if (PartitionedSink::replaceWildcards(path, "").find_first_of("*?{") != std::string::npos)
+            throw Exception(ErrorCodes::DATABASE_ACCESS_DENIED, "URI '{}' contains globs, so the table is in readonly mode", uris.back());
+
         return std::make_shared<PartitionedHDFSSink>(
             partition_by_ast,
             current_uri,