Merge branch 'master' of github.com:ClickHouse/ClickHouse into clang-…

…18-ci

.gitmodules

@@ -6,7 +6,7 @@
 	url = https://github.com/facebook/zstd
 [submodule "contrib/lz4"]
 	path = contrib/lz4
-	url = https://github.com/ClickHouse/lz4
+	url = https://github.com/lz4/lz4
 [submodule "contrib/librdkafka"]
 	path = contrib/librdkafka
 	url = https://github.com/ClickHouse/librdkafka

docker/test/integration/runner/Dockerfile

@@ -101,7 +101,8 @@ RUN python3 -m pip install --no-cache-dir \
     retry==0.9.2 \
     bs4==0.0.2 \
     lxml==5.1.0 \
-    urllib3==2.0.7
+    urllib3==2.0.7 \
+    jwcrypto==1.5.6
 # bs4, lxml are for cloud tests, do not delete
 
 # Hudi supports only spark 3.3.*, not 3.4

docs/en/engines/table-engines/mergetree-family/replication.md

@@ -10,7 +10,7 @@ sidebar_label: Data Replication
 In ClickHouse Cloud replication is managed for you. Please create your tables without adding arguments.  For example, in the text below you would replace:
 
 ```sql
-ENGINE = ReplicatedReplacingMergeTree(
+ENGINE = ReplicatedMergeTree(
     '/clickhouse/tables/{shard}/table_name',
     '{replica}',
     ver
@@ -20,7 +20,7 @@ ENGINE = ReplicatedReplacingMergeTree(
 with:
 
 ```sql
-ENGINE = ReplicatedReplacingMergeTree
+ENGINE = ReplicatedMergeTree
 ```
 :::
 
@@ -140,11 +140,11 @@ The system monitors data synchronicity on replicas and is able to recover after
 :::note
 In ClickHouse Cloud replication is managed for you. Please create your tables without adding arguments.  For example, in the text below you would replace:
 ```
-ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/{shard}/table_name', '{replica}', ver)
+ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/table_name', '{replica}', ver)
 ```
 with:
 ```
-ENGINE = ReplicatedReplacingMergeTree
+ENGINE = ReplicatedMergeTree
 ```
 :::
 
@@ -177,7 +177,7 @@ CREATE TABLE table_name
     CounterID UInt32,
     UserID UInt32,
     ver UInt16
-) ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/{shard}/table_name', '{replica}', ver)
+) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/table_name', '{replica}', ver)
 PARTITION BY toYYYYMM(EventDate)
 ORDER BY (CounterID, EventDate, intHash32(UserID))
 SAMPLE BY intHash32(UserID);

src/Columns/ColumnArray.cpp

@@ -1283,7 +1283,7 @@ ColumnPtr ColumnArray::replicateTuple(const Offsets & replicate_offsets) const
 
 size_t ColumnArray::getNumberOfDimensions() const
 {
-    const auto * nested_array = checkAndGetColumn<ColumnArray>(*data);
+    const auto * nested_array = checkAndGetColumn<ColumnArray>(&*data);
     if (!nested_array)
         return 1;
     return 1 + nested_array->getNumberOfDimensions();   /// Every modern C++ compiler optimizes tail recursion.

src/Columns/ColumnLowCardinality.cpp

@@ -903,7 +903,7 @@ ColumnPtr ColumnLowCardinality::cloneWithDefaultOnNull() const
 
 bool isColumnLowCardinalityNullable(const IColumn & column)
 {
-    if (const auto * lc_column = checkAndGetColumn<ColumnLowCardinality>(column))
+    if (const auto * lc_column = checkAndGetColumn<ColumnLowCardinality>(&column))
         return lc_column->nestedIsNullable();
     return false;
 }

src/Columns/ColumnUnique.h

@@ -376,7 +376,7 @@ size_t ColumnUnique<ColumnType>::uniqueInsertFrom(const IColumn & src, size_t n)
     if (is_nullable && src.isNullAt(n))
         return getNullValueIndex();
 
-    if (const auto * nullable = checkAndGetColumn<ColumnNullable>(src))
+    if (const auto * nullable = checkAndGetColumn<ColumnNullable>(&src))
         return uniqueInsertFrom(nullable->getNestedColumn(), n);
 
     auto ref = src.getDataAt(n);
@@ -569,7 +569,7 @@ MutableColumnPtr ColumnUnique<ColumnType>::uniqueInsertRangeImpl(
         return nullptr;
     };
 
-    if (const auto * nullable_column = checkAndGetColumn<ColumnNullable>(src))
+    if (const auto * nullable_column = checkAndGetColumn<ColumnNullable>(&src))
     {
         src_column = typeid_cast<const ColumnType *>(&nullable_column->getNestedColumn());
         null_map = &nullable_column->getNullMapData();

src/Columns/FilterDescription.cpp

@@ -32,7 +32,7 @@ ConstantFilterDescription::ConstantFilterDescription(const IColumn & column)
 
         if (!typeid_cast<const ColumnUInt8 *>(column_nested.get()))
         {
-            const ColumnNullable * column_nested_nullable = checkAndGetColumn<ColumnNullable>(*column_nested);
+            const ColumnNullable * column_nested_nullable = checkAndGetColumn<ColumnNullable>(&*column_nested);
             if (!column_nested_nullable || !typeid_cast<const ColumnUInt8 *>(&column_nested_nullable->getNestedColumn()))
             {
                 throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_COLUMN_FOR_FILTER,
@@ -66,7 +66,7 @@ FilterDescription::FilterDescription(const IColumn & column_)
         return;
     }
 
-    if (const auto * nullable_column = checkAndGetColumn<ColumnNullable>(column))
+    if (const auto * nullable_column = checkAndGetColumn<ColumnNullable>(&column))
     {
         ColumnPtr nested_column = nullable_column->getNestedColumnPtr();
         MutableColumnPtr mutable_holder = IColumn::mutate(std::move(nested_column));

src/Columns/IColumn.h

@@ -640,12 +640,16 @@ template <>
 struct IsMutableColumns<> { static const bool value = true; };
 
 
+/// Throws LOGICAL_ERROR if the type doesn't match.
 template <typename Type>
-const Type * checkAndGetColumn(const IColumn & column)
+const Type & checkAndGetColumn(const IColumn & column)
 {
-    return typeid_cast<const Type *>(&column);
+    return typeid_cast<const Type &>(column);
 }
 
+/// Returns nullptr if the type doesn't match.
+/// If you're going to dereference the returned pointer without checking for null, use the
+/// `const IColumn &` overload above instead.
 template <typename Type>
 const Type * checkAndGetColumn(const IColumn * column)
 {

src/Columns/MaskOperations.cpp

@@ -205,10 +205,10 @@ static MaskInfo extractMaskImpl(
     auto column = col->convertToFullColumnIfLowCardinality();
 
     /// Special implementation for Null and Const columns.
-    if (column->onlyNull() || checkAndGetColumn<ColumnConst>(*column))
+    if (column->onlyNull() || checkAndGetColumn<ColumnConst>(&*column))
         return extractMaskFromConstOrNull<inverted>(mask, column, null_value, nulls);
 
-    if (const auto * nullable_column = checkAndGetColumn<ColumnNullable>(*column))
+    if (const auto * nullable_column = checkAndGetColumn<ColumnNullable>(&*column))
     {
         const PaddedPODArray<UInt8> & null_map = nullable_column->getNullMapData();
         return extractMaskImpl<inverted>(mask, nullable_column->getNestedColumnPtr(), null_value, &null_map, nulls);

src/Common/ColumnsHashing.h

@@ -44,8 +44,8 @@ struct HashMethodOneNumber
     {
         if constexpr (nullable)
         {
-            const auto * null_column = checkAndGetColumn<ColumnNullable>(key_columns[0]);
-            vec = null_column->getNestedColumnPtr()->getRawData().data();
+            const auto & null_column = checkAndGetColumn<ColumnNullable>(*key_columns[0]);
+            vec = null_column.getNestedColumnPtr()->getRawData().data();
         }
         else
         {
@@ -57,8 +57,8 @@ struct HashMethodOneNumber
     {
         if constexpr (nullable)
         {
-            const auto * null_column = checkAndGetColumn<ColumnNullable>(column);
-            vec = null_column->getNestedColumnPtr()->getRawData().data();
+            const auto & null_column = checkAndGetColumn<ColumnNullable>(*column);
+            vec = null_column.getNestedColumnPtr()->getRawData().data();
         }
         else
         {
@@ -105,7 +105,7 @@ struct HashMethodString
         const IColumn * column;
         if constexpr (nullable)
         {
-            column = checkAndGetColumn<ColumnNullable>(key_columns[0])->getNestedColumnPtr().get();
+            column = checkAndGetColumn<ColumnNullable>(*key_columns[0]).getNestedColumnPtr().get();
         }
         else
         {
@@ -153,7 +153,7 @@ struct HashMethodFixedString
         const IColumn * column;
         if constexpr (nullable)
         {
-            column = checkAndGetColumn<ColumnNullable>(key_columns[0])->getNestedColumnPtr().get();
+            column = checkAndGetColumn<ColumnNullable>(*key_columns[0]).getNestedColumnPtr().get();
         }
         else
         {

src/Common/ColumnsHashingImpl.h

@@ -305,7 +305,7 @@ class HashMethodBase
         }
 
         if constexpr (nullable)
-            null_map = &checkAndGetColumn<ColumnNullable>(column)->getNullMapColumn();
+            null_map = &checkAndGetColumn<ColumnNullable>(*column).getNullMapColumn();
     }
 
     template <typename Data, typename KeyHolder>

src/Core/DecimalComparison.h

@@ -170,11 +170,11 @@ class DecimalComparison
 
             if (c0_is_const && c1_is_const)
             {
-                const ColumnConst * c0_const = checkAndGetColumnConst<ColVecA>(c0.get());
-                const ColumnConst * c1_const = checkAndGetColumnConst<ColVecB>(c1.get());
+                const ColumnConst & c0_const = checkAndGetColumnConst<ColVecA>(*c0);
+                const ColumnConst & c1_const = checkAndGetColumnConst<ColVecB>(*c1);
 
-                A a = c0_const->template getValue<A>();
-                B b = c1_const->template getValue<B>();
+                A a = c0_const.template getValue<A>();
+                B b = c1_const.template getValue<B>();
                 UInt8 res = apply<scale_left, scale_right>(a, b, scale);
                 return DataTypeUInt8().createColumnConst(c0->size(), toField(res));
             }
@@ -184,17 +184,17 @@ class DecimalComparison
 
             if (c0_is_const)
             {
-                const ColumnConst * c0_const = checkAndGetColumnConst<ColVecA>(c0.get());
-                A a = c0_const->template getValue<A>();
+                const ColumnConst & c0_const = checkAndGetColumnConst<ColVecA>(*c0);
+                A a = c0_const.template getValue<A>();
                 if (const ColVecB * c1_vec = checkAndGetColumn<ColVecB>(c1.get()))
                     constantVector<scale_left, scale_right>(a, c1_vec->getData(), vec_res, scale);
                 else
                     throw Exception(ErrorCodes::LOGICAL_ERROR, "Wrong column in Decimal comparison");
             }
             else if (c1_is_const)
             {
-                const ColumnConst * c1_const = checkAndGetColumnConst<ColVecB>(c1.get());
-                B b = c1_const->template getValue<B>();
+                const ColumnConst & c1_const = checkAndGetColumnConst<ColVecB>(*c1);
+                B b = c1_const.template getValue<B>();
                 if (const ColVecA * c0_vec = checkAndGetColumn<ColVecA>(c0.get()))
                     vectorConstant<scale_left, scale_right>(c0_vec->getData(), b, vec_res, scale);
                 else

src/Daemon/BaseDaemon.cpp

@@ -706,6 +706,8 @@ BaseDaemon::~BaseDaemon()
         }
 
     signal_pipe.close();
+
+    SentryWriter::resetInstance();
 }
 
 

src/Daemon/SentryWriter.cpp

@@ -72,6 +72,10 @@ SentryWriter * SentryWriter::getInstance()
 {
     return SentryWriter::instance.get();
 }
+void SentryWriter::resetInstance()
+{
+    SentryWriter::instance.reset();
+}
 
 SentryWriter::SentryWriter(Poco::Util::LayeredConfiguration & config)
 {
@@ -254,6 +258,7 @@ void SentryWriter::sendError(Type type, int sig_or_error, const std::string & er
 
 void SentryWriter::initializeInstance(Poco::Util::LayeredConfiguration &) {}
 SentryWriter * SentryWriter::getInstance() { return nullptr; }
+void SentryWriter::resetInstance() {}
 
 SentryWriter::SentryWriter(Poco::Util::LayeredConfiguration &) {}
 SentryWriter::~SentryWriter() = default;

src/Daemon/SentryWriter.h

@@ -22,9 +22,16 @@ class SentryWriter
 
     /// Initialize static SentryWriter instance
     static void initializeInstance(Poco::Util::LayeredConfiguration & config);
+
     /// @return nullptr if initializeInstance() was not called (i.e. for non-server) or SentryWriter object
     static SentryWriter * getInstance();
 
+    /// SentryWriter static instance should be reset explicitly to avoid
+    /// possible use-after-free, since it may use some global objects (i.e.
+    /// OpenSSL), while sending final statistics
+    /// (SENTRY_SESSION_STATUS_EXITED).
+    static void resetInstance();
+
     void onSignal(
         int sig,
         const std::string & error_message,

src/DataTypes/ObjectUtils.cpp

@@ -47,7 +47,7 @@ size_t getNumberOfDimensions(const IDataType & type)
 
 size_t getNumberOfDimensions(const IColumn & column)
 {
-    if (const auto * column_array = checkAndGetColumn<ColumnArray>(column))
+    if (const auto * column_array = checkAndGetColumn<ColumnArray>(&column))
         return column_array->getNumberOfDimensions();
     return 0;
 }

src/DataTypes/Serializations/SerializationBool.cpp

@@ -28,15 +28,15 @@ constexpr char str_false[6] = "false";
 const ColumnUInt8 * checkAndGetSerializeColumnType(const IColumn & column)
 {
     const auto * col = checkAndGetColumn<ColumnUInt8>(&column);
-    if (!checkAndGetColumn<ColumnUInt8>(&column))
+    if (!col)
         throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Bool type can only serialize columns of type UInt8.{}", column.getName());
     return col;
 }
 
 ColumnUInt8 * checkAndGetDeserializeColumnType(IColumn & column)
 {
     auto * col =  typeid_cast<ColumnUInt8 *>(&column);
-    if (!checkAndGetColumn<ColumnUInt8>(&column))
+    if (!col)
         throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Bool type can only deserialize columns of type UInt8.{}",
                         column.getName());
     return col;

src/DataTypes/Serializations/SerializationInterval.cpp

@@ -17,7 +17,7 @@ namespace ErrorCodes
 void SerializationKustoInterval::serializeText(
     const IColumn & column, const size_t row, WriteBuffer & ostr, const FormatSettings &) const
 {
-    const auto * interval_column = checkAndGetColumn<ColumnInterval>(column);
+    const auto * interval_column = checkAndGetColumn<ColumnInterval>(&column);
     if (!interval_column)
         throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Expected column of underlying type of Interval");
 

src/DataTypes/Serializations/SerializationLowCardinality.cpp

@@ -477,7 +477,7 @@ void SerializationLowCardinality::serializeBinaryBulkWithMultipleStreams(
                             settings.low_cardinality_max_dictionary_size);
     }
 
-    if (const auto * nullable_keys = checkAndGetColumn<ColumnNullable>(*keys))
+    if (const auto * nullable_keys = checkAndGetColumn<ColumnNullable>(&*keys))
         keys = nullable_keys->getNestedColumnPtr();
 
     bool need_additional_keys = !keys->empty();

src/Dictionaries/HierarchyDictionariesUtils.cpp

@@ -95,7 +95,7 @@ namespace
                 parent_key_column_non_null = parent_key_column_typed->getNestedColumnPtr();
             }
 
-            const auto * parent_key_column_typed = checkAndGetColumn<ColumnVector<UInt64>>(*parent_key_column_non_null);
+            const auto * parent_key_column_typed = checkAndGetColumn<ColumnVector<UInt64>>(&*parent_key_column_non_null);
             if (!parent_key_column_typed)
                 throw Exception(ErrorCodes::UNSUPPORTED_METHOD,
                     "Parent key column should be UInt64. Actual {}",
@@ -166,7 +166,7 @@ ColumnPtr getKeysHierarchyDefaultImplementation(
     valid_keys = 0;
 
     key_column = key_column->convertToFullColumnIfConst();
-    const auto * key_column_typed = checkAndGetColumn<ColumnVector<UInt64>>(*key_column);
+    const auto * key_column_typed = checkAndGetColumn<ColumnVector<UInt64>>(&*key_column);
     if (!key_column_typed)
         throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "Key column should be UInt64");
 
@@ -224,11 +224,11 @@ ColumnUInt8::Ptr getKeysIsInHierarchyDefaultImplementation(
     key_column = key_column->convertToFullColumnIfConst();
     in_key_column = in_key_column->convertToFullColumnIfConst();
 
-    const auto * key_column_typed = checkAndGetColumn<ColumnVector<UInt64>>(*key_column);
+    const auto * key_column_typed = checkAndGetColumn<ColumnVector<UInt64>>(&*key_column);
     if (!key_column_typed)
         throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "Key column should be UInt64");
 
-    const auto * in_key_column_typed = checkAndGetColumn<ColumnVector<UInt64>>(*in_key_column);
+    const auto * in_key_column_typed = checkAndGetColumn<ColumnVector<UInt64>>(&*in_key_column);
     if (!in_key_column_typed)
         throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "Key column should be UInt64");