Merge branch 'master' of github.com:ClickHouse/ClickHouse into clang-…

…18-ci
den-crane · May 11, 2024 · eff0d7c · eff0d7c
2 parents c4bb09d + f9a7a50
commit eff0d7c
Show file tree

Hide file tree

Showing 11 changed files with 307 additions and 29 deletions.
diff --git a/.github/workflows/merge_queue.yml b/.github/workflows/merge_queue.yml
@@ -0,0 +1,96 @@
+# yamllint disable rule:comments-indentation
+name: MergeQueueCI
+
+env:
+  # Force the stdout and stderr streams to be unbuffered
+  PYTHONUNBUFFERED: 1
+
+on:  # yamllint disable-line rule:truthy
+  merge_group:
+
+jobs:
+  RunConfig:
+    runs-on: [self-hosted, style-checker-aarch64]
+    outputs:
+      data: ${{ steps.runconfig.outputs.CI_DATA }}
+    steps:
+      - name: DebugInfo
+        uses: hmarr/debug-action@f7318c783045ac39ed9bb497e22ce835fdafbfe6
+      - name: Check out repository code
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true # to ensure correct digests
+          fetch-depth: 0 # to get version
+          filter: tree:0
+      - name: Python unit tests
+        run: |
+          cd "$GITHUB_WORKSPACE/tests/ci"
+          echo "Testing the main ci directory"
+          python3 -m unittest discover -s . -p 'test_*.py'
+          for dir in *_lambda/; do
+            echo "Testing $dir"
+            python3 -m unittest discover -s "$dir" -p 'test_*.py'
+          done
+      - name: PrepareRunConfig
+        id: runconfig
+        run: |
+            python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --configure --outfile ${{ runner.temp }}/ci_run_data.json
+
+            echo "::group::CI configuration"
+            python3 -m json.tool ${{ runner.temp }}/ci_run_data.json
+            echo "::endgroup::"
+
+            {
+              echo 'CI_DATA<<EOF'
+              cat  ${{ runner.temp }}/ci_run_data.json
+              echo 'EOF'
+            } >> "$GITHUB_OUTPUT"
+  BuildDockers:
+    needs: [RunConfig]
+    if: ${{ !failure() && !cancelled() && toJson(fromJson(needs.RunConfig.outputs.data).docker_data.missing_multi) != '[]' }}
+    uses: ./.github/workflows/reusable_docker.yml
+    with:
+      data: ${{ needs.RunConfig.outputs.data }}
+  StyleCheck:
+    needs: [RunConfig, BuildDockers]
+    if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).jobs_data.jobs_to_do, 'Style check')}}
+    uses: ./.github/workflows/reusable_test.yml
+    with:
+      test_name: Style check
+      runner_type: style-checker
+      run_command: |
+          python3 style_check.py
+      data: ${{ needs.RunConfig.outputs.data }}
+    secrets:
+      secret_envs: |
+        ROBOT_CLICKHOUSE_SSH_KEY<<RCSK
+        ${{secrets.ROBOT_CLICKHOUSE_SSH_KEY}}
+        RCSK
+  FastTest:
+    needs: [RunConfig, BuildDockers]
+    if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).jobs_data.jobs_to_do, 'Fast test') }}
+    uses: ./.github/workflows/reusable_test.yml
+    with:
+      test_name: Fast test
+      runner_type: builder
+      data: ${{ needs.RunConfig.outputs.data }}
+      run_command: |
+          python3 fast_test_check.py
+
+  ################################# Stage Final #################################
+  #
+  FinishCheck:
+    if: ${{ !failure() && !cancelled() }}
+    needs: [RunConfig, BuildDockers, StyleCheck, FastTest]
+    runs-on: [self-hosted, style-checker]
+    steps:
+      - name: Check out repository code
+        uses: ClickHouse/checkout@v1
+      - name: Check sync status
+        run: |
+          cd "$GITHUB_WORKSPACE/tests/ci"
+          python3 sync_pr.py --status
+      - name: Finish label
+        run: |
+          cd "$GITHUB_WORKSPACE/tests/ci"
+          python3 finish_check.py ${{ (contains(needs.*.result, 'failure') && github.event_name == 'merge_group') && '--pipeline-failure' || '' }}
diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml
@@ -6,7 +6,6 @@ env:
   PYTHONUNBUFFERED: 1
 
 on:  # yamllint disable-line rule:truthy
-  merge_group:
   pull_request:
     types:
       - synchronize
@@ -30,7 +29,6 @@ jobs:
           fetch-depth: 0 # to get version
           filter: tree:0
       - name: Labels check
-        if: ${{ github.event_name != 'merge_group' }}
         run: |
           cd "$GITHUB_WORKSPACE/tests/ci"
           python3 run_check.py
@@ -58,7 +56,6 @@ jobs:
               echo 'EOF'
             } >> "$GITHUB_OUTPUT"
       - name: Re-create GH statuses for skipped jobs if any
-        if: ${{ github.event_name != 'merge_group' }}
         run: |
             python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --infile ${{ runner.temp }}/ci_run_data.json --update-gh-statuses
   BuildDockers:
@@ -83,7 +80,7 @@ jobs:
         ${{secrets.ROBOT_CLICKHOUSE_SSH_KEY}}
         RCSK
   FastTest:
-    needs: [RunConfig, BuildDockers]
+    needs: [RunConfig, BuildDockers, StyleCheck]
     if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).jobs_data.jobs_to_do, 'Fast test') }}
     uses: ./.github/workflows/reusable_test.yml
     with:
@@ -163,20 +160,16 @@ jobs:
     steps:
       - name: Check out repository code
         uses: ClickHouse/checkout@v1
-      - name: Check sync status
-        if: ${{ github.event_name == 'merge_group' }}
-        run: |
-          cd "$GITHUB_WORKSPACE/tests/ci"
-          python3 sync_pr.py --status
       - name: Finish label
         run: |
           cd "$GITHUB_WORKSPACE/tests/ci"
-          python3 finish_check.py ${{ (contains(needs.*.result, 'failure') && github.event_name == 'merge_group') && '--pipeline-failure' || '' }}
-      - name: Auto merge if approved
-        if: ${{ github.event_name != 'merge_group' }}
-        run: |
-          cd "$GITHUB_WORKSPACE/tests/ci"
-          python3 merge_pr.py --check-approved
+          python3 finish_check.py
+      # FIXME: merge on approval does not work with MQ. Could be fixed by using defaul GH's automerge after some corrections in Mergeable Check status
+      # - name: Auto merge if approved
+      #   if: ${{ github.event_name != 'merge_group' }}
+      #   run: |
+      #     cd "$GITHUB_WORKSPACE/tests/ci"
+      #     python3 merge_pr.py --check-approved
 
 
 #############################################################################################

diff --git a/src/Analyzer/QueryTreePassManager.cpp b/src/Analyzer/QueryTreePassManager.cpp
@@ -147,7 +147,7 @@ class ValidationChecker : public InDepthQueryTreeVisitor<ValidationChecker>
                     continue;
 
                 throw Exception(ErrorCodes::LOGICAL_ERROR,
-                    "Function {} expects {} argument to have {} type but receives {} after running {} pass",
+                    "Function {} expects argument {} to have {} type but receives {} after running {} pass",
                     function->toAST()->formatForErrorMessage(),
                     i + 1,
                     expected_argument_type->getName(),

diff --git a/src/Functions/IFunction.cpp b/src/Functions/IFunction.cpp
@@ -94,8 +94,8 @@ ColumnPtr replaceLowCardinalityColumnsByNestedAndGetDictionaryIndexes(
     {
         if (const auto * column_const = checkAndGetColumn<ColumnConst>(column.column.get()))
         {
-            column.column = column_const->removeLowCardinality()->cloneResized(num_rows);
-            column.type = removeLowCardinality(column.type);
+            column.column = ColumnConst::create(recursiveRemoveLowCardinality(column_const->getDataColumnPtr()), num_rows);
+            column.type = recursiveRemoveLowCardinality(column.type);
         }
     }
 

diff --git a/src/Functions/if.cpp b/src/Functions/if.cpp
@@ -1278,16 +1278,16 @@ class FunctionIf : public FunctionIfBase
     /// Get result types by argument types. If the function does not apply to these arguments, throw an exception.
     DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
     {
-        if (arguments[0]->onlyNull())
-            return arguments[2];
-
-        if (arguments[0]->isNullable())
-            return getReturnTypeImpl({
-                removeNullable(arguments[0]), arguments[1], arguments[2]});
+        if (!arguments[0]->onlyNull())
+        {
+            if (arguments[0]->isNullable())
+                return getReturnTypeImpl({
+                    removeNullable(arguments[0]), arguments[1], arguments[2]});
 
-        if (!WhichDataType(arguments[0]).isUInt8())
-            throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of first argument (condition) of function if. "
-                "Must be UInt8.", arguments[0]->getName());
+            if (!WhichDataType(arguments[0]).isUInt8())
+                throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of first argument (condition) of function if. "
+                    "Must be UInt8.", arguments[0]->getName());
+        }
 
         if (use_variant_when_no_common_type)
             return getLeastSupertypeOrVariant(DataTypes{arguments[1], arguments[2]});

diff --git a/src/Interpreters/OptimizeIfWithConstantConditionVisitor.cpp b/src/Interpreters/OptimizeIfWithConstantConditionVisitor.cpp
@@ -27,6 +27,11 @@ static bool tryExtractConstValueFromCondition(const ASTPtr & condition, bool & v
             value = literal->value.get<Int64>();
             return true;
         }
+        if (literal->value.getType() == Field::Types::Null)
+        {
+            value = false;
+            return true;
+        }
     }
 
     /// cast of numeric constant in condition to UInt8

diff --git a/tests/queries/0_stateless/02362_part_log_merge_algorithm.sql b/tests/queries/0_stateless/02362_part_log_merge_algorithm.sql
@@ -8,7 +8,7 @@ SETTINGS old_parts_lifetime=600;
 INSERT INTO data_horizontal VALUES (1);
 OPTIMIZE TABLE data_horizontal FINAL;
 SYSTEM FLUSH LOGS;
-SELECT table, part_name, event_type, merge_algorithm FROM system.part_log WHERE event_date >= yesterday() AND database = currentDatabase() AND table = 'data_horizontal' ORDER BY event_time_microseconds;
+SELECT table, part_name, event_type, merge_algorithm FROM system.part_log WHERE event_date >= yesterday() AND database = currentDatabase() AND table = 'data_horizontal' AND event_type IN ('NewPart', 'MergeParts') ORDER BY event_time_microseconds;
 
 CREATE TABLE data_vertical
 (
@@ -25,4 +25,4 @@ INSERT INTO data_vertical VALUES (1, '1');
 INSERT INTO data_vertical VALUES (2, '2');
 OPTIMIZE TABLE data_vertical FINAL;
 SYSTEM FLUSH LOGS;
-SELECT table, part_name, event_type, merge_algorithm FROM system.part_log WHERE event_date >= yesterday() AND database = currentDatabase() AND table = 'data_vertical' ORDER BY event_time_microseconds;
+SELECT table, part_name, event_type, merge_algorithm FROM system.part_log WHERE event_date >= yesterday() AND database = currentDatabase() AND table = 'data_vertical' AND event_type IN ('NewPart', 'MergeParts') ORDER BY event_time_microseconds;
diff --git a/tests/queries/0_stateless/02944_variant_as_common_type_analyzer.reference b/tests/queries/0_stateless/02944_variant_as_common_type_analyzer.reference
@@ -0,0 +1,103 @@
+Variant(Array(UInt8), String)	[1,2,3]
+Variant(Array(UInt8), String)	[1,2,3]
+Variant(Array(UInt8), String)	str_1
+Variant(Array(UInt8), String)	str_1
+Variant(Array(UInt8), String)	str_1
+Variant(Array(UInt8), String)	str_1
+Variant(Array(UInt8), String)	str_1
+Variant(Array(UInt8), String)	str_1
+Variant(Array(UInt8), String)	[1,2,3]
+Variant(Array(UInt8), String)	[1,2,3]
+Variant(Array(UInt8), String)	str_1
+Variant(Array(UInt8), String)	str_1
+Variant(Array(UInt8), String)	str_1
+Variant(Array(UInt8), String)	str_1
+Variant(Array(UInt8), String)	str_1
+Variant(Array(UInt8), String)	str_1
+Variant(Array(UInt8), String)	[1,2,3]
+Variant(Array(UInt8), String)	[1,2,3]
+Variant(Array(UInt8), String)	str_1
+Variant(Array(UInt8), String)	str_1
+Variant(Array(UInt8), String)	str_1
+Variant(Array(UInt8), String)	str_1
+Variant(Array(UInt8), String)	str_1
+Variant(Array(UInt8), String)	str_1
+Variant(Array(UInt64), String)	str_0
+Variant(Array(UInt64), String)	str_1
+Variant(Array(UInt64), String)	str_2
+Variant(Array(UInt64), String)	str_3
+Variant(Array(UInt64), String)	str_0
+Variant(Array(UInt64), String)	str_1
+Variant(Array(UInt64), String)	str_2
+Variant(Array(UInt64), String)	str_3
+Variant(Array(UInt64), String)	[0]
+Variant(Array(UInt64), String)	[0,1]
+Variant(Array(UInt64), String)	[0,1,2]
+Variant(Array(UInt64), String)	[0,1,2,3]
+Variant(Array(UInt64), String)	[0]
+Variant(Array(UInt64), String)	[0,1]
+Variant(Array(UInt64), String)	[0,1,2]
+Variant(Array(UInt64), String)	[0,1,2,3]
+Variant(Array(UInt64), String)	str_0
+Variant(Array(UInt64), String)	str_1
+Variant(Array(UInt64), String)	str_2
+Variant(Array(UInt64), String)	str_3
+Variant(Array(UInt64), String)	str_0
+Variant(Array(UInt64), String)	str_1
+Variant(Array(UInt64), String)	str_2
+Variant(Array(UInt64), String)	str_3
+Variant(Array(UInt64), String)	str_0
+Variant(Array(UInt64), String)	str_1
+Variant(Array(UInt64), String)	str_2
+Variant(Array(UInt64), String)	str_3
+Variant(Array(UInt64), String)	str_0
+Variant(Array(UInt64), String)	str_1
+Variant(Array(UInt64), String)	str_2
+Variant(Array(UInt64), String)	str_3
+Variant(Array(UInt64), String)	str_0
+Variant(Array(UInt64), String)	[0,1]
+Variant(Array(UInt64), String)	str_2
+Variant(Array(UInt64), String)	[0,1,2,3]
+Variant(Array(UInt64), String)	str_0
+Variant(Array(UInt64), String)	[0,1]
+Variant(Array(UInt64), String)	str_2
+Variant(Array(UInt64), String)	[0,1,2,3]
+Variant(Array(UInt64), String)	str_0
+Variant(Array(UInt64), String)	[0,1]
+Variant(Array(UInt64), String)	str_2
+Variant(Array(UInt64), String)	[0,1,2,3]
+Variant(Array(UInt64), String)	str_0
+Variant(Array(UInt64), String)	[0,1]
+Variant(Array(UInt64), String)	str_2
+Variant(Array(UInt64), String)	[0,1,2,3]
+Variant(Array(UInt64), String, UInt64)	[0]
+Variant(Array(UInt64), String, UInt64)	1
+Variant(Array(UInt64), String, UInt64)	str_2
+Variant(Array(UInt64), String, UInt64)	[0,1,2,3]
+Variant(Array(UInt64), String, UInt64)	4
+Variant(Array(UInt64), String, UInt64)	str_5
+Variant(Array(UInt64), String, UInt64)	[0]
+Variant(Array(UInt64), String, UInt64)	1
+Variant(Array(UInt64), String, UInt64)	str_2
+Variant(Array(UInt64), String, UInt64)	[0,1,2,3]
+Variant(Array(UInt64), String, UInt64)	4
+Variant(Array(UInt64), String, UInt64)	str_5
+Variant(Array(UInt64), String, UInt64)	[0]
+Variant(Array(UInt64), String, UInt64)	1
+Variant(Array(UInt64), String, UInt64)	str_2
+Variant(Array(UInt64), String, UInt64)	[0,1,2,3]
+Variant(Array(UInt64), String, UInt64)	4
+Variant(Array(UInt64), String, UInt64)	str_5
+Variant(Array(UInt64), String, UInt64)	[0]
+Variant(Array(UInt64), String, UInt64)	1
+Variant(Array(UInt64), String, UInt64)	str_2
+Variant(Array(UInt64), String, UInt64)	[0,1,2,3]
+Variant(Array(UInt64), String, UInt64)	4
+Variant(Array(UInt64), String, UInt64)	str_5
+Array(Variant(String, UInt8))	[1,'str_1',2,'str_2']
+Array(Variant(Array(String), Array(UInt8)))	[[1,2,3],['str_1','str_2','str_3']]
+Array(Variant(Array(UInt8), Array(Variant(Array(String), Array(UInt8)))))	[[[1,2,3],['str_1','str_2','str_3']],[1,2,3]]
+Array(Variant(Array(Array(UInt8)), Array(UInt8)))	[[1,2,3],[[1,2,3]]]
+Map(String, Variant(String, UInt8))	{'a':1,'b':'str_1'}
+Map(String, Variant(Map(String, Variant(String, UInt8)), UInt8))	{'a':1,'b':{'c':2,'d':'str_1'}}
+Map(String, Variant(Array(Array(UInt8)), Array(UInt8), UInt8))	{'a':1,'b':[1,2,3],'c':[[4,5,6]]}