From 71b07157d3134e336c836379951e7cbee6e827ac Mon Sep 17 00:00:00 2001 From: Mark Zitnik Date: Tue, 6 Aug 2024 10:35:59 +0300 Subject: [PATCH] Core, Test: Refactor tests to support ClickHouse Cloud (#349) * Build, Core: Test with ClickHouse Cloud * Spark 3.5: Test with ClickHouse Cloud * Spark 3.4: Test with ClickHouse Cloud * Spark 3.3: Test with ClickHouse Cloud * Add withNodeClient username/password/opts & change Table engine from Log() to Memory() * Add ssl only if isSslEnabled is true * Extend table name to a void tests from hitting same table name * Add connection timeout & set database name for the Client * Fix spotless * Disable parallel tun of Cloud tests * Do not expect AnalysisException when running cloud test against Spark 3.3 * Code cleanup * move import & set parallel = 1 when testing * Add conditional sleep to older version * Add missing import * Fix Versions * Moving back to parallel for local tests --------- Co-authored-by: Cheng Pan --- .github/workflows/cloud.yml | 58 +++++++++++++++++++ build.gradle | 14 +++++ .../com/clickhouse/spark/HashSuite.scala | 12 +++- ...ClickHouseSuite.scala => UtilsSuite.scala} | 12 +++- .../clickhouse/spark/client/NodeClient.scala | 7 ++- .../java/org/scalatest/tags/Cloud.java | 24 ++++++++ .../spark/base/ClickHouseCloudMixIn.scala | 38 ++++++++++++ .../spark/base/ClickHouseProvider.scala | 51 ++++++++++++++++ .../spark/base/ClickHouseSingleMixIn.scala | 36 +++++------- spark-3.3/build.gradle | 4 ++ .../single/ClickHouseDataTypeSuite.scala | 12 +++- ...ite.scala => ClickHouseGenericSuite.scala} | 9 ++- .../single/ClickHouseTableDDLSuite.scala | 11 +++- .../single/SparkClickHouseSingleTest.scala | 19 +++--- .../sql/clickhouse/single/TPCDSSuite.scala | 10 +++- .../WriteDistributionAndOrderingSuite.scala | 13 ++++- spark-3.4/build.gradle | 4 ++ .../single/ClickHouseDataTypeSuite.scala | 12 +++- ...ite.scala => ClickHouseGenericSuite.scala} | 9 ++- .../single/ClickHouseTableDDLSuite.scala | 11 +++- .../single/SparkClickHouseSingleTest.scala | 19 +++--- .../sql/clickhouse/single/TPCDSSuite.scala | 10 +++- .../WriteDistributionAndOrderingSuite.scala | 13 ++++- spark-3.5/build.gradle | 4 ++ .../single/ClickHouseDataTypeSuite.scala | 12 +++- ...ite.scala => ClickHouseGenericSuite.scala} | 9 ++- .../single/ClickHouseTableDDLSuite.scala | 11 +++- .../single/SparkClickHouseSingleTest.scala | 19 +++--- .../sql/clickhouse/single/TPCDSSuite.scala | 10 +++- .../WriteDistributionAndOrderingSuite.scala | 13 ++++- 30 files changed, 406 insertions(+), 80 deletions(-) create mode 100644 .github/workflows/cloud.yml rename clickhouse-core-it/src/test/scala/com/clickhouse/spark/{UtilsClickHouseSuite.scala => UtilsSuite.scala} (74%) create mode 100644 clickhouse-core/src/testFixtures/java/org/scalatest/tags/Cloud.java create mode 100644 clickhouse-core/src/testFixtures/scala/com/clickhouse/spark/base/ClickHouseCloudMixIn.scala create mode 100644 clickhouse-core/src/testFixtures/scala/com/clickhouse/spark/base/ClickHouseProvider.scala rename spark-3.3/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/{ClickHouseSingleSuite.scala => ClickHouseGenericSuite.scala} (97%) rename spark-3.4/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/{ClickHouseSingleSuite.scala => ClickHouseGenericSuite.scala} (97%) rename spark-3.5/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/{ClickHouseSingleSuite.scala => ClickHouseGenericSuite.scala} (97%) diff --git a/.github/workflows/cloud.yml b/.github/workflows/cloud.yml new file mode 100644 index 00000000..f8db9725 --- /dev/null +++ b/.github/workflows/cloud.yml @@ -0,0 +1,58 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: "ClickHouse Cloud" + +on: + push: + branches: + - "branch-*" + - "main" + pull_request: + branches: + - "branch-*" + - "main" + +jobs: + run-tests-with-clickhouse-cloud: + runs-on: ubuntu-22.04 + strategy: + max-parallel: 1 + fail-fast: false + matrix: + spark: [ 3.3, 3.4, 3.5 ] + scala: [ 2.12, 2.13 ] + env: + CLICKHOUSE_CLOUD_HOST: ${{ secrets.INTEGRATIONS_TEAM_TESTS_CLOUD_HOST_SMT }} + CLICKHOUSE_CLOUD_PASSWORD: ${{ secrets.INTEGRATIONS_TEAM_TESTS_CLOUD_PASSWORD_SMT }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + distribution: zulu + java-version: 8 + cache: gradle + - run: >- + ./gradlew clean cloudTest --no-daemon --refresh-dependencies + -Dspark_binary_version=${{ matrix.spark }} + -Dscala_binary_version=${{ matrix.scala }} + -PmavenCentralMirror=https://maven-central.storage-download.googleapis.com/maven2/ + - name: Upload test logs + if: failure() + uses: actions/upload-artifact@v3 + with: + name: log-clickhouse-cloud-spark-${{ matrix.spark }}-scala-${{ matrix.scala }} + path: | + **/build/unit-tests.log + log/** diff --git a/build.gradle b/build.gradle index c8f929d4..c910b075 100644 --- a/build.gradle +++ b/build.gradle @@ -118,6 +118,7 @@ subprojects { environment("ROOT_PROJECT_DIR", rootProject.projectDir) tags { exclude 'org.scalatest.tags.Slow' + exclude 'org.scalatest.tags.Cloud' } } @@ -130,6 +131,15 @@ subprojects { } } + tasks.register('cloudTest', Test) { + jvmArgs += extraJvmArgs + maxParallelForks = 1 + environment("ROOT_PROJECT_DIR", rootProject.projectDir) + tags { + include 'org.scalatest.tags.Cloud' + } + } + scoverage { scoverageVersion = "2.0.11" reportDir.set(file("${rootProject.buildDir}/reports/scoverage")) @@ -220,6 +230,10 @@ project(":clickhouse-core-it") { slowTest { classpath += files("${project(':clickhouse-core').projectDir}/src/testFixtures/conf") } + + cloudTest { + classpath += files("${project(':clickhouse-core').projectDir}/src/testFixtures/conf") + } } boolean isVersionFileExists() { diff --git a/clickhouse-core-it/src/test/scala/com/clickhouse/spark/HashSuite.scala b/clickhouse-core-it/src/test/scala/com/clickhouse/spark/HashSuite.scala index bf06bced..6dd3fbd5 100644 --- a/clickhouse-core-it/src/test/scala/com/clickhouse/spark/HashSuite.scala +++ b/clickhouse-core-it/src/test/scala/com/clickhouse/spark/HashSuite.scala @@ -14,12 +14,18 @@ package com.clickhouse.spark -import com.clickhouse.spark.base.ClickHouseSingleMixIn +import com.clickhouse.spark.base.{ClickHouseCloudMixIn, ClickHouseProvider, ClickHouseSingleMixIn} import com.clickhouse.spark.client.NodeClient import com.clickhouse.spark.hash.{CityHash64, HashUtils, Murmurhash2_32, Murmurhash2_64, Murmurhash3_32, Murmurhash3_64} -import com.clickhouse.spark.hash._ +import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.tags.Cloud -class HashSuite extends ClickHouseSingleMixIn with Logging { +@Cloud +class ClickHouseCloudHashSuite extends HashSuite with ClickHouseCloudMixIn + +class ClickHouseSingleHashSuite extends HashSuite with ClickHouseSingleMixIn + +abstract class HashSuite extends AnyFunSuite with ClickHouseProvider with Logging { def testHash( client: NodeClient, diff --git a/clickhouse-core-it/src/test/scala/com/clickhouse/spark/UtilsClickHouseSuite.scala b/clickhouse-core-it/src/test/scala/com/clickhouse/spark/UtilsSuite.scala similarity index 74% rename from clickhouse-core-it/src/test/scala/com/clickhouse/spark/UtilsClickHouseSuite.scala rename to clickhouse-core-it/src/test/scala/com/clickhouse/spark/UtilsSuite.scala index 50a8525a..d4b305e8 100644 --- a/clickhouse-core-it/src/test/scala/com/clickhouse/spark/UtilsClickHouseSuite.scala +++ b/clickhouse-core-it/src/test/scala/com/clickhouse/spark/UtilsSuite.scala @@ -14,10 +14,18 @@ package com.clickhouse.spark -import com.clickhouse.spark.base.ClickHouseSingleMixIn +import com.clickhouse.spark.base.{ClickHouseCloudMixIn, ClickHouseProvider, ClickHouseSingleMixIn} +import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.tags.Cloud + import java.time.{LocalDateTime, ZoneId} -class UtilsClickHouseSuite extends ClickHouseSingleMixIn with Logging { +@Cloud +class ClickHouseCloudUtilsSuite extends UtilsSuite with ClickHouseCloudMixIn + +class ClickHouseSingleUtilsSuite extends UtilsSuite with ClickHouseSingleMixIn + +abstract class UtilsSuite extends AnyFunSuite with ClickHouseProvider with Logging { test("parse date with nano seconds") { withNodeClient() { client => diff --git a/clickhouse-core/src/main/scala/com/clickhouse/spark/client/NodeClient.scala b/clickhouse-core/src/main/scala/com/clickhouse/spark/client/NodeClient.scala index c733abd7..0d777d95 100644 --- a/clickhouse-core/src/main/scala/com/clickhouse/spark/client/NodeClient.scala +++ b/clickhouse-core/src/main/scala/com/clickhouse/spark/client/NodeClient.scala @@ -39,6 +39,8 @@ object NodeClient { } class NodeClient(val nodeSpec: NodeSpec) extends AutoCloseable with Logging { + // TODO: add configurable timeout + private val timeout: Int = 30000 private lazy val userAgent = { val title = getClass.getPackage.getImplementationTitle @@ -53,7 +55,6 @@ class NodeClient(val nodeSpec: NodeSpec) extends AutoCloseable with Logging { "Spark-ClickHouse-Connector" } } - private val node: ClickHouseNode = ClickHouseNode.builder() .options(nodeSpec.options) .host(nodeSpec.host) @@ -158,6 +159,7 @@ class NodeClient(val nodeSpec: NodeSpec) extends AutoCloseable with Logging { val req = client.read(node) .query(sql, queryId).asInstanceOf[ClickHouseRequest[_]] .format(ClickHouseFormat.valueOf(outputFormat)).asInstanceOf[ClickHouseRequest[_]] + .option(ClickHouseClientOption.CONNECTION_TIMEOUT, timeout).asInstanceOf[ClickHouseRequest[_]] settings.foreach { case (k, v) => req.set(k, v).asInstanceOf[ClickHouseRequest[_]] } Try(req.executeAndWait()) match { case Success(resp) => Right(deserializer(resp.getInputStream)) @@ -193,6 +195,7 @@ class NodeClient(val nodeSpec: NodeSpec) extends AutoCloseable with Logging { .query(sql, queryId).asInstanceOf[ClickHouseRequest[_]] .compressServerResponse(outputCompressionType).asInstanceOf[ClickHouseRequest[_]] .format(ClickHouseFormat.valueOf(outputFormat)).asInstanceOf[ClickHouseRequest[_]] + .option(ClickHouseClientOption.CONNECTION_TIMEOUT, timeout).asInstanceOf[ClickHouseRequest[_]] settings.foreach { case (k, v) => req.set(k, v).asInstanceOf[ClickHouseRequest[_]] } Try(req.executeAndWait()) match { case Success(resp) => resp @@ -211,4 +214,6 @@ class NodeClient(val nodeSpec: NodeSpec) extends AutoCloseable with Logging { |$sql |""".stripMargin ) + def ping(timeout: Int = timeout) = + client.ping(node, timeout) } diff --git a/clickhouse-core/src/testFixtures/java/org/scalatest/tags/Cloud.java b/clickhouse-core/src/testFixtures/java/org/scalatest/tags/Cloud.java new file mode 100644 index 00000000..af9290f9 --- /dev/null +++ b/clickhouse-core/src/testFixtures/java/org/scalatest/tags/Cloud.java @@ -0,0 +1,24 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.scalatest.tags; + +import java.lang.annotation.*; +import org.scalatest.TagAnnotation; + +@TagAnnotation +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.METHOD, ElementType.TYPE}) +@Inherited +public @interface Cloud {} diff --git a/clickhouse-core/src/testFixtures/scala/com/clickhouse/spark/base/ClickHouseCloudMixIn.scala b/clickhouse-core/src/testFixtures/scala/com/clickhouse/spark/base/ClickHouseCloudMixIn.scala new file mode 100644 index 00000000..1c4b15bb --- /dev/null +++ b/clickhouse-core/src/testFixtures/scala/com/clickhouse/spark/base/ClickHouseCloudMixIn.scala @@ -0,0 +1,38 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.clickhouse.spark.base + +import com.clickhouse.data.ClickHouseVersion +import com.clickhouse.spark.Utils + +trait ClickHouseCloudMixIn extends ClickHouseProvider { + + override def clickhouseHost: String = Utils.load("CLICKHOUSE_CLOUD_HOST") + + override def clickhouseHttpPort: Int = Utils.load("CLICKHOUSE_CLOUD_HTTP_PORT", "8443").toInt + + override def clickhouseTcpPort: Int = Utils.load("CLICKHOUSE_CLOUD_TCP_PORT", "9000").toInt + + override def clickhouseUser: String = Utils.load("CLICKHOUSE_CLOUD_USER", "default") + + override def clickhousePassword: String = Utils.load("CLICKHOUSE_CLOUD_PASSWORD") + + override def clickhouseDatabase: String = "default" + + override def clickhouseVersion: ClickHouseVersion = ClickHouseVersion.of("latest") + + override def isSslEnabled: Boolean = true + override def isCloud: Boolean = true +} diff --git a/clickhouse-core/src/testFixtures/scala/com/clickhouse/spark/base/ClickHouseProvider.scala b/clickhouse-core/src/testFixtures/scala/com/clickhouse/spark/base/ClickHouseProvider.scala new file mode 100644 index 00000000..203695af --- /dev/null +++ b/clickhouse-core/src/testFixtures/scala/com/clickhouse/spark/base/ClickHouseProvider.scala @@ -0,0 +1,51 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.clickhouse.spark.base + +import com.clickhouse.client.ClickHouseProtocol +import com.clickhouse.client.ClickHouseProtocol.HTTP +import com.clickhouse.data.ClickHouseVersion +import com.clickhouse.spark.Utils +import com.clickhouse.spark.client.NodeClient +import com.clickhouse.spark.spec.NodeSpec +import scala.collection.JavaConverters._ + +trait ClickHouseProvider { + def clickhouseHost: String + def clickhouseHttpPort: Int + def clickhouseTcpPort: Int + def clickhouseUser: String + def clickhousePassword: String + def clickhouseDatabase: String + def clickhouseVersion: ClickHouseVersion + def isSslEnabled: Boolean + def isCloud: Boolean = false + + def withNodeClient(protocol: ClickHouseProtocol = HTTP)(block: NodeClient => Unit): Unit = + Utils.tryWithResource { + NodeClient(NodeSpec( + clickhouseHost, + Some(clickhouseHttpPort), + Some(clickhouseTcpPort), + protocol, + username = clickhouseUser, + database = clickhouseDatabase, + password = clickhousePassword, + options = Map("ssl" -> isSslEnabled.toString).asJava + )) + } { + client => block(client) + } +} diff --git a/clickhouse-core/src/testFixtures/scala/com/clickhouse/spark/base/ClickHouseSingleMixIn.scala b/clickhouse-core/src/testFixtures/scala/com/clickhouse/spark/base/ClickHouseSingleMixIn.scala index 7b3db306..2165c84d 100644 --- a/clickhouse-core/src/testFixtures/scala/com/clickhouse/spark/base/ClickHouseSingleMixIn.scala +++ b/clickhouse-core/src/testFixtures/scala/com/clickhouse/spark/base/ClickHouseSingleMixIn.scala @@ -15,32 +15,26 @@ package com.clickhouse.spark.base import com.clickhouse.spark.Utils -import com.clickhouse.client.ClickHouseProtocol -import com.clickhouse.client.ClickHouseProtocol._ import com.clickhouse.data.ClickHouseVersion -import com.clickhouse.spark.client.NodeClient -import com.clickhouse.spark.spec.NodeSpec import com.dimafeng.testcontainers.{ForAllTestContainer, JdbcDatabaseContainer, SingleContainer} import org.scalatest.funsuite.AnyFunSuite import org.testcontainers.containers.ClickHouseContainer import org.testcontainers.utility.{DockerImageName, MountableFile} - import java.nio.file.{Path, Paths} - import scala.collection.JavaConverters._ -trait ClickHouseSingleMixIn extends AnyFunSuite with ForAllTestContainer { +trait ClickHouseSingleMixIn extends AnyFunSuite with ForAllTestContainer with ClickHouseProvider { // format: off - val CLICKHOUSE_IMAGE: String = Utils.load("CLICKHOUSE_IMAGE", "clickhouse/clickhouse-server:23.8") - val CLICKHOUSE_USER: String = Utils.load("CLICKHOUSE_USER", "default") - val CLICKHOUSE_PASSWORD: String = Utils.load("CLICKHOUSE_PASSWORD", "") - val CLICKHOUSE_DB: String = Utils.load("CLICKHOUSE_DB", "") + private val CLICKHOUSE_IMAGE: String = Utils.load("CLICKHOUSE_IMAGE", "clickhouse/clickhouse-server:23.8") + private val CLICKHOUSE_USER: String = Utils.load("CLICKHOUSE_USER", "default") + private val CLICKHOUSE_PASSWORD: String = Utils.load("CLICKHOUSE_PASSWORD", "") + private val CLICKHOUSE_DB: String = Utils.load("CLICKHOUSE_DB", "") private val CLICKHOUSE_HTTP_PORT = 8123 private val CLICKHOUSE_TPC_PORT = 9000 // format: on - protected val clickhouseVersion: ClickHouseVersion = ClickHouseVersion.of(CLICKHOUSE_IMAGE.split(":").last) + override val clickhouseVersion: ClickHouseVersion = ClickHouseVersion.of(CLICKHOUSE_IMAGE.split(":").last) protected val rootProjectDir: Path = { val thisClassURI = this.getClass.getProtectionDomain.getCodeSource.getLocation.toURI @@ -77,16 +71,12 @@ trait ClickHouseSingleMixIn extends AnyFunSuite with ForAllTestContainer { ) .asInstanceOf[ClickHouseContainer] } - // format: off - def clickhouseHost: String = container.host - def clickhouseHttpPort: Int = container.mappedPort(CLICKHOUSE_HTTP_PORT) - def clickhouseTcpPort: Int = container.mappedPort(CLICKHOUSE_TPC_PORT) - // format: on - def withNodeClient(protocol: ClickHouseProtocol = HTTP)(block: NodeClient => Unit): Unit = - Utils.tryWithResource { - NodeClient(NodeSpec(clickhouseHost, Some(clickhouseHttpPort), Some(clickhouseTcpPort), protocol)) - } { - client => block(client) - } + override def clickhouseHost: String = container.host + override def clickhouseHttpPort: Int = container.mappedPort(CLICKHOUSE_HTTP_PORT) + override def clickhouseTcpPort: Int = container.mappedPort(CLICKHOUSE_TPC_PORT) + override def clickhouseUser: String = CLICKHOUSE_USER + override def clickhousePassword: String = CLICKHOUSE_PASSWORD + override def clickhouseDatabase: String = CLICKHOUSE_DB + override def isSslEnabled: Boolean = false } diff --git a/spark-3.3/build.gradle b/spark-3.3/build.gradle index 49599922..edaea5ec 100644 --- a/spark-3.3/build.gradle +++ b/spark-3.3/build.gradle @@ -96,4 +96,8 @@ project(":clickhouse-spark-it-${spark_binary_version}_$scala_binary_version") { slowTest { classpath += files("${project(':clickhouse-core').projectDir}/src/testFixtures/conf") } + + cloudTest { + classpath += files("${project(':clickhouse-core').projectDir}/src/testFixtures/conf") + } } diff --git a/spark-3.3/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/ClickHouseDataTypeSuite.scala b/spark-3.3/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/ClickHouseDataTypeSuite.scala index b6881019..ad00f4e5 100644 --- a/spark-3.3/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/ClickHouseDataTypeSuite.scala +++ b/spark-3.3/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/ClickHouseDataTypeSuite.scala @@ -14,15 +14,22 @@ package org.apache.spark.sql.clickhouse.single +import com.clickhouse.spark.base.{ClickHouseCloudMixIn, ClickHouseSingleMixIn} import org.apache.spark.sql.clickhouse.ClickHouseSQLConf.USE_NULLABLE_QUERY_SCHEMA import org.apache.spark.sql.clickhouse.SparkUtils import org.apache.spark.sql.types.DataTypes.{createArrayType, createMapType} import org.apache.spark.sql.types._ import org.apache.spark.sql.{DataFrame, Row} +import org.scalatest.tags.Cloud import java.math.MathContext -class ClickHouseDataTypeSuite extends SparkClickHouseSingleTest { +@Cloud +class ClickHouseCloudDataTypeSuite extends ClickHouseDataTypeSuite with ClickHouseCloudMixIn + +class ClickHouseSingleDataTypeSuite extends ClickHouseDataTypeSuite with ClickHouseSingleMixIn + +abstract class ClickHouseDataTypeSuite extends SparkClickHouseSingleTest { val SPARK_43390_ENABLED: Boolean = sys.env.contains("SPARK_43390_ENABLED") || { SparkUtils.MAJOR_MINOR_VERSION match { @@ -177,6 +184,9 @@ class ClickHouseDataTypeSuite extends SparkClickHouseSingleTest { : Unit = { val db = "test_kv_db" val tbl = "test_kv_tbl" + if (!clickhouseVersion.isNewerOrEqualTo("23.3") || isCloud) { + Thread.sleep(1000) + } withKVTable(db, tbl, valueColDef = valueColDef) { prepare(db, tbl) val df = spark.sql(s"SELECT key, value FROM $db.$tbl ORDER BY key") diff --git a/spark-3.3/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/ClickHouseSingleSuite.scala b/spark-3.3/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/ClickHouseGenericSuite.scala similarity index 97% rename from spark-3.3/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/ClickHouseSingleSuite.scala rename to spark-3.3/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/ClickHouseGenericSuite.scala index b80715ee..7d10753d 100644 --- a/spark-3.3/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/ClickHouseSingleSuite.scala +++ b/spark-3.3/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/ClickHouseGenericSuite.scala @@ -14,12 +14,19 @@ package org.apache.spark.sql.clickhouse.single +import com.clickhouse.spark.base.{ClickHouseCloudMixIn, ClickHouseSingleMixIn} import org.apache.spark.sql.Row import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.execution.datasources.v2.BatchScanExec import org.apache.spark.sql.types._ +import org.scalatest.tags.Cloud -class ClickHouseSingleSuite extends SparkClickHouseSingleTest { +@Cloud +class ClickHouseCloudGenericSuite extends ClickHouseDataTypeSuite with ClickHouseCloudMixIn + +class ClickHouseSingleGenericSuite extends ClickHouseDataTypeSuite with ClickHouseSingleMixIn + +abstract class ClickHouseGenericSuite extends SparkClickHouseSingleTest { import testImplicits._ diff --git a/spark-3.3/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/ClickHouseTableDDLSuite.scala b/spark-3.3/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/ClickHouseTableDDLSuite.scala index 7409a590..f3918098 100644 --- a/spark-3.3/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/ClickHouseTableDDLSuite.scala +++ b/spark-3.3/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/ClickHouseTableDDLSuite.scala @@ -14,15 +14,22 @@ package org.apache.spark.sql.clickhouse.single +import com.clickhouse.spark.base.{ClickHouseCloudMixIn, ClickHouseSingleMixIn} import org.apache.spark.sql.Row +import org.scalatest.tags.Cloud -class ClickHouseTableDDLSuite extends SparkClickHouseSingleTest { +@Cloud +class ClickHouseCloudTableDDLSuite extends ClickHouseTableDDLSuite with ClickHouseCloudMixIn + +class ClickHouseSingleTableDDLSuite extends ClickHouseTableDDLSuite with ClickHouseSingleMixIn + +abstract class ClickHouseTableDDLSuite extends SparkClickHouseSingleTest { import testImplicits._ test("clickhouse command runner") { withTable("default.abc") { - runClickHouseSQL("CREATE TABLE default.abc(a UInt8) ENGINE=Log()") + runClickHouseSQL("CREATE TABLE default.abc(a UInt8) ENGINE=Memory()") checkAnswer( spark.sql("""DESC default.abc""").select($"col_name", $"data_type").limit(1), Row("a", "smallint") :: Nil diff --git a/spark-3.3/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/SparkClickHouseSingleTest.scala b/spark-3.3/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/SparkClickHouseSingleTest.scala index 65f7f372..d9e7890a 100644 --- a/spark-3.3/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/SparkClickHouseSingleTest.scala +++ b/spark-3.3/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/SparkClickHouseSingleTest.scala @@ -14,13 +14,13 @@ package org.apache.spark.sql.clickhouse.single +import com.clickhouse.spark.base.ClickHouseProvider import org.apache.spark.SparkConf import org.apache.spark.sql.clickhouse.SparkTest import org.apache.spark.sql.functions.month import org.apache.spark.sql.types.StructType -import com.clickhouse.spark.base.ClickHouseSingleMixIn -trait SparkClickHouseSingleTest extends SparkTest with ClickHouseSingleMixIn { +trait SparkClickHouseSingleTest extends SparkTest with ClickHouseProvider { import testImplicits._ @@ -34,10 +34,11 @@ trait SparkClickHouseSingleTest extends SparkTest with ClickHouseSingleMixIn { .set("spark.sql.catalog.clickhouse.host", clickhouseHost) .set("spark.sql.catalog.clickhouse.http_port", clickhouseHttpPort.toString) .set("spark.sql.catalog.clickhouse.protocol", "http") - .set("spark.sql.catalog.clickhouse.user", CLICKHOUSE_USER) - .set("spark.sql.catalog.clickhouse.password", CLICKHOUSE_PASSWORD) - .set("spark.sql.catalog.clickhouse.database", CLICKHOUSE_DB) + .set("spark.sql.catalog.clickhouse.user", clickhouseUser) + .set("spark.sql.catalog.clickhouse.password", clickhousePassword) + .set("spark.sql.catalog.clickhouse.database", clickhouseDatabase) .set("spark.sql.catalog.clickhouse.option.custom_http_params", "async_insert=1,wait_for_async_insert=1") + .set("spark.sql.catalog.clickhouse.option.ssl", isSslEnabled.toString) // extended configurations .set("spark.clickhouse.write.batchSize", "2") .set("spark.clickhouse.write.maxRetry", "2") @@ -51,9 +52,11 @@ trait SparkClickHouseSingleTest extends SparkTest with ClickHouseSingleMixIn { "host" -> clickhouseHost, "http_port" -> clickhouseHttpPort.toString, "protocol" -> "http", - "user" -> CLICKHOUSE_USER, - "password" -> CLICKHOUSE_PASSWORD, - "database" -> CLICKHOUSE_DB + "user" -> clickhouseUser, + "password" -> clickhousePassword, + "database" -> clickhouseDatabase, + "option.custom_http_params" -> "async_insert=1,wait_for_async_insert=1", + "option.ssl" -> isSslEnabled.toString ) def withTable( diff --git a/spark-3.3/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/TPCDSSuite.scala b/spark-3.3/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/TPCDSSuite.scala index cda9793e..cad773c1 100644 --- a/spark-3.3/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/TPCDSSuite.scala +++ b/spark-3.3/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/TPCDSSuite.scala @@ -14,12 +14,18 @@ package org.apache.spark.sql.clickhouse.single +import com.clickhouse.spark.base.{ClickHouseCloudMixIn, ClickHouseSingleMixIn} import org.apache.spark.SparkConf import org.apache.spark.sql.clickhouse.TPCDSTestUtils -import org.scalatest.tags.Slow +import org.scalatest.tags.{Cloud, Slow} + +@Cloud +class ClickHouseCloudTPCDSSuite extends TPCDSSuite with ClickHouseCloudMixIn @Slow -class TPCDSSuite extends SparkClickHouseSingleTest { +class ClickHouseSingleTPCDSSuite extends TPCDSSuite with ClickHouseSingleMixIn + +abstract class TPCDSSuite extends SparkClickHouseSingleTest { override protected def sparkConf: SparkConf = super.sparkConf .set("spark.sql.catalog.tpcds", "org.apache.kyuubi.spark.connector.tpcds.TPCDSCatalog") diff --git a/spark-3.3/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/WriteDistributionAndOrderingSuite.scala b/spark-3.3/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/WriteDistributionAndOrderingSuite.scala index fe9ba535..2826cd32 100644 --- a/spark-3.3/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/WriteDistributionAndOrderingSuite.scala +++ b/spark-3.3/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/WriteDistributionAndOrderingSuite.scala @@ -14,14 +14,21 @@ package org.apache.spark.sql.clickhouse.single +import com.clickhouse.spark.base.{ClickHouseCloudMixIn, ClickHouseSingleMixIn} import org.apache.spark.sql.clickhouse.ClickHouseSQLConf._ import org.apache.spark.sql.functions._ import org.apache.spark.sql.types.StringType import org.apache.spark.sql.{AnalysisException, Row} +import org.scalatest.tags.Cloud -import java.time.LocalDate +@Cloud +class ClickHouseCloudsWriteDistributionAndOrderingSuite + extends WriteDistributionAndOrderingSuite with ClickHouseCloudMixIn -class WriteDistributionAndOrderingSuite extends SparkClickHouseSingleTest { +class ClickHouseSinglesWriteDistributionAndOrderingSuite + extends WriteDistributionAndOrderingSuite with ClickHouseSingleMixIn + +abstract class WriteDistributionAndOrderingSuite extends SparkClickHouseSingleTest { import testImplicits._ @@ -78,7 +85,7 @@ class WriteDistributionAndOrderingSuite extends SparkClickHouseSingleTest { WRITE_REPARTITION_BY_PARTITION.key -> repartitionByPartition.toString, WRITE_LOCAL_SORT_BY_KEY.key -> localSortByKey.toString ) { - if (!ignoreUnsupportedTransform && repartitionByPartition) { + if (!ignoreUnsupportedTransform && repartitionByPartition && !isCloud) { intercept[AnalysisException](write()) } else { write() diff --git a/spark-3.4/build.gradle b/spark-3.4/build.gradle index 2d9b58d4..2747bed0 100644 --- a/spark-3.4/build.gradle +++ b/spark-3.4/build.gradle @@ -96,4 +96,8 @@ project(":clickhouse-spark-it-${spark_binary_version}_$scala_binary_version") { slowTest { classpath += files("${project(':clickhouse-core').projectDir}/src/testFixtures/conf") } + + cloudTest { + classpath += files("${project(':clickhouse-core').projectDir}/src/testFixtures/conf") + } } diff --git a/spark-3.4/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/ClickHouseDataTypeSuite.scala b/spark-3.4/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/ClickHouseDataTypeSuite.scala index b6881019..ad00f4e5 100644 --- a/spark-3.4/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/ClickHouseDataTypeSuite.scala +++ b/spark-3.4/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/ClickHouseDataTypeSuite.scala @@ -14,15 +14,22 @@ package org.apache.spark.sql.clickhouse.single +import com.clickhouse.spark.base.{ClickHouseCloudMixIn, ClickHouseSingleMixIn} import org.apache.spark.sql.clickhouse.ClickHouseSQLConf.USE_NULLABLE_QUERY_SCHEMA import org.apache.spark.sql.clickhouse.SparkUtils import org.apache.spark.sql.types.DataTypes.{createArrayType, createMapType} import org.apache.spark.sql.types._ import org.apache.spark.sql.{DataFrame, Row} +import org.scalatest.tags.Cloud import java.math.MathContext -class ClickHouseDataTypeSuite extends SparkClickHouseSingleTest { +@Cloud +class ClickHouseCloudDataTypeSuite extends ClickHouseDataTypeSuite with ClickHouseCloudMixIn + +class ClickHouseSingleDataTypeSuite extends ClickHouseDataTypeSuite with ClickHouseSingleMixIn + +abstract class ClickHouseDataTypeSuite extends SparkClickHouseSingleTest { val SPARK_43390_ENABLED: Boolean = sys.env.contains("SPARK_43390_ENABLED") || { SparkUtils.MAJOR_MINOR_VERSION match { @@ -177,6 +184,9 @@ class ClickHouseDataTypeSuite extends SparkClickHouseSingleTest { : Unit = { val db = "test_kv_db" val tbl = "test_kv_tbl" + if (!clickhouseVersion.isNewerOrEqualTo("23.3") || isCloud) { + Thread.sleep(1000) + } withKVTable(db, tbl, valueColDef = valueColDef) { prepare(db, tbl) val df = spark.sql(s"SELECT key, value FROM $db.$tbl ORDER BY key") diff --git a/spark-3.4/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/ClickHouseSingleSuite.scala b/spark-3.4/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/ClickHouseGenericSuite.scala similarity index 97% rename from spark-3.4/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/ClickHouseSingleSuite.scala rename to spark-3.4/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/ClickHouseGenericSuite.scala index ca905941..79a6b31e 100644 --- a/spark-3.4/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/ClickHouseSingleSuite.scala +++ b/spark-3.4/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/ClickHouseGenericSuite.scala @@ -14,12 +14,19 @@ package org.apache.spark.sql.clickhouse.single +import com.clickhouse.spark.base.{ClickHouseCloudMixIn, ClickHouseSingleMixIn} import org.apache.spark.sql.Row import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.execution.datasources.v2.BatchScanExec import org.apache.spark.sql.types._ +import org.scalatest.tags.Cloud -class ClickHouseSingleSuite extends SparkClickHouseSingleTest { +@Cloud +class ClickHouseCloudGenericSuite extends ClickHouseDataTypeSuite with ClickHouseCloudMixIn + +class ClickHouseSingleGenericSuite extends ClickHouseDataTypeSuite with ClickHouseSingleMixIn + +abstract class ClickHouseGenericSuite extends SparkClickHouseSingleTest { import testImplicits._ diff --git a/spark-3.4/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/ClickHouseTableDDLSuite.scala b/spark-3.4/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/ClickHouseTableDDLSuite.scala index 7409a590..f3918098 100644 --- a/spark-3.4/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/ClickHouseTableDDLSuite.scala +++ b/spark-3.4/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/ClickHouseTableDDLSuite.scala @@ -14,15 +14,22 @@ package org.apache.spark.sql.clickhouse.single +import com.clickhouse.spark.base.{ClickHouseCloudMixIn, ClickHouseSingleMixIn} import org.apache.spark.sql.Row +import org.scalatest.tags.Cloud -class ClickHouseTableDDLSuite extends SparkClickHouseSingleTest { +@Cloud +class ClickHouseCloudTableDDLSuite extends ClickHouseTableDDLSuite with ClickHouseCloudMixIn + +class ClickHouseSingleTableDDLSuite extends ClickHouseTableDDLSuite with ClickHouseSingleMixIn + +abstract class ClickHouseTableDDLSuite extends SparkClickHouseSingleTest { import testImplicits._ test("clickhouse command runner") { withTable("default.abc") { - runClickHouseSQL("CREATE TABLE default.abc(a UInt8) ENGINE=Log()") + runClickHouseSQL("CREATE TABLE default.abc(a UInt8) ENGINE=Memory()") checkAnswer( spark.sql("""DESC default.abc""").select($"col_name", $"data_type").limit(1), Row("a", "smallint") :: Nil diff --git a/spark-3.4/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/SparkClickHouseSingleTest.scala b/spark-3.4/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/SparkClickHouseSingleTest.scala index 65f7f372..d9e7890a 100644 --- a/spark-3.4/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/SparkClickHouseSingleTest.scala +++ b/spark-3.4/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/SparkClickHouseSingleTest.scala @@ -14,13 +14,13 @@ package org.apache.spark.sql.clickhouse.single +import com.clickhouse.spark.base.ClickHouseProvider import org.apache.spark.SparkConf import org.apache.spark.sql.clickhouse.SparkTest import org.apache.spark.sql.functions.month import org.apache.spark.sql.types.StructType -import com.clickhouse.spark.base.ClickHouseSingleMixIn -trait SparkClickHouseSingleTest extends SparkTest with ClickHouseSingleMixIn { +trait SparkClickHouseSingleTest extends SparkTest with ClickHouseProvider { import testImplicits._ @@ -34,10 +34,11 @@ trait SparkClickHouseSingleTest extends SparkTest with ClickHouseSingleMixIn { .set("spark.sql.catalog.clickhouse.host", clickhouseHost) .set("spark.sql.catalog.clickhouse.http_port", clickhouseHttpPort.toString) .set("spark.sql.catalog.clickhouse.protocol", "http") - .set("spark.sql.catalog.clickhouse.user", CLICKHOUSE_USER) - .set("spark.sql.catalog.clickhouse.password", CLICKHOUSE_PASSWORD) - .set("spark.sql.catalog.clickhouse.database", CLICKHOUSE_DB) + .set("spark.sql.catalog.clickhouse.user", clickhouseUser) + .set("spark.sql.catalog.clickhouse.password", clickhousePassword) + .set("spark.sql.catalog.clickhouse.database", clickhouseDatabase) .set("spark.sql.catalog.clickhouse.option.custom_http_params", "async_insert=1,wait_for_async_insert=1") + .set("spark.sql.catalog.clickhouse.option.ssl", isSslEnabled.toString) // extended configurations .set("spark.clickhouse.write.batchSize", "2") .set("spark.clickhouse.write.maxRetry", "2") @@ -51,9 +52,11 @@ trait SparkClickHouseSingleTest extends SparkTest with ClickHouseSingleMixIn { "host" -> clickhouseHost, "http_port" -> clickhouseHttpPort.toString, "protocol" -> "http", - "user" -> CLICKHOUSE_USER, - "password" -> CLICKHOUSE_PASSWORD, - "database" -> CLICKHOUSE_DB + "user" -> clickhouseUser, + "password" -> clickhousePassword, + "database" -> clickhouseDatabase, + "option.custom_http_params" -> "async_insert=1,wait_for_async_insert=1", + "option.ssl" -> isSslEnabled.toString ) def withTable( diff --git a/spark-3.4/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/TPCDSSuite.scala b/spark-3.4/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/TPCDSSuite.scala index cda9793e..cad773c1 100644 --- a/spark-3.4/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/TPCDSSuite.scala +++ b/spark-3.4/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/TPCDSSuite.scala @@ -14,12 +14,18 @@ package org.apache.spark.sql.clickhouse.single +import com.clickhouse.spark.base.{ClickHouseCloudMixIn, ClickHouseSingleMixIn} import org.apache.spark.SparkConf import org.apache.spark.sql.clickhouse.TPCDSTestUtils -import org.scalatest.tags.Slow +import org.scalatest.tags.{Cloud, Slow} + +@Cloud +class ClickHouseCloudTPCDSSuite extends TPCDSSuite with ClickHouseCloudMixIn @Slow -class TPCDSSuite extends SparkClickHouseSingleTest { +class ClickHouseSingleTPCDSSuite extends TPCDSSuite with ClickHouseSingleMixIn + +abstract class TPCDSSuite extends SparkClickHouseSingleTest { override protected def sparkConf: SparkConf = super.sparkConf .set("spark.sql.catalog.tpcds", "org.apache.kyuubi.spark.connector.tpcds.TPCDSCatalog") diff --git a/spark-3.4/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/WriteDistributionAndOrderingSuite.scala b/spark-3.4/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/WriteDistributionAndOrderingSuite.scala index 7fc0972d..6469c07d 100644 --- a/spark-3.4/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/WriteDistributionAndOrderingSuite.scala +++ b/spark-3.4/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/WriteDistributionAndOrderingSuite.scala @@ -14,14 +14,21 @@ package org.apache.spark.sql.clickhouse.single +import com.clickhouse.spark.base.{ClickHouseCloudMixIn, ClickHouseSingleMixIn} import org.apache.spark.sql.clickhouse.ClickHouseSQLConf._ import org.apache.spark.sql.functions._ import org.apache.spark.sql.types.StringType -import org.apache.spark.sql.{AnalysisException, Row} +import org.apache.spark.sql.Row +import org.scalatest.tags.Cloud -import java.time.LocalDate +@Cloud +class ClickHouseCloudsWriteDistributionAndOrderingSuite + extends WriteDistributionAndOrderingSuite with ClickHouseCloudMixIn -class WriteDistributionAndOrderingSuite extends SparkClickHouseSingleTest { +class ClickHouseSinglesWriteDistributionAndOrderingSuite + extends WriteDistributionAndOrderingSuite with ClickHouseSingleMixIn + +abstract class WriteDistributionAndOrderingSuite extends SparkClickHouseSingleTest { import testImplicits._ diff --git a/spark-3.5/build.gradle b/spark-3.5/build.gradle index a3c02522..559293b2 100644 --- a/spark-3.5/build.gradle +++ b/spark-3.5/build.gradle @@ -97,4 +97,8 @@ project(":clickhouse-spark-it-${spark_binary_version}_$scala_binary_version") { slowTest { classpath += files("${project(':clickhouse-core').projectDir}/src/testFixtures/conf") } + + cloudTest { + classpath += files("${project(':clickhouse-core').projectDir}/src/testFixtures/conf") + } } diff --git a/spark-3.5/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/ClickHouseDataTypeSuite.scala b/spark-3.5/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/ClickHouseDataTypeSuite.scala index 03f9f93c..62f6a15d 100644 --- a/spark-3.5/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/ClickHouseDataTypeSuite.scala +++ b/spark-3.5/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/ClickHouseDataTypeSuite.scala @@ -14,15 +14,22 @@ package org.apache.spark.sql.clickhouse.single +import com.clickhouse.spark.base.{ClickHouseCloudMixIn, ClickHouseSingleMixIn} import org.apache.spark.sql.clickhouse.ClickHouseSQLConf.USE_NULLABLE_QUERY_SCHEMA import org.apache.spark.sql.clickhouse.SparkUtils import org.apache.spark.sql.types.DataTypes.{createArrayType, createMapType} import org.apache.spark.sql.types._ import org.apache.spark.sql.{DataFrame, Row} +import org.scalatest.tags.Cloud import java.math.MathContext -class ClickHouseDataTypeSuite extends SparkClickHouseSingleTest { +@Cloud +class ClickHouseCloudDataTypeSuite extends ClickHouseDataTypeSuite with ClickHouseCloudMixIn + +class ClickHouseSingleDataTypeSuite extends ClickHouseDataTypeSuite with ClickHouseSingleMixIn + +abstract class ClickHouseDataTypeSuite extends SparkClickHouseSingleTest { val SPARK_43390_ENABLED: Boolean = sys.env.contains("SPARK_43390_ENABLED") || { SparkUtils.MAJOR_MINOR_VERSION match { @@ -178,6 +185,9 @@ class ClickHouseDataTypeSuite extends SparkClickHouseSingleTest { : Unit = { val db = "test_kv_db" val tbl = "test_kv_tbl" + if (!clickhouseVersion.isNewerOrEqualTo("23.3") || isCloud) { + Thread.sleep(1000) + } withKVTable(db, tbl, valueColDef = valueColDef) { prepare(db, tbl) val df = spark.sql(s"SELECT key, value FROM $db.$tbl ORDER BY key") diff --git a/spark-3.5/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/ClickHouseSingleSuite.scala b/spark-3.5/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/ClickHouseGenericSuite.scala similarity index 97% rename from spark-3.5/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/ClickHouseSingleSuite.scala rename to spark-3.5/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/ClickHouseGenericSuite.scala index a458782f..74725190 100644 --- a/spark-3.5/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/ClickHouseSingleSuite.scala +++ b/spark-3.5/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/ClickHouseGenericSuite.scala @@ -14,12 +14,19 @@ package org.apache.spark.sql.clickhouse.single +import com.clickhouse.spark.base.{ClickHouseCloudMixIn, ClickHouseSingleMixIn} import org.apache.spark.sql.Row import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.execution.datasources.v2.BatchScanExec import org.apache.spark.sql.types._ +import org.scalatest.tags.Cloud -class ClickHouseSingleSuite extends SparkClickHouseSingleTest { +@Cloud +class ClickHouseCloudGenericSuite extends ClickHouseDataTypeSuite with ClickHouseCloudMixIn + +class ClickHouseSingleGenericSuite extends ClickHouseDataTypeSuite with ClickHouseSingleMixIn + +abstract class ClickHouseGenericSuite extends SparkClickHouseSingleTest { import testImplicits._ diff --git a/spark-3.5/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/ClickHouseTableDDLSuite.scala b/spark-3.5/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/ClickHouseTableDDLSuite.scala index 7409a590..f3918098 100644 --- a/spark-3.5/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/ClickHouseTableDDLSuite.scala +++ b/spark-3.5/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/ClickHouseTableDDLSuite.scala @@ -14,15 +14,22 @@ package org.apache.spark.sql.clickhouse.single +import com.clickhouse.spark.base.{ClickHouseCloudMixIn, ClickHouseSingleMixIn} import org.apache.spark.sql.Row +import org.scalatest.tags.Cloud -class ClickHouseTableDDLSuite extends SparkClickHouseSingleTest { +@Cloud +class ClickHouseCloudTableDDLSuite extends ClickHouseTableDDLSuite with ClickHouseCloudMixIn + +class ClickHouseSingleTableDDLSuite extends ClickHouseTableDDLSuite with ClickHouseSingleMixIn + +abstract class ClickHouseTableDDLSuite extends SparkClickHouseSingleTest { import testImplicits._ test("clickhouse command runner") { withTable("default.abc") { - runClickHouseSQL("CREATE TABLE default.abc(a UInt8) ENGINE=Log()") + runClickHouseSQL("CREATE TABLE default.abc(a UInt8) ENGINE=Memory()") checkAnswer( spark.sql("""DESC default.abc""").select($"col_name", $"data_type").limit(1), Row("a", "smallint") :: Nil diff --git a/spark-3.5/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/SparkClickHouseSingleTest.scala b/spark-3.5/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/SparkClickHouseSingleTest.scala index ca80c49f..d9e7890a 100644 --- a/spark-3.5/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/SparkClickHouseSingleTest.scala +++ b/spark-3.5/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/SparkClickHouseSingleTest.scala @@ -14,13 +14,13 @@ package org.apache.spark.sql.clickhouse.single -import com.clickhouse.spark.base.ClickHouseSingleMixIn +import com.clickhouse.spark.base.ClickHouseProvider import org.apache.spark.SparkConf import org.apache.spark.sql.clickhouse.SparkTest import org.apache.spark.sql.functions.month import org.apache.spark.sql.types.StructType -trait SparkClickHouseSingleTest extends SparkTest with ClickHouseSingleMixIn { +trait SparkClickHouseSingleTest extends SparkTest with ClickHouseProvider { import testImplicits._ @@ -34,10 +34,11 @@ trait SparkClickHouseSingleTest extends SparkTest with ClickHouseSingleMixIn { .set("spark.sql.catalog.clickhouse.host", clickhouseHost) .set("spark.sql.catalog.clickhouse.http_port", clickhouseHttpPort.toString) .set("spark.sql.catalog.clickhouse.protocol", "http") - .set("spark.sql.catalog.clickhouse.user", CLICKHOUSE_USER) - .set("spark.sql.catalog.clickhouse.password", CLICKHOUSE_PASSWORD) - .set("spark.sql.catalog.clickhouse.database", CLICKHOUSE_DB) + .set("spark.sql.catalog.clickhouse.user", clickhouseUser) + .set("spark.sql.catalog.clickhouse.password", clickhousePassword) + .set("spark.sql.catalog.clickhouse.database", clickhouseDatabase) .set("spark.sql.catalog.clickhouse.option.custom_http_params", "async_insert=1,wait_for_async_insert=1") + .set("spark.sql.catalog.clickhouse.option.ssl", isSslEnabled.toString) // extended configurations .set("spark.clickhouse.write.batchSize", "2") .set("spark.clickhouse.write.maxRetry", "2") @@ -51,9 +52,11 @@ trait SparkClickHouseSingleTest extends SparkTest with ClickHouseSingleMixIn { "host" -> clickhouseHost, "http_port" -> clickhouseHttpPort.toString, "protocol" -> "http", - "user" -> CLICKHOUSE_USER, - "password" -> CLICKHOUSE_PASSWORD, - "database" -> CLICKHOUSE_DB + "user" -> clickhouseUser, + "password" -> clickhousePassword, + "database" -> clickhouseDatabase, + "option.custom_http_params" -> "async_insert=1,wait_for_async_insert=1", + "option.ssl" -> isSslEnabled.toString ) def withTable( diff --git a/spark-3.5/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/TPCDSSuite.scala b/spark-3.5/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/TPCDSSuite.scala index cda9793e..cad773c1 100644 --- a/spark-3.5/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/TPCDSSuite.scala +++ b/spark-3.5/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/TPCDSSuite.scala @@ -14,12 +14,18 @@ package org.apache.spark.sql.clickhouse.single +import com.clickhouse.spark.base.{ClickHouseCloudMixIn, ClickHouseSingleMixIn} import org.apache.spark.SparkConf import org.apache.spark.sql.clickhouse.TPCDSTestUtils -import org.scalatest.tags.Slow +import org.scalatest.tags.{Cloud, Slow} + +@Cloud +class ClickHouseCloudTPCDSSuite extends TPCDSSuite with ClickHouseCloudMixIn @Slow -class TPCDSSuite extends SparkClickHouseSingleTest { +class ClickHouseSingleTPCDSSuite extends TPCDSSuite with ClickHouseSingleMixIn + +abstract class TPCDSSuite extends SparkClickHouseSingleTest { override protected def sparkConf: SparkConf = super.sparkConf .set("spark.sql.catalog.tpcds", "org.apache.kyuubi.spark.connector.tpcds.TPCDSCatalog") diff --git a/spark-3.5/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/WriteDistributionAndOrderingSuite.scala b/spark-3.5/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/WriteDistributionAndOrderingSuite.scala index 7fc0972d..6469c07d 100644 --- a/spark-3.5/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/WriteDistributionAndOrderingSuite.scala +++ b/spark-3.5/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/single/WriteDistributionAndOrderingSuite.scala @@ -14,14 +14,21 @@ package org.apache.spark.sql.clickhouse.single +import com.clickhouse.spark.base.{ClickHouseCloudMixIn, ClickHouseSingleMixIn} import org.apache.spark.sql.clickhouse.ClickHouseSQLConf._ import org.apache.spark.sql.functions._ import org.apache.spark.sql.types.StringType -import org.apache.spark.sql.{AnalysisException, Row} +import org.apache.spark.sql.Row +import org.scalatest.tags.Cloud -import java.time.LocalDate +@Cloud +class ClickHouseCloudsWriteDistributionAndOrderingSuite + extends WriteDistributionAndOrderingSuite with ClickHouseCloudMixIn -class WriteDistributionAndOrderingSuite extends SparkClickHouseSingleTest { +class ClickHouseSinglesWriteDistributionAndOrderingSuite + extends WriteDistributionAndOrderingSuite with ClickHouseSingleMixIn + +abstract class WriteDistributionAndOrderingSuite extends SparkClickHouseSingleTest { import testImplicits._