Skip to content

Commit

Permalink
Spark 3.4,3.5: Use correct identifier in view DESCRIBE cmd (apache#11751
Browse files Browse the repository at this point in the history 
)
  • Loading branch information
@Ppei-Wang
Ppei-Wang authored Dec 18, 2024
1 parent e3628c1 commit b428fbc
Show file tree
Hide file tree
Showing 4 changed files with 132 additions and 2 deletions.
4 changes: 3 additions & 1 deletion ...ons/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DescribeV2ViewExec.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,13 +55,15 @@ case class DescribeV2ViewExec(
private def describeExtended: Seq[InternalRow] = {
val outputColumns = view.queryColumnNames.mkString("[", ", ", "]")
val properties: Map[String, String] = view.properties.asScala.toMap -- ViewCatalog.RESERVED_PROPERTIES.asScala
val viewCatalogAndNamespace: Seq[String] = view.currentCatalog +: view.currentNamespace.toSeq
val viewCatalogAndNamespace: Seq[String] = view.name.split("\\.").take(2)
val viewProperties = properties.toSeq.sortBy(_._1).map {
case (key, value) =>
s"'${escapeSingleQuotedString(key)}' = '${escapeSingleQuotedString(value)}'"
}.mkString("[", ", ", "]")


// omitting view text here because it is shown as
// part of SHOW CREATE TABLE and can result in weird formatting in the DESCRIBE output
toCatalystRow("# Detailed View Information", "", "") ::
toCatalystRow("Comment", view.properties.getOrDefault(ViewCatalog.PROP_COMMENT, ""), "") ::
toCatalystRow("View Catalog and Namespace", viewCatalogAndNamespace.quoted, "") ::
Expand Down
63 changes: 63 additions & 0 deletions spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestViews.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@
import org.apache.iceberg.view.View;
import org.apache.iceberg.view.ViewHistoryEntry;
import org.apache.iceberg.view.ViewProperties;
import org.apache.iceberg.view.ViewUtil;
import org.apache.iceberg.view.ViewVersion;
import org.apache.spark.sql.AnalysisException;
import org.apache.spark.sql.Dataset;
Expand Down Expand Up @@ -1422,6 +1423,68 @@ public void describeExtendedView() {
""));
}

@Test
public void createAndDescribeViewInDefaultNamespace() {
String viewName = viewName("createViewInDefaultNamespace");
String sql = String.format("SELECT id, data FROM %s WHERE id <= 3", tableName);

sql("CREATE VIEW %s (id, data) AS %s", viewName, sql);
TableIdentifier identifier = TableIdentifier.of(NAMESPACE, viewName);
View view = viewCatalog().loadView(identifier);
assertThat(view.currentVersion().defaultCatalog()).isNull();
assertThat(view.name()).isEqualTo(ViewUtil.fullViewName(catalogName, identifier));
assertThat(view.currentVersion().defaultNamespace()).isEqualTo(NAMESPACE);

String location = viewCatalog().loadView(identifier).location();
assertThat(sql("DESCRIBE EXTENDED %s.%s", NAMESPACE, viewName))
.contains(
row("id", "int", ""),
row("data", "string", ""),
row("", "", ""),
row("# Detailed View Information", "", ""),
row("Comment", "", ""),
row("View Catalog and Namespace", String.format("%s.%s", catalogName, NAMESPACE), ""),
row("View Query Output Columns", "[id, data]", ""),
row(
"View Properties",
String.format(
"['format-version' = '1', 'location' = '%s', 'provider' = 'iceberg']",
location),
""));
}

@Test
public void createAndDescribeViewWithoutCurrentNamespace() {
String viewName = viewName("createViewWithoutCurrentNamespace");
Namespace namespace = Namespace.of("test_namespace");
String sql = String.format("SELECT id, data FROM %s WHERE id <= 3", tableName);

sql("CREATE NAMESPACE IF NOT EXISTS %s", namespace);
sql("CREATE VIEW %s.%s (id, data) AS %s", namespace, viewName, sql);
TableIdentifier identifier = TableIdentifier.of(namespace, viewName);
View view = viewCatalog().loadView(identifier);
assertThat(view.currentVersion().defaultCatalog()).isNull();
assertThat(view.name()).isEqualTo(ViewUtil.fullViewName(catalogName, identifier));
assertThat(view.currentVersion().defaultNamespace()).isEqualTo(NAMESPACE);

String location = viewCatalog().loadView(identifier).location();
assertThat(sql("DESCRIBE EXTENDED %s.%s", namespace, viewName))
.contains(
row("id", "int", ""),
row("data", "string", ""),
row("", "", ""),
row("# Detailed View Information", "", ""),
row("Comment", "", ""),
row("View Catalog and Namespace", String.format("%s.%s", catalogName, namespace), ""),
row("View Query Output Columns", "[id, data]", ""),
row(
"View Properties",
String.format(
"['format-version' = '1', 'location' = '%s', 'provider' = 'iceberg']",
location),
""));
}

@Test
public void showViewProperties() {
String viewName = viewName("showViewProps");
Expand Down
4 changes: 3 additions & 1 deletion ...ons/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DescribeV2ViewExec.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,13 +55,15 @@ case class DescribeV2ViewExec(
private def describeExtended: Seq[InternalRow] = {
val outputColumns = view.queryColumnNames.mkString("[", ", ", "]")
val properties: Map[String, String] = view.properties.asScala.toMap -- ViewCatalog.RESERVED_PROPERTIES.asScala
val viewCatalogAndNamespace: Seq[String] = view.currentCatalog +: view.currentNamespace.toSeq
val viewCatalogAndNamespace: Seq[String] = view.name.split("\\.").take(2)
val viewProperties = properties.toSeq.sortBy(_._1).map {
case (key, value) =>
s"'${escapeSingleQuotedString(key)}' = '${escapeSingleQuotedString(value)}'"
}.mkString("[", ", ", "]")


// omitting view text here because it is shown as
// part of SHOW CREATE TABLE and can result in weird formatting in the DESCRIBE output
toCatalystRow("# Detailed View Information", "", "") ::
toCatalystRow("Comment", view.properties.getOrDefault(ViewCatalog.PROP_COMMENT, ""), "") ::
toCatalystRow("View Catalog and Namespace", viewCatalogAndNamespace.quoted, "") ::
Expand Down
63 changes: 63 additions & 0 deletions spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestViews.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@
import org.apache.iceberg.view.View;
import org.apache.iceberg.view.ViewHistoryEntry;
import org.apache.iceberg.view.ViewProperties;
import org.apache.iceberg.view.ViewUtil;
import org.apache.iceberg.view.ViewVersion;
import org.apache.spark.sql.AnalysisException;
import org.apache.spark.sql.Dataset;
Expand Down Expand Up @@ -1451,6 +1452,68 @@ public void describeExtendedView() {
""));
}

@TestTemplate
public void createAndDescribeViewInDefaultNamespace() {
String viewName = viewName("createViewInDefaultNamespace");
String sql = String.format("SELECT id, data FROM %s WHERE id <= 3", tableName);

sql("CREATE VIEW %s (id, data) AS %s", viewName, sql);
TableIdentifier identifier = TableIdentifier.of(NAMESPACE, viewName);
View view = viewCatalog().loadView(identifier);
assertThat(view.currentVersion().defaultCatalog()).isNull();
assertThat(view.name()).isEqualTo(ViewUtil.fullViewName(catalogName, identifier));
assertThat(view.currentVersion().defaultNamespace()).isEqualTo(NAMESPACE);

String location = viewCatalog().loadView(identifier).location();
assertThat(sql("DESCRIBE EXTENDED %s.%s", NAMESPACE, viewName))
.contains(
row("id", "int", ""),
row("data", "string", ""),
row("", "", ""),
row("# Detailed View Information", "", ""),
row("Comment", "", ""),
row("View Catalog and Namespace", String.format("%s.%s", catalogName, NAMESPACE), ""),
row("View Query Output Columns", "[id, data]", ""),
row(
"View Properties",
String.format(
"['format-version' = '1', 'location' = '%s', 'provider' = 'iceberg']",
location),
""));
}

@TestTemplate
public void createAndDescribeViewWithoutCurrentNamespace() {
String viewName = viewName("createViewWithoutCurrentNamespace");
Namespace namespace = Namespace.of("test_namespace");
String sql = String.format("SELECT id, data FROM %s WHERE id <= 3", tableName);

sql("CREATE NAMESPACE IF NOT EXISTS %s", namespace);
sql("CREATE VIEW %s.%s (id, data) AS %s", namespace, viewName, sql);
TableIdentifier identifier = TableIdentifier.of(namespace, viewName);
View view = viewCatalog().loadView(identifier);
assertThat(view.currentVersion().defaultCatalog()).isNull();
assertThat(view.name()).isEqualTo(ViewUtil.fullViewName(catalogName, identifier));
assertThat(view.currentVersion().defaultNamespace()).isEqualTo(NAMESPACE);

String location = viewCatalog().loadView(identifier).location();
assertThat(sql("DESCRIBE EXTENDED %s.%s", namespace, viewName))
.contains(
row("id", "int", ""),
row("data", "string", ""),
row("", "", ""),
row("# Detailed View Information", "", ""),
row("Comment", "", ""),
row("View Catalog and Namespace", String.format("%s.%s", catalogName, namespace), ""),
row("View Query Output Columns", "[id, data]", ""),
row(
"View Properties",
String.format(
"['format-version' = '1', 'location' = '%s', 'provider' = 'iceberg']",
location),
""));
}

@TestTemplate
public void showViewProperties() {
String viewName = viewName("showViewProps");
Expand Down

0 comments on commit b428fbc

Please sign in to comment.