[SPARK-39342][SQL] ShowTablePropertiesCommand/ShowTablePropertiesExec…

… should redact properties ### What changes were proposed in this pull request? ShowTablePropertiesCommand/ShowTablePropertiesExec should redact sensitive properties. ### Why are the changes needed? Show table properties should redact sensitive information too. ### Does this PR introduce _any_ user-facing change? When user use `SHOW TABLE PROPERTIES`, sensitive information will be redacted. ### How was this patch tested? Added UT Closes apache#36730 from AngersZhuuuu/SPARK-39342. Authored-by: Angerszhuuuu <[email protected]> Signed-off-by: Wenchen Fan <[email protected]>
sluk3r · May 31, 2022 · 1b39764 · 1b39764
1 parent ef9e3f9
commit 1b39764
Show file tree

Hide file tree

Showing 5 changed files with 9 additions and 7 deletions.
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala
@@ -927,18 +927,18 @@ case class ShowTablePropertiesCommand(
       Seq.empty[Row]
     } else {
       val catalogTable = catalog.getTableMetadata(table)
+      val properties = conf.redactOptions(catalogTable.properties)
       propertyKey match {
         case Some(p) =>
-          val propValue = catalogTable
-            .properties
+          val propValue = properties
             .getOrElse(p, s"Table ${catalogTable.qualifiedName} does not have property: $p")
           if (output.length == 1) {
             Seq(Row(propValue))
           } else {
             Seq(Row(p, propValue))
           }
         case None =>
-          catalogTable.properties.filterKeys(!_.startsWith(CatalogTable.VIEW_PREFIX))
+          properties.filterKeys(!_.startsWith(CatalogTable.VIEW_PREFIX))
             .toSeq.sortBy(_._1).map(p => Row(p._1, p._2)).toSeq
       }
     }

diff --git a/...rc/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowTablePropertiesExec.scala b/...rc/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowTablePropertiesExec.scala
@@ -34,7 +34,7 @@ case class ShowTablePropertiesExec(
     import scala.collection.JavaConverters._
 
     // The reserved properties are accessible through DESCRIBE
-    val properties = catalogTable.properties.asScala
+    val properties = conf.redactOptions(catalogTable.properties.asScala.toMap)
       .filter { case (k, _) => !CatalogV2Util.TABLE_RESERVED_PROPERTIES.contains(k) }
     propertyKey match {
       case Some(p) =>

diff --git a/sql/core/src/test/resources/sql-tests/inputs/show-tblproperties.sql b/sql/core/src/test/resources/sql-tests/inputs/show-tblproperties.sql
@@ -1,6 +1,6 @@
 -- create a table with properties
 CREATE TABLE tbl (a INT, b STRING, c INT) USING parquet
-TBLPROPERTIES('p1'='v1', 'p2'='v2');
+TBLPROPERTIES('p1'='v1', 'p2'='v2', password = 'password');
 
 SHOW TBLPROPERTIES tbl;
 SHOW TBLPROPERTIES tbl("p1");

diff --git a/sql/core/src/test/resources/sql-tests/results/show-tblproperties.sql.out b/sql/core/src/test/resources/sql-tests/results/show-tblproperties.sql.out
@@ -4,7 +4,7 @@
 
 -- !query
 CREATE TABLE tbl (a INT, b STRING, c INT) USING parquet
-TBLPROPERTIES('p1'='v1', 'p2'='v2')
+TBLPROPERTIES('p1'='v1', 'p2'='v2', password = 'password')
 -- !query schema
 struct<>
 -- !query output
@@ -18,6 +18,7 @@ struct<key:string,value:string>
 -- !query output
 p1	v1
 p2	v2
+password	*********(redacted)
 
 
 -- !query

diff --git a/...re/src/test/scala/org/apache/spark/sql/execution/command/ShowTblPropertiesSuiteBase.scala b/...re/src/test/scala/org/apache/spark/sql/execution/command/ShowTblPropertiesSuiteBase.scala
@@ -41,14 +41,15 @@ trait ShowTblPropertiesSuiteBase extends QueryTest with DDLCommandTestUtils {
       val user = "andrew"
       val status = "new"
       spark.sql(s"CREATE TABLE $tbl (id bigint, data string) $defaultUsing " +
-        s"TBLPROPERTIES ('user'='$user', 'status'='$status')")
+        s"TBLPROPERTIES ('user'='$user', 'status'='$status', 'password' = 'password')")
       val properties = sql(s"SHOW TBLPROPERTIES $tbl")
         .filter("key != 'transient_lastDdlTime'")
         .filter("key != 'option.serialization.format'")
       val schema = new StructType()
         .add("key", StringType, nullable = false)
         .add("value", StringType, nullable = false)
       val expected = Seq(
+        Row("password", "*********(redacted)"),
         Row("status", status),
         Row("user", user))