Support showing query digest in SHOW PROCESS LIST

Summary:
Add a new show_query_digest session var so that SHOW PROCESSLIST and I_S.PROCESSLIST shows query digest instead of original query. This is needed to ensure privacy / leaking PII when our tooling dumps out the queries.

Note this feature requires sql_stats_control to be on so that parser will generate digest tokens as part of tokenization. Without sql_stats_control, it'll just show `<digest_missing: sql_stats_control required>`, instead of original query string, for obvious privacy reasons. We have it in our production (at least in places where it matters) so this should be a fine requirement.

The implementation took a bit longer than expected mostly because CSET_STRING / LEX_STRING / String types in MySQL are rather difficult to work with, especially if you want to avoid unnecessary allocation and have correct const-ness when converting between them, so I changed CSET_STRING in thd_info to be just regular `char *` which are much easier.

Reference Patch: facebook@898d2fd

Porting Notes: sql_stats_control are no longer required in 8.0.

Reviewed By: luqun

Differential Revision: D26264225

mysql-test/r/mysqld--help-notwin.result

@@ -2361,6 +2361,8 @@ The following options may be given as the first argument:
  'INFORMATION_SCHEMA.COLUMNS' table as a comment in
  COLUMN_TYPE field. This variable is deprecated and will
  be removed in a future release.
+ --show-query-digest Show query digest instead of full query in show process
+ list.
  --show-replica-auth-info 
  Include user and password in SHOW REPLICAS statements.
  --show-slave-auth-info 
@@ -3456,6 +3458,7 @@ show-create-table-contain-privacy-policy FALSE
 show-create-table-verbosity FALSE
 show-gipk-in-create-table-and-information-schema TRUE
 show-old-temporals FALSE
+show-query-digest FALSE
 show-replica-auth-info FALSE
 show-slave-auth-info FALSE
 skip-core-dump-on-error TRUE

mysql-test/r/show_processlist_digest.result

@@ -0,0 +1,22 @@
+SET @show_query_digest_save = @@show_query_digest;
+SET show_query_digest=1;
+SELECT GET_LOCK('t', 1000);
+GET_LOCK('t', 1000)
+1
+SET NAMES latin1;
+SELECT 1+2, "abc", GET_LOCK('t',1000) AS 'abc';;
+SHOW PROCESSLIST;
+Id	User	Host	db	Command	Time	State	Info	Tid
+###	event_scheduler	###	NULL	Daemon	###	###	NULL	###
+###	root	###	test	Query	###	###	SELECT ? + ?, ... , `GET_LOCK` (...) AS ?	###
+###	root	###	test	Query	###	###	SHOW PROCESSLIST	###
+SELECT INFO FROM INFORMATION_SCHEMA.PROCESSLIST WHERE ID != CONNECTION_ID();
+INFO
+NULL
+SELECT 1+2, "abc", GET_LOCK('t',1000) AS 'abc'
+SELECT RELEASE_LOCK('t');
+RELEASE_LOCK('t')
+1
+1+2	abc	abc
+3	abc	1
+SET show_query_digest=@show_query_digest_save;

mysql-test/suite/sys_vars/r/show_query_digest_basic.result

@@ -0,0 +1,83 @@
+SET @session_start_value = @@session.show_query_digest;
+SELECT @session_start_value;
+@session_start_value
+0
+SET @@session.show_query_digest = 0;
+SET @@session.show_query_digest = DEFAULT;
+SELECT @@session.show_query_digest;
+@@session.show_query_digest
+0
+SET @@session.show_query_digest = 1;
+SET @@session.show_query_digest = DEFAULT;
+SELECT @@session.show_query_digest;
+@@session.show_query_digest
+0
+SET show_query_digest = 1;
+SELECT @@show_query_digest;
+@@show_query_digest
+1
+SELECT session.show_query_digest;
+ERROR 42S02: Unknown table 'session' in field list
+SELECT local.show_query_digest;
+ERROR 42S02: Unknown table 'local' in field list
+SET session show_query_digest = 0;
+SELECT @@session.show_query_digest;
+@@session.show_query_digest
+0
+SET @@session.show_query_digest = 0;
+SELECT @@session.show_query_digest;
+@@session.show_query_digest
+0
+SET @@session.show_query_digest = 1;
+SELECT @@session.show_query_digest;
+@@session.show_query_digest
+1
+SET @@session.show_query_digest = -1;
+ERROR 42000: Variable 'show_query_digest' can't be set to the value of '-1'
+SET @@session.show_query_digest = 2;
+ERROR 42000: Variable 'show_query_digest' can't be set to the value of '2'
+SET @@session.show_query_digest = "T";
+ERROR 42000: Variable 'show_query_digest' can't be set to the value of 'T'
+SET @@session.show_query_digest = "Y";
+ERROR 42000: Variable 'show_query_digest' can't be set to the value of 'Y'
+SET @@global.show_query_digest = 1;
+SELECT @@global.show_query_digest;
+@@global.show_query_digest
+1
+SET @@global.show_query_digest = 0;
+SELECT count(VARIABLE_VALUE) FROM performance_schema.global_variables WHERE VARIABLE_NAME='show_query_digest';
+count(VARIABLE_VALUE)
+1
+SELECT IF(@@session.show_query_digest, "ON", "OFF") = VARIABLE_VALUE
+FROM performance_schema.session_variables
+WHERE VARIABLE_NAME='show_query_digest';
+IF(@@session.show_query_digest, "ON", "OFF") = VARIABLE_VALUE
+1
+SELECT @@session.show_query_digest;
+@@session.show_query_digest
+1
+SELECT VARIABLE_VALUE
+FROM performance_schema.session_variables
+WHERE VARIABLE_NAME='show_query_digest';
+VARIABLE_VALUE
+ON
+SET @@session.show_query_digest = OFF;
+SELECT @@session.show_query_digest;
+@@session.show_query_digest
+0
+SET @@session.show_query_digest = ON;
+SELECT @@session.show_query_digest;
+@@session.show_query_digest
+1
+SET @@session.show_query_digest = TRUE;
+SELECT @@session.show_query_digest;
+@@session.show_query_digest
+1
+SET @@session.show_query_digest = FALSE;
+SELECT @@session.show_query_digest;
+@@session.show_query_digest
+0
+SET @@session.show_query_digest = @session_start_value;
+SELECT @@session.show_query_digest;
+@@session.show_query_digest
+0

mysql-test/suite/sys_vars/t/show_query_digest_basic.test

@@ -0,0 +1,135 @@
+####################### mysql-test\t\show_query_digest_basic.test ###############
+#                                                                             #
+# Variable Name: show_query_digest                                              #
+# Scope: SESSION                                                              #
+# Access Type: Dynamic                                                        #
+# Data Type: boolean                                                          #
+# Default Value:                                                              #
+# Valid Values: 0,1                                                           #
+#                                                                             #
+# Description: Test Cases of Dynamic System Variable show_query_digest          #
+#              that checks the behavior of this variable in the following ways#
+#              * Default Value                                                #
+#              * Valid & Invalid values                                       #
+#              * Scope & Access method                                        #
+#              * Data Integrity                                               #
+#                                                                             #
+###############################################################################
+
+--source include/load_sysvars.inc
+
+
+##############################################################################
+#      Saving initial value of show_query_digest in a temporary variable       #
+##############################################################################
+
+SET @session_start_value = @@session.show_query_digest;
+SELECT @session_start_value;
+
+########################################################################
+#           Display the DEFAULT value of show_query_digest               #
+########################################################################
+
+SET @@session.show_query_digest = 0;
+SET @@session.show_query_digest = DEFAULT;
+SELECT @@session.show_query_digest;
+
+SET @@session.show_query_digest = 1;
+SET @@session.show_query_digest = DEFAULT;
+SELECT @@session.show_query_digest;
+
+
+#############################################################################
+#   Check if show_query_digest can be accessed with and without @@ sign       #
+#############################################################################
+
+SET show_query_digest = 1;
+SELECT @@show_query_digest;
+
+--Error ER_UNKNOWN_TABLE
+SELECT session.show_query_digest;
+
+--Error ER_UNKNOWN_TABLE
+SELECT local.show_query_digest;
+
+SET session show_query_digest = 0;
+SELECT @@session.show_query_digest;
+
+
+########################################################################
+#      change the value of show_query_digest to a valid value            #
+########################################################################
+
+SET @@session.show_query_digest = 0;
+SELECT @@session.show_query_digest;
+SET @@session.show_query_digest = 1;
+SELECT @@session.show_query_digest;
+
+
+###########################################################################
+#       Change the value of show_query_digest to invalid value              #
+###########################################################################
+
+--Error ER_WRONG_VALUE_FOR_VAR
+SET @@session.show_query_digest = -1;
+--Error ER_WRONG_VALUE_FOR_VAR
+SET @@session.show_query_digest = 2;
+--Error ER_WRONG_VALUE_FOR_VAR
+SET @@session.show_query_digest = "T";
+--Error ER_WRONG_VALUE_FOR_VAR
+SET @@session.show_query_digest = "Y";
+
+###########################################################################
+#       Test if accessing global show_query_digest gives error              #
+###########################################################################
+
+SET @@global.show_query_digest = 1;
+SELECT @@global.show_query_digest;
+SET @@global.show_query_digest = 0;
+
+#########################################################################
+#     Check if the value in GLOBAL Table contains variable value        #
+#########################################################################
+
+SELECT count(VARIABLE_VALUE) FROM performance_schema.global_variables WHERE VARIABLE_NAME='show_query_digest';
+
+#########################################################################
+#     Check if the value in GLOBAL Table matches value in variable      #
+#########################################################################
+
+SELECT IF(@@session.show_query_digest, "ON", "OFF") = VARIABLE_VALUE
+FROM performance_schema.session_variables
+WHERE VARIABLE_NAME='show_query_digest';
+SELECT @@session.show_query_digest;
+SELECT VARIABLE_VALUE
+FROM performance_schema.session_variables
+WHERE VARIABLE_NAME='show_query_digest';
+
+###################################################################
+#        Check if ON and OFF values can be used on variable       #
+###################################################################
+
+SET @@session.show_query_digest = OFF;
+SELECT @@session.show_query_digest;
+SET @@session.show_query_digest = ON;
+SELECT @@session.show_query_digest;
+
+###################################################################
+#      Check if TRUE and FALSE values can be used on variable     #
+###################################################################
+
+SET @@session.show_query_digest = TRUE;
+SELECT @@session.show_query_digest;
+SET @@session.show_query_digest = FALSE;
+SELECT @@session.show_query_digest;
+
+##############################
+#   Restore initial value    #
+##############################
+
+SET @@session.show_query_digest = @session_start_value;
+SELECT @@session.show_query_digest;
+
+###############################################################
+#                END OF show_query_digest TESTS                 #
+###############################################################

mysql-test/t/show_processlist_digest.test

@@ -0,0 +1,30 @@
+#
+# SHOW PROCESSLIST with show_query_digest=1
+#
+SET @show_query_digest_save = @@show_query_digest;
+SET show_query_digest=1;
+
+SELECT GET_LOCK('t', 1000);
+--connect (con1,localhost,root,,)
+--connection con1
+SET NAMES latin1;
+--send SELECT 1+2, "abc", GET_LOCK('t',1000) AS 'abc';
+--connection default
+# Make sure con1 has switched from "SET NAMES" to "SELECT GET_LOCK"
+let $wait_timeout= 10;
+let $wait_condition= SELECT COUNT(*) FROM INFORMATION_SCHEMA.PROCESSLIST WHERE INFO LIKE '%GET_LOCK%' AND ID != CONNECTION_ID();
+--source include/wait_condition.inc
+--replace_column 1 ### 3 ### 6 ### 7 ### 9 ### 10 ### 11 ### 12 ###
+--replace_result 'Query attributes' 'Query'
+--sorted_result
+SHOW PROCESSLIST;
+--sorted_result
+SELECT INFO FROM INFORMATION_SCHEMA.PROCESSLIST WHERE ID != CONNECTION_ID();
+SELECT RELEASE_LOCK('t');
+--connection con1
+--reap
+--disconnect con1
+--source include/wait_until_disconnected.inc
+--connection default
+
+SET show_query_digest=@show_query_digest_save;

sql/mysqld.cc

@@ -1533,6 +1533,8 @@ ulonglong global_conn_mem_counter = 0;
 bool opt_group_replication_plugin_hooks = false;
 bool opt_core_file = false;
 bool skip_core_dump_on_error = false;
+bool show_query_digest = false;
+
 /* write_control_level:
  * Global variable to control write throttling for short running queries and
  * abort for long running queries.

sql/sql_show.cc

@@ -2853,7 +2853,9 @@ class thread_info {
   unsigned long long start_time_in_micro;
   uint command;
   const char *user, *host, *db, *proc_info, *state_info;
-  CSET_STRING query_string;
+  String digest_string;
+  const char *query_string = NULL;
+  const CHARSET_INFO *query_string_charset = &my_charset_bin;
 };
 
 // For sorting by thread_id.
@@ -2978,11 +2980,42 @@ int fill_schema_authinfo(THD *thd, Table_ref *tables, Item *) {
   DBUG_RETURN(0);
 }
 
+static const char *missing_digest = "<digest_missing>";
+
+static void get_query_digest(THD *tmp, String *digest_string, const char **str,
+                             const CHARSET_INFO **cs) {
+  if (tmp->m_digest != NULL) {
+    compute_digest_text(&tmp->m_digest->m_digest_storage, digest_string);
+  }
+
+  if (digest_string->is_empty() ||
+      (digest_string->length() == 1 && digest_string->ptr()[0] == 0)) {
+    /* We couldn't compute digest for whatever reason */
+    *str = missing_digest;
+    *cs = &my_charset_utf8mb3_bin;
+  } else {
+    *str = digest_string->c_ptr_safe();
+    *cs = digest_string->charset();
+  }
+}
+
 /**
   This class implements callback function used by mysqld_list_processes() to
   list all the client process information.
 */
-typedef Mem_root_array<thread_info *> Thread_info_array;
+class Thread_info_array : public Mem_root_array<thread_info *> {
+ public:
+  Thread_info_array(MEM_ROOT *mem_root)
+      : Mem_root_array<thread_info *>(mem_root) {}
+  ~Thread_info_array() {
+    // Destruct thread_info because it has non-trivial members that allocates
+    // using my_malloc and not THD mem_root
+    for (auto it = begin(); it != end(); it++) {
+      (*it)->~thread_info();
+    }
+  }
+};
+
 class List_process_list : public Do_THD_Impl {
  private:
   /* Username of connected client. */
@@ -3118,11 +3151,17 @@ class List_process_list : public Do_THD_Impl {
       /* No else. We need fall-through */
       /* If we managed to create query info, set a copy on thd_info. */
       if (query_str) {
-        const size_t width = min<size_t>(m_max_query_length, query_length);
-        const char *q = m_client_thd->strmake(query_str, width);
-        /* Safety: in case strmake failed, we set length to 0. */
-        thd_info->query_string =
-            CSET_STRING(q, q ? width : 0, inspect_thd->charset());
+        if (m_client_thd->variables.show_query_digest) {
+          get_query_digest(inspect_thd, &thd_info->digest_string,
+                           &thd_info->query_string,
+                           &thd_info->query_string_charset);
+        } else {
+          const size_t width = min<size_t>(m_max_query_length, query_length);
+          const char *q = m_client_thd->strmake(query_str, width);
+          /* Safety: in case strmake failed, we set length to 0. */
+          thd_info->query_string = q;
+          thd_info->query_string_charset = inspect_thd->charset();
+        }
       }
     }
     mysql_mutex_unlock(&inspect_thd->LOCK_thd_query);
@@ -3216,8 +3255,7 @@ void mysqld_list_processes(THD *thd, const char *user, bool verbose,
     } else
       protocol->store_null();
     protocol->store(thd_info->state_info, system_charset_info);
-    protocol->store(thd_info->query_string.str(),
-                    thd_info->query_string.charset());
+    protocol->store(thd_info->query_string, thd_info->query_string_charset);
     protocol->store((ulonglong)thd_info->tid);
     if (protocol->end_row()) break; /* purecov: inspected */
   }

sql/sys_vars.cc

@@ -9576,3 +9576,7 @@ static Sys_var_bool Sys_show_binlogs_encryption(
     "show_binlogs_encryption",
     "Scan binlogs to determine encryption property during show binlogs",
     GLOBAL_VAR(show_binlogs_encryption), CMD_LINE(OPT_ARG), DEFAULT(true));
+static Sys_var_bool Sys_show_query_digest(
+    "show_query_digest",
+    "Show query digest instead of full query in show process list.",
+    SESSION_VAR(show_query_digest), CMD_LINE(OPT_ARG), DEFAULT(false));