WL#8955: Add multi-valued index support

WL#8763: Support multi-value functional index for InnoDB

Complete implementation and bug fixes.

Change-Id: Ie571964b136302f406c0b2756c90d1e465b68438

include/decimal.h

@@ -1,4 +1,4 @@
-/* Copyright (c) 2000, 2018, Oracle and/or its affiliates. All rights reserved.
+/* Copyright (c) 2000, 2019, Oracle and/or its affiliates. All rights reserved.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License, version 2.0,
@@ -67,7 +67,8 @@ int decimal2double(const decimal_t *from, double *to);
 int double2decimal(double from, decimal_t *to);
 int decimal_actual_fraction(const decimal_t *from);
 int decimal2bin(decimal_t *from, uchar *to, int precision, int scale);
-int bin2decimal(const uchar *from, decimal_t *to, int precision, int scale);
+int bin2decimal(const uchar *from, decimal_t *to, int precision, int scale,
+                bool keep_prec = false);
 
 /**
   Convert decimal to lldiv_t.

include/field_types.h

@@ -72,8 +72,9 @@ enum enum_field_types
   MYSQL_TYPE_VARCHAR,
   MYSQL_TYPE_BIT,
   MYSQL_TYPE_TIMESTAMP2,
-  MYSQL_TYPE_DATETIME2, /**< Internal to MySQL. Not used in protocol */
-  MYSQL_TYPE_TIME2,     /**< Internal to MySQL. Not used in protocol */
+  MYSQL_TYPE_DATETIME2,         /**< Internal to MySQL. Not used in protocol */
+  MYSQL_TYPE_TIME2,             /**< Internal to MySQL. Not used in protocol */
+  MYSQL_TYPE_TYPED_ARRAY = 244, /** Used for replication only */
   MYSQL_TYPE_JSON = 245,
   MYSQL_TYPE_NEWDECIMAL = 246,
   MYSQL_TYPE_ENUM = 247,

include/my_base.h

@@ -413,7 +413,14 @@ enum ha_extra_function {
   /* Insertion is done in intermediate table during copy alter operation. */
   HA_EXTRA_END_ALTER_COPY,
   /* Do not use auto-increment locking. */
-  HA_EXTRA_NO_AUTOINC_LOCKING
+  HA_EXTRA_NO_AUTOINC_LOCKING,
+  /*
+    Allocate and use unique filter to weed out duplicates.
+    Used with multi-valued index.
+  */
+  HA_EXTRA_ENABLE_UNIQUE_RECORD_FILTER,
+  /* Disable and free unique record filter. */
+  HA_EXTRA_DISABLE_UNIQUE_RECORD_FILTER
 };
 
 /* Compatible option, to be deleted in 6.0 */
@@ -466,40 +473,69 @@ enum ha_base_keytype {
 /** Do not allow duplicate records. */
 #define HA_NOSAME 1
 /** Pack string key to previous key (optimization supported by MyISAM). */
-#define HA_PACK_KEY 2
+#define HA_PACK_KEY (1 << 1)
+/*
+  Bits in KEY::flags, MI/HP_KEYDEF::flag which are automatically
+  calculated based on other flags/members in these structures
+  (often from info about key parts).
+*/
+/** Some key part packs space. Internal to MyISAM. */
+#define HA_SPACE_PACK_USED (1 << 2)
+/** Some key part has variable length. Internal to MyISAM and Heap engines. */
+#define HA_VAR_LENGTH_KEY (1 << 3)
 /**
   Auto-increment key.
 
   @note Not used by SQL-layer/ for KEY::flags. Only set by MyISAM and
         Heap SEs in MI/HP_KEYDEF::flag.
 */
-#define HA_AUTO_KEY 16
+#define HA_AUTO_KEY (1 << 4)
 /** Packing of all keys to previous key (optimization supported by MyISAM). */
-#define HA_BINARY_PACK_KEY 32
+#define HA_BINARY_PACK_KEY (1 << 5)
+/** Some key part is nullable. */
+#define HA_NULL_PART_KEY (1 << 6)
 /** Full-text key. */
-#define HA_FULLTEXT 128
+#define HA_FULLTEXT (1 << 7)
 /**
   Flag in MI_KEYDEF::flag which marks MyISAM's "uniques".
 
   @note Internal to MyISAM. Current server doesn't use this feature.
 */
-#define HA_UNIQUE_CHECK 256
+#define HA_UNIQUE_CHECK (1 << 8)
+/** Internal bit used when sorting records. Internal to MyISAM. */
+#define HA_SORT_ALLOWS_SAME (1 << 9)
 /** Spatial key. */
-#define HA_SPATIAL 1024
+#define HA_SPATIAL (1 << 10)
 /**
   NULLs in key are compared as equal.
 
   @note Used only for internal temporary tables created by optimizer.
 */
-#define HA_NULL_ARE_EQUAL 2048
+#define HA_NULL_ARE_EQUAL (1 << 11)
+/** Key has comment. */
+#define HA_USES_COMMENT (1 << 12)
 /** Key was automatically created to support Foreign Key constraint. */
-#define HA_GENERATED_KEY 8192
+#define HA_GENERATED_KEY (1 << 13)
 
 /* The combination of the above can be used for key type comparison. */
 #define HA_KEYFLAG_MASK                                                       \
   (HA_NOSAME | HA_PACK_KEY | HA_AUTO_KEY | HA_BINARY_PACK_KEY | HA_FULLTEXT | \
    HA_UNIQUE_CHECK | HA_SPATIAL | HA_NULL_ARE_EQUAL | HA_GENERATED_KEY)
 
+/** Fulltext index uses [pre]parser */
+#define HA_USES_PARSER (1 << 14)
+/** Key uses KEY_BLOCK_SIZE option. */
+#define HA_USES_BLOCK_SIZE (1 << 15)
+/**
+  Key contains partial segments.
+
+  @note This flag is internal to SQL-layer by design. It is not supposed to
+        be used to storage engines. It is intended to pass information into
+        internal static sort_keys(KEY *, KEY *) function.
+
+  This flag can be calculated -- it's based on key lengths comparison.
+*/
+#define HA_KEY_HAS_PART_KEY_SEG (1 << 16)
 /**
   Key was renamed (or is result of renaming a key).
 
@@ -514,37 +550,8 @@ enum ha_base_keytype {
 #define HA_KEY_RENAMED (1 << 17)
 /** Set if a key is on any virtual generated columns */
 #define HA_VIRTUAL_GEN_KEY (1 << 18)
-
-/*
-  Bits in KEY::flags, MI/HP_KEYDEF::flag which are automatically
-  calculated based on other flags/members in these structures
-  (often from info about key parts).
-*/
-
-/** Some key part packs space. Internal to MyISAM. */
-#define HA_SPACE_PACK_USED 4
-/** Some key part has variable length. Internal to MyISAM and Heap engines. */
-#define HA_VAR_LENGTH_KEY 8
-/** Some key part is nullable. */
-#define HA_NULL_PART_KEY 64
-/** Internal bit used when sorting records. Internal to MyISAM. */
-#define HA_SORT_ALLOWS_SAME 512
-/** Key has comment. */
-#define HA_USES_COMMENT 4096
-/** Fulltext index uses [pre]parser */
-#define HA_USES_PARSER 16384
-/** Key uses KEY_BLOCK_SIZE option. */
-#define HA_USES_BLOCK_SIZE 32768
-/**
-  Key contains partial segments.
-
-  @note This flag is internal to SQL-layer by design. It is not supposed to
-        be used to storage engines. It is intended to pass information into
-        internal static sort_keys(KEY *, KEY *) function.
-
-  This flag can be calculated -- it's based on key lengths comparison.
-*/
-#define HA_KEY_HAS_PART_KEY_SEG 65536
+/** Multi-valued key */
+#define HA_MULTI_VALUED_KEY (1 << 19)
 
 /* These flags can be added to key-seg-flag */
 

include/my_time.h

@@ -361,6 +361,7 @@ int my_date_to_str(const MYSQL_TIME &my_time, char *to);
 int my_datetime_to_str(const MYSQL_TIME &my_time, char *to, unsigned int dec);
 int my_TIME_to_str(const MYSQL_TIME &my_time, char *to, unsigned int dec);
 
+void my_date_to_binary(const MYSQL_TIME *ltime, unsigned char *ptr);
 int my_timeval_to_str(const struct timeval *tm, char *to, unsigned int dec);
 
 /**

include/mysql.h.pp

@@ -22,6 +22,7 @@
   MYSQL_TYPE_TIMESTAMP2,
   MYSQL_TYPE_DATETIME2,
   MYSQL_TYPE_TIME2,
+  MYSQL_TYPE_TYPED_ARRAY = 244,
   MYSQL_TYPE_JSON = 245,
   MYSQL_TYPE_NEWDECIMAL = 246,
   MYSQL_TYPE_ENUM = 247,

libbinlogevents/include/rows_event.h

@@ -387,6 +387,17 @@ namespace binary_log {
     the length of the geometry: 1, 2, 3, or 4.</td>
   </tr>
 
+  <tr>
+    <td>MYSQL_TYPE_TYPED_ARRAY</td><td>15</td>
+    <td>up to 4 bytes</td>
+    <td>
+      - The first byte holds the MySQL type for the elements.
+      - The following 0, 1, 2, or 3 bytes holds the metadata for the MySQL
+        type for the elements. The contents of these bytes depends on the
+        element type, as described in the other rows of this table.
+    </td>
+  </tr>
+
   </table>
 
   The table below lists all optional metadata types, along with the numerical

libbinlogevents/src/rows_event.cpp

@@ -85,7 +85,7 @@ Table_map_event::Table_map_event(const char *buf,
 
   if (READER_CALL(available_to_read) > 0) {
     READER_TRY_SET(m_field_metadata_size, net_field_length_ll);
-    if (m_field_metadata_size > (m_colcnt * 2))
+    if (m_field_metadata_size > (m_colcnt * 4))
       READER_THROW("Invalid m_field_metadata_size");
     unsigned int num_null_bytes = (m_colcnt + 7) / 8;
     READER_TRY_CALL(alloc_and_memcpy, &m_field_metadata, m_field_metadata_size,

mysql-test/r/delete.result

@@ -776,6 +776,29 @@ foobar	10000
 foobar	10000
 DROP TABLE t1, t2;
 #
+# Bug#29153485: ASSERTION IN JOIN_READ_CONST_TABLE()
+#
+CREATE TABLE d3 (
+pk int(11) NOT NULL AUTO_INCREMENT,
+col_int int(11) DEFAULT NULL,
+col_varchar varchar(1) DEFAULT NULL,
+PRIMARY KEY (pk)
+) ;
+INSERT INTO d3 VALUES (96,9,'t'),(97,0,'x');
+CREATE TABLE e3 (
+col_varchar varchar(1) DEFAULT NULL,
+pk int(11) NOT NULL AUTO_INCREMENT,
+PRIMARY KEY (pk)
+) ;
+INSERT INTO e3 VALUES ('v',986);
+DELETE OUTR1.* FROM e3 AS OUTR1 INNER JOIN d3 AS OUTR2 ON 
+( OUTR1.col_varchar = OUTR2.col_varchar )
+LEFT JOIN d3 AS OUTR3 ON ( OUTR1. pk = OUTR3.pk )
+WHERE OUTR1.pk =
+( SELECT DISTINCT INNR1.col_int FROM d3 AS INNR1 WHERE INNR1.pk <> 8) ;
+ERROR 21000: Subquery returns more than 1 row
+DROP TABLE d3,e3;
+#
 # Bug#27455809; DELETE FROM ...WHERE NOT EXISTS WITH TABLE ALIAS CREATES AN ERROR 1064 (42000
 #
 CREATE TABLE t1 (c1 INT);

mysql-test/r/functional_index.result

@@ -98,15 +98,15 @@ ERROR 42000: Cannot create a functional index on a function that returns a JSON
 INSERT INTO t1 (f1) VALUES (CAST(1000 AS JSON));
 ERROR 22003: Value is out of range for functional index 'my_functional_index' at row 1
 INSERT INTO t1 (f2) VALUES ("lorem ipsum");
-ERROR 22007: Truncated incorrect CHAR(1) value: 'lorem ipsum'
+ERROR 01000: Data truncated for functional index 'my_functional_index_2' at row 1
 # In non-strict mode, a warning should be printed.
 SET @@sql_mode='';
 INSERT INTO t1 (f1) VALUES (CAST(1000 AS JSON));
 Warnings:
 Warning	3752	Value is out of range for functional index 'my_functional_index' at row 1
 INSERT INTO t1 (f2) VALUES ("lorem ipsum");
 Warnings:
-Warning	1292	Truncated incorrect CHAR(1) value: 'lorem ipsum'
+Warning	3751	Data truncated for functional index 'my_functional_index_2' at row 1
 DROP TABLE t1;
 SET @@sql_mode=DEFAULT;
 CREATE TABLE t1 (t1_id INT NOT NULL AUTO_INCREMENT PRIMARY KEY);
@@ -205,13 +205,10 @@ f1
 {"id": 1}
 {"id": 2}
 {"id": 3}
-Warnings:
-Warning	1235	This version of MySQL doesn't yet support 'comparison of JSON in the IN operator'
 EXPLAIN SELECT * FROM t1 WHERE f1->"$.id" IN (1,2,3);
 id	select_type	table	partitions	type	possible_keys	key	key_len	ref	rows	filtered	Extra
 1	SIMPLE	t1	NULL	range	idx1	idx1	9	NULL	3	100.00	Using where
 Warnings:
-Warning	1235	This version of MySQL doesn't yet support 'comparison of JSON in the IN operator'
 Note	1003	/* select#1 */ select `test`.`t1`.`f1` AS `f1` from `test`.`t1` where (cast(json_extract(`test`.`t1`.`f1`,_utf8mb4'$.id') as unsigned) in (1,2,3))
 DROP TABLE t1;
 Test string index over a JSON key

mysql-test/r/functional_index_debug.result

@@ -124,6 +124,24 @@ t1	CREATE TABLE `t1` (
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci
 DROP TABLE t1;
 SET SESSION debug="-d,show_hidden_columns";
+SET SESSION debug="+d,show_hidden_columns";
+CREATE TABLE t1(j json, INDEX mv_idx((CAST(j AS UNSIGNED ARRAY))));
+SHOW CREATE TABLE t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `j` json DEFAULT NULL,
+  `680963a1f6dcb151c3e713b46b4c3452` bigint array GENERATED ALWAYS AS (cast(`j` as unsigned array)) VIRTUAL,
+  KEY `mv_idx` ((cast(`j` as unsigned array)))
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci
+SELECT 680963a1f6dcb151c3e713b46b4c3452 FROM t1 ;
+680963a1f6dcb151c3e713b46b4c3452
+EXPLAIN SELECT 680963a1f6dcb151c3e713b46b4c3452 FROM t1 ;
+id	select_type	table	partitions	type	possible_keys	key	key_len	ref	rows	filtered	Extra
+1	SIMPLE	t1	NULL	ALL	NULL	NULL	NULL	NULL	1	100.00	NULL
+Warnings:
+Note	1003	/* select#1 */ select cast(`test`.`t1`.`j` as unsigned array) AS `680963a1f6dcb151c3e713b46b4c3452` from `test`.`t1`
+SET SESSION debug="-d,show_hidden_columns";
+DROP TABLE t1;
 #
 # Bug#28037195 ASSERTION `FALSE' IN IN
 #              ITEM_FIELD::REPLACE_FIELD_PROCESSOR AT SQL_TABLE.CC:5850

mysql-test/r/heap_btree.result

@@ -344,7 +344,7 @@ UNIQUE KEY USING BTREE (c2,c1)) ENGINE = MEMORY;
 INSERT INTO t1(c1,c2) VALUES (5,1), (4,1), (3,5), (2,3), (1,3);
 EXPLAIN SELECT c2, COUNT(c1) FROM t1 GROUP BY c2;
 id	select_type	table	partitions	type	possible_keys	key	key_len	ref	rows	filtered	Extra
-1	SIMPLE	t1	NULL	index	NULL	c2	8	NULL	5	100.00	NULL
+1	SIMPLE	t1	NULL	index	c2	c2	8	NULL	5	100.00	NULL
 Warnings:
 Note	1003	/* select#1 */ select `test`.`t1`.`c2` AS `c2`,count(`test`.`t1`.`c1`) AS `COUNT(c1)` from `test`.`t1` group by `test`.`t1`.`c2`
 SELECT c2, COUNT(c1) FROM t1 GROUP BY c2;

mysql-test/r/heap_hash.result

@@ -466,7 +466,7 @@ UNIQUE KEY USING HASH (c2,c1)) ENGINE = MEMORY;
 INSERT INTO t1(c1,c2) VALUES (5,1), (4,1), (3,5), (2,3), (1,3);
 EXPLAIN SELECT c2, COUNT(c1) FROM t1 GROUP BY c2;
 id	select_type	table	partitions	type	possible_keys	key	key_len	ref	rows	filtered	Extra
-1	SIMPLE	t1	NULL	ALL	NULL	NULL	NULL	NULL	5	100.00	Using temporary
+1	SIMPLE	t1	NULL	ALL	c2	NULL	NULL	NULL	5	100.00	Using temporary
 Warnings:
 Note	1003	/* select#1 */ select `test`.`t1`.`c2` AS `c2`,count(`test`.`t1`.`c1`) AS `COUNT(c1)` from `test`.`t1` group by `test`.`t1`.`c2`
 SELECT c2, COUNT(c1) FROM t1 GROUP BY c2;

mysql-test/r/information_schema_keywords.result

@@ -19,6 +19,7 @@ ALWAYS	0
 ANALYZE	1
 AND	1
 ANY	0
+ARRAY	0
 AS	1
 ASC	1
 ASCII	0
@@ -347,6 +348,7 @@ MEDIUM	0
 MEDIUMBLOB	1
 MEDIUMINT	1
 MEDIUMTEXT	1
+MEMBER	1
 MEMORY	0
 MERGE	0
 MESSAGE_TEXT	0

mysql-test/r/innodb_explain_json_non_select_all.result

@@ -4018,6 +4018,7 @@ EXPLAIN
   } /* query_block */
 }
 Warnings:
+Warning	1739	Cannot use range access on index 'a' due to type or collation conversion on field 'b'
 Note	1003	/* select#1 */ select `test`.`t2`.`a` AS `a`,`test`.`t2`.`b` AS `b`,`test`.`t2`.`c` AS `c`,`test`.`t2`.`d` AS `d` from `test`.`t2` where (`test`.`t2`.`b` = 10) order by `test`.`t2`.`a`,`test`.`t2`.`c` limit 5
 # Status of "equivalent" SELECT query execution:
 Variable_name	Value
@@ -5042,6 +5043,7 @@ EXPLAIN
   } /* query_block */
 }
 Warnings:
+Warning	1739	Cannot use range access on index 'a' due to type or collation conversion on field 'b'
 Note	1003	/* select#1 */ select `test`.`t2`.`a` AS `a`,`test`.`t2`.`b` AS `b`,`test`.`t2`.`c` AS `c`,`test`.`t2`.`d` AS `d` from `test`.`t2` where (`test`.`t2`.`b` = 10) order by `test`.`t2`.`a`,`test`.`t2`.`c` limit 5
 # Status of "equivalent" SELECT query execution:
 Variable_name	Value

mysql-test/r/innodb_explain_json_non_select_none.result

@@ -4051,6 +4051,7 @@ EXPLAIN
   } /* query_block */
 }
 Warnings:
+Warning	1739	Cannot use range access on index 'a' due to type or collation conversion on field 'b'
 Note	1003	/* select#1 */ select `test`.`t2`.`a` AS `a`,`test`.`t2`.`b` AS `b`,`test`.`t2`.`c` AS `c`,`test`.`t2`.`d` AS `d` from `test`.`t2` where (`test`.`t2`.`b` = 10) order by `test`.`t2`.`a`,`test`.`t2`.`c` limit 5
 # Status of "equivalent" SELECT query execution:
 Variable_name	Value
@@ -5075,6 +5076,7 @@ EXPLAIN
   } /* query_block */
 }
 Warnings:
+Warning	1739	Cannot use range access on index 'a' due to type or collation conversion on field 'b'
 Note	1003	/* select#1 */ select `test`.`t2`.`a` AS `a`,`test`.`t2`.`b` AS `b`,`test`.`t2`.`c` AS `c`,`test`.`t2`.`d` AS `d` from `test`.`t2` where (`test`.`t2`.`b` = 10) order by `test`.`t2`.`a`,`test`.`t2`.`c` limit 5
 # Status of "equivalent" SELECT query execution:
 Variable_name	Value

mysql-test/r/myisam_explain_json_non_select_all.result

@@ -3939,6 +3939,7 @@ EXPLAIN
   } /* query_block */
 }
 Warnings:
+Warning	1739	Cannot use range access on index 'a' due to type or collation conversion on field 'b'
 Note	1003	/* select#1 */ select `test`.`t2`.`a` AS `a`,`test`.`t2`.`b` AS `b`,`test`.`t2`.`c` AS `c`,`test`.`t2`.`d` AS `d` from `test`.`t2` where (`test`.`t2`.`b` = 10) order by `test`.`t2`.`a`,`test`.`t2`.`c` limit 5
 # Status of "equivalent" SELECT query execution:
 Variable_name	Value
@@ -4956,6 +4957,7 @@ EXPLAIN
   } /* query_block */
 }
 Warnings:
+Warning	1739	Cannot use range access on index 'a' due to type or collation conversion on field 'b'
 Note	1003	/* select#1 */ select `test`.`t2`.`a` AS `a`,`test`.`t2`.`b` AS `b`,`test`.`t2`.`c` AS `c`,`test`.`t2`.`d` AS `d` from `test`.`t2` where (`test`.`t2`.`b` = 10) order by `test`.`t2`.`a`,`test`.`t2`.`c` limit 5
 # Status of "equivalent" SELECT query execution:
 Variable_name	Value

mysql-test/r/myisam_explain_json_non_select_none.result

@@ -3972,6 +3972,7 @@ EXPLAIN
   } /* query_block */
 }
 Warnings:
+Warning	1739	Cannot use range access on index 'a' due to type or collation conversion on field 'b'
 Note	1003	/* select#1 */ select `test`.`t2`.`a` AS `a`,`test`.`t2`.`b` AS `b`,`test`.`t2`.`c` AS `c`,`test`.`t2`.`d` AS `d` from `test`.`t2` where (`test`.`t2`.`b` = 10) order by `test`.`t2`.`a`,`test`.`t2`.`c` limit 5
 # Status of "equivalent" SELECT query execution:
 Variable_name	Value
@@ -4989,6 +4990,7 @@ EXPLAIN
   } /* query_block */
 }
 Warnings:
+Warning	1739	Cannot use range access on index 'a' due to type or collation conversion on field 'b'
 Note	1003	/* select#1 */ select `test`.`t2`.`a` AS `a`,`test`.`t2`.`b` AS `b`,`test`.`t2`.`c` AS `c`,`test`.`t2`.`d` AS `d` from `test`.`t2` where (`test`.`t2`.`b` = 10) order by `test`.`t2`.`a`,`test`.`t2`.`c` limit 5
 # Status of "equivalent" SELECT query execution:
 Variable_name	Value