Bug #23296299 : HANDLE_FATAL_SIGNAL (SIG=11) IN

                MY_TOSORT_UTF32

This patch is specific for mysql-5.5

ISSUE: When a charater that is larger than possible to
handle is passed to function my_tosort_utf32(), it results
in segmentation fault. In the scenario mentioned in the bug
AES_ENCRYPT function is used which returns large value.
This value is further passed to my_tosort_utf32 function.
This causes to cross array bound for array uni_plane,
resulting in segment violation.

SOLUTION:
This issue has got addressed in 5.6 onward releases
through worklog 2673.

The fix is similar backport of that.
Check for maximum character before accessing the array
uni_plane. In addition to function my_tosort_utf32, the
same potential problem is also present in functions
my_tolower_utf16, my_toupper_utf16, my_tosort_utf16,
my_tolower_utf32, my_toupper_utf32, my_tosort_unicode,
my_tolower_utf8mb4 and my_toupper_utf8mb4.
Fixed these functions as well.

include/m_ctype.h

@@ -1,4 +1,4 @@
-/* Copyright (c) 2000, 2013, Oracle and/or its affiliates. All rights reserved.
+/* Copyright (c) 2000, 2016, Oracle and/or its affiliates. All rights reserved.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -33,11 +33,11 @@ extern "C" {
 #define MY_CS_TO_UPPER_TABLE_SIZE	256
 #define MY_CS_SORT_ORDER_TABLE_SIZE	256
 #define MY_CS_TO_UNI_TABLE_SIZE		256
-
 #define CHARSET_DIR	"charsets/"
 
 #define my_wc_t ulong
 
+#define MY_CS_MAX_CHAR 0xFFFF
 #define MY_CS_REPLACEMENT_CHARACTER 0xFFFD
 
 /*

strings/ctype-ucs2.c

@@ -1,4 +1,4 @@
-/* Copyright (c) 2003, 2013, Oracle and/or its affiliates. All rights reserved.
+/* Copyright (c) 2003, 2016, Oracle and/or its affiliates. All rights reserved.
    
    This library is free software; you can redistribute it and/or
    modify it under the terms of the GNU Library General Public
@@ -1099,7 +1099,7 @@ static inline void
 my_tolower_utf16(MY_UNICASE_INFO **uni_plane, my_wc_t *wc)
 {
   int page= *wc >> 8;
-  if (page < 256 && uni_plane[page])
+  if (page < 256 && *wc <= MY_CS_MAX_CHAR && uni_plane[page])
     *wc= uni_plane[page][*wc & 0xFF].tolower;
 }
 
@@ -1108,7 +1108,7 @@ static inline void
 my_toupper_utf16(MY_UNICASE_INFO **uni_plane, my_wc_t *wc)
 {
   int page= *wc >> 8;
-  if (page < 256 && uni_plane[page])
+  if (page < 256 && *wc <= MY_CS_MAX_CHAR && uni_plane[page])
     *wc= uni_plane[page][*wc & 0xFF].toupper;
 }
 
@@ -1117,7 +1117,7 @@ static inline void
 my_tosort_utf16(MY_UNICASE_INFO **uni_plane, my_wc_t *wc)
 {
   int page= *wc >> 8;
-  if (page < 256)
+  if (page < 256 && *wc <= MY_CS_MAX_CHAR)
   {
     if (uni_plane[page])
       *wc= uni_plane[page][*wc & 0xFF].sort;
@@ -1728,7 +1728,7 @@ static inline void
 my_tolower_utf32(MY_UNICASE_INFO **uni_plane, my_wc_t *wc)
 {
   int page= *wc >> 8;
-  if (page < 256 && uni_plane[page])
+  if (page < 256 && *wc <= MY_CS_MAX_CHAR && uni_plane[page])
     *wc= uni_plane[page][*wc & 0xFF].tolower;
 }
 
@@ -1737,7 +1737,7 @@ static inline void
 my_toupper_utf32(MY_UNICASE_INFO **uni_plane, my_wc_t *wc)
 {
   int page= *wc >> 8;
-  if (page < 256 && uni_plane[page])
+  if (page < 256 && *wc <= MY_CS_MAX_CHAR && uni_plane[page])
     *wc= uni_plane[page][*wc & 0xFF].toupper;
 }
 
@@ -1746,7 +1746,7 @@ static inline void
 my_tosort_utf32(MY_UNICASE_INFO **uni_plane, my_wc_t *wc)
 {
   int page= *wc >> 8;
-  if (page < 256)
+  if (page < 256 && *wc <= MY_CS_MAX_CHAR)
   {
     if (uni_plane[page])
       *wc= uni_plane[page][*wc & 0xFF].sort;

strings/ctype-utf8.c

@@ -1,4 +1,4 @@
-/* Copyright (c) 2000, 2013, Oracle and/or its affiliates. All rights reserved.
+/* Copyright (c) 2000, 2016, Oracle and/or its affiliates. All rights reserved.
 
    This library is free software; you can redistribute it and/or
    modify it under the terms of the GNU Library General Public
@@ -1941,7 +1941,7 @@ static inline void
 my_tosort_unicode(MY_UNICASE_INFO **uni_plane, my_wc_t *wc)
 {
   int page= *wc >> 8;
-  if (page < 256)
+  if (page < 256 && *wc <= MY_CS_MAX_CHAR)
   {
     if (uni_plane[page])
       *wc= uni_plane[page][*wc & 0xFF].sort;
@@ -5023,7 +5023,7 @@ static inline void
 my_tolower_utf8mb4(MY_UNICASE_INFO **uni_plane, my_wc_t *wc)
 {
   int page= *wc >> 8;
-  if (page < 256 && uni_plane[page])
+  if (page < 256 && *wc <= MY_CS_MAX_CHAR && uni_plane[page])
     *wc= uni_plane[page][*wc & 0xFF].tolower;
 }
 
@@ -5032,7 +5032,7 @@ static inline void
 my_toupper_utf8mb4(MY_UNICASE_INFO **uni_plane, my_wc_t *wc)
 {
   int page= *wc >> 8;
-  if (page < 256 && uni_plane[page])
+  if (page < 256 && *wc <= MY_CS_MAX_CHAR && uni_plane[page])
     *wc= uni_plane[page][*wc & 0xFF].toupper;
 }