Bug#28483283: BACKPORT BUGFIX 26929724 TO MYSQL 5.5 AND 5.6

Description: MYISAM index corruption occurs for bulk insert
or repair table which involves "repair by sorting"
algorithm.

Analysis: The index corruption happens because of the
incorrect sorting done by "my_qsort2()". This happens
for a bulk insert with more than 450001959 rows or
repair table with more than 450001959 rows. In 5.7,
"my_qsort2()" is replaced by std::sort() as part of
Bug#26929724.

Fix:- Back ported Bug#26929724 fix to ensure MyISAM repair
by sorting algorithm uses std::sort().

Author: Arun Kuruvila

storage/myisam/CMakeLists.txt

@@ -1,4 +1,4 @@
-# Copyright (c) 2006, 2012, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2006, 2018, Oracle and/or its affiliates. All rights reserved.
 # 
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -24,7 +24,7 @@ SET(MYISAM_SOURCES  ft_boolean_search.c ft_nlq_search.c ft_parser.c ft_static.c
 				mi_rfirst.c mi_rlast.c mi_rnext.c mi_rnext_same.c mi_rprev.c mi_rrnd.c
 				mi_rsame.c mi_rsamepos.c mi_scan.c mi_search.c mi_static.c mi_statrec.c
 				mi_unique.c mi_update.c mi_write.c rt_index.c rt_key.c rt_mbr.c
-				rt_split.c sort.c sp_key.c mi_extrafunc.h myisamdef.h
+				rt_split.c sort.cc sp_key.c mi_extrafunc.h myisamdef.h
 				rt_index.h mi_rkey.c)
 
 MYSQL_ADD_PLUGIN(myisam ${MYISAM_SOURCES} 

storage/myisam/myisamdef.h

@@ -1,5 +1,5 @@
 /*
-   Copyright (c) 2000, 2014, Oracle and/or its affiliates. All rights reserved.
+   Copyright (c) 2000, 2018, Oracle and/or its affiliates. All rights reserved.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -474,6 +474,10 @@ extern mysql_mutex_t THR_LOCK_myisam;
 #define mysql_rwlock_wrlock(A) {}
 #define mysql_rwlock_rdlock(A) {}
 #define mysql_rwlock_unlock(A) {}
+#endif
+
+#ifdef  __cplusplus
+extern "C" {
 #endif
 
 	/* Some extern variables */
@@ -637,6 +641,10 @@ extern ulonglong mi_safe_mul(ulonglong a,ulonglong b);
 extern int _mi_ft_update(MI_INFO *info, uint keynr, uchar *keybuf,
 			 const uchar *oldrec, const uchar *newrec, my_off_t pos);
 
+#ifdef __cplusplus
+}
+#endif
+
 struct st_sort_info;
 
 

storage/myisam/sort.c renamed to storage/myisam/sort.cc

@@ -1,4 +1,4 @@
-/* Copyright (c) 2000, 2016, Oracle and/or its affiliates. All rights reserved.
+/* Copyright (c) 2000, 2018, Oracle and/or its affiliates. All rights reserved.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -18,6 +18,9 @@
   them in sorted order through SORT_INFO functions.
 */
 
+#include <algorithm>
+#include <functional>
+#include <my_global.h>
 #include "fulltext.h"
 #if defined(__WIN__)
 #include <fcntl.h>
@@ -36,6 +39,17 @@
 #define MYF_RW  MYF(MY_NABP | MY_WME | MY_WAIT_IF_FULL)
 #define DISK_BUFFER_SIZE (IO_SIZE*16)
 
+class Key_compare :
+  public std::binary_function<const uchar*, const uchar*, bool>
+{
+public:
+  Key_compare(MI_SORT_PARAM *param) : info(param) {}
+  bool operator()(const uchar *a, const uchar *b)
+  {
+    return info->key_cmp(info, &a, &b) < 0;
+  }
+  MI_SORT_PARAM *info;
+};
 
 /*
  Pointers of functions for store and read keys from temp file
@@ -568,7 +582,7 @@ int thr_write_keys(MI_SORT_PARAM *sort_param)
         length=param->sort_buffer_length;
         while (length >= MIN_SORT_BUFFER)
         {
-          if ((mergebuf= my_malloc(length, MYF(0))))
+          if ((mergebuf= (uchar *) my_malloc(length, MYF(0))))
               break;
           length=length*3/4;
         }
@@ -655,8 +669,8 @@ static int write_keys(MI_SORT_PARAM *info, register uchar **sort_keys,
   uint sort_length=info->key_length;
   DBUG_ENTER("write_keys");
 
-  my_qsort2((uchar*) sort_keys,count,sizeof(uchar*),(qsort2_cmp) info->key_cmp,
-            info);
+  std::sort(sort_keys, sort_keys + count, Key_compare(info));
+
   if (!my_b_inited(tempfile) &&
       open_cached_file(tempfile, my_tmpdir(info->tmpdir), "ST",
                        DISK_BUFFER_SIZE, info->sort_info->param->myf_rw))
@@ -698,8 +712,8 @@ static int write_keys_varlen(MI_SORT_PARAM *info,
   int err;
   DBUG_ENTER("write_keys_varlen");
 
-  my_qsort2((uchar*) sort_keys,count,sizeof(uchar*),(qsort2_cmp) info->key_cmp,
-            info);
+  std::sort(sort_keys, sort_keys + count, Key_compare(info));
+
   if (!my_b_inited(tempfile) &&
       open_cached_file(tempfile, my_tmpdir(info->tmpdir), "ST",
                        DISK_BUFFER_SIZE, info->sort_info->param->myf_rw))
@@ -740,8 +754,8 @@ static int write_index(MI_SORT_PARAM *info, register uchar **sort_keys,
 {
   DBUG_ENTER("write_index");
 
-  my_qsort2((uchar*) sort_keys,(size_t) count,sizeof(uchar*),
-           (qsort2_cmp) info->key_cmp,info);
+  std::sort(sort_keys, sort_keys + count, Key_compare(info));
+
   while (count--)
   {
     if ((*info->key_write)(info,*sort_keys++))