Drizzled Public API Documentation

mi_check.cc

00001 /* Copyright (C) 2000-2006 MySQL AB
00002 
00003    This program is free software; you can redistribute it and/or modify
00004    it under the terms of the GNU General Public License as published by
00005    the Free Software Foundation; version 2 of the License.
00006 
00007    This program is distributed in the hope that it will be useful,
00008    but WITHOUT ANY WARRANTY; without even the implied warranty of
00009    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
00010    GNU General Public License for more details.
00011 
00012    You should have received a copy of the GNU General Public License
00013    along with this program; if not, write to the Free Software
00014    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA */
00015 
00016 /* Describe, check and repair of MyISAM tables */
00017 
00018 /*
00019   About checksum calculation.
00020 
00021   There are two types of checksums. Table checksum and row checksum.
00022 
00023   Row checksum is an additional byte at the end of dynamic length
00024   records. It must be calculated if the table is configured for them.
00025   Otherwise they must not be used. The variable
00026   MYISAM_SHARE::calc_checksum determines if row checksums are used.
00027   MI_INFO::checksum is used as temporary storage during row handling.
00028   For parallel repair we must assure that only one thread can use this
00029   variable. There is no problem on the write side as this is done by one
00030   thread only. But when checking a record after read this could go
00031   wrong. But since all threads read through a common read buffer, it is
00032   sufficient if only one thread checks it.
00033 
00034   Table checksum is an eight byte value in the header of the index file.
00035   It can be calculated even if row checksums are not used. The variable
00036   MI_CHECK::glob_crc is calculated over all records.
00037   MI_SORT_PARAM::calc_checksum determines if this should be done. This
00038   variable is not part of MI_CHECK because it must be set per thread for
00039   parallel repair. The global glob_crc must be changed by one thread
00040   only. And it is sufficient to calculate the checksum once only.
00041 */
00042 
00043 #include "myisam_priv.h"
00044 #include <drizzled/internal/m_string.h>
00045 #include <stdarg.h>
00046 #ifdef HAVE_SYS_VADVISE_H
00047 #include <sys/vadvise.h>
00048 #endif
00049 #ifdef HAVE_SYS_TYPES
00050 #include <sys/types.h>
00051 #endif
00052 #ifdef HAVE_SYS_MMAN_H
00053 #include <sys/mman.h>
00054 #endif
00055 #include <drizzled/util/test.h>
00056 #include <drizzled/error.h>
00057 
00058 #include <algorithm>
00059 
00060 using namespace std;
00061 using namespace drizzled;
00062 using namespace drizzled::internal;
00063 
00064 
00065 #define my_off_t2double(A)  ((double) (my_off_t) (A))
00066 
00067 /* Functions defined in this file */
00068 
00069 static int check_k_link(MI_CHECK *param, MI_INFO *info,uint32_t nr);
00070 static int chk_index(MI_CHECK *param, MI_INFO *info,MI_KEYDEF *keyinfo,
00071          my_off_t page, unsigned char *buff, ha_rows *keys,
00072          ha_checksum *key_checksum, uint32_t level);
00073 static uint32_t isam_key_length(MI_INFO *info,MI_KEYDEF *keyinfo);
00074 static ha_checksum calc_checksum(ha_rows count);
00075 static int writekeys(MI_SORT_PARAM *sort_param);
00076 static int sort_one_index(MI_CHECK *param, MI_INFO *info,MI_KEYDEF *keyinfo,
00077         my_off_t pagepos, int new_file);
00078 int sort_key_read(MI_SORT_PARAM *sort_param,void *key);
00079 int sort_get_next_record(MI_SORT_PARAM *sort_param);
00080 int sort_key_cmp(MI_SORT_PARAM *sort_param, const void *a,const void *b);
00081 int sort_key_write(MI_SORT_PARAM *sort_param, const void *a);
00082 my_off_t get_record_for_key(MI_INFO *info,MI_KEYDEF *keyinfo,
00083                             unsigned char *key);
00084 int sort_insert_key(MI_SORT_PARAM  *sort_param,
00085                     register SORT_KEY_BLOCKS *key_block,
00086                     unsigned char *key, my_off_t prev_block);
00087 int sort_delete_record(MI_SORT_PARAM *sort_param);
00088 
00089 /*static int flush_pending_blocks(MI_CHECK *param);*/
00090 static SORT_KEY_BLOCKS  *alloc_key_blocks(MI_CHECK *param, uint32_t blocks,
00091             uint32_t buffer_length);
00092 static ha_checksum mi_byte_checksum(const unsigned char *buf, uint32_t length);
00093 static void set_data_file_type(SORT_INFO *sort_info, MYISAM_SHARE *share);
00094 
00095 void myisamchk_init(MI_CHECK *param)
00096 {
00097   memset(param, 0, sizeof(*param));
00098   param->opt_follow_links=1;
00099   param->keys_in_use= ~(uint64_t) 0;
00100   param->search_after_block=HA_OFFSET_ERROR;
00101   param->auto_increment_value= 0;
00102   param->use_buffers=USE_BUFFER_INIT;
00103   param->read_buffer_length=READ_BUFFER_INIT;
00104   param->write_buffer_length=READ_BUFFER_INIT;
00105   param->sort_buffer_length=SORT_BUFFER_INIT;
00106   param->sort_key_blocks=BUFFERS_WHEN_SORTING;
00107   param->tmpfile_createflag=O_RDWR | O_TRUNC | O_EXCL;
00108   param->myf_rw=MYF(MY_NABP | MY_WME | MY_WAIT_IF_FULL);
00109   param->start_check_pos=0;
00110   param->max_record_length= INT64_MAX;
00111   param->key_cache_block_size= KEY_CACHE_BLOCK_SIZE;
00112   param->stats_method= MI_STATS_METHOD_NULLS_NOT_EQUAL;
00113 }
00114 
00115   /* Check the status flags for the table */
00116 
00117 int chk_status(MI_CHECK *param, register MI_INFO *info)
00118 {
00119   MYISAM_SHARE *share=info->s;
00120 
00121   if (mi_is_crashed_on_repair(info))
00122     mi_check_print_warning(param,
00123          "Table is marked as crashed and last repair failed");
00124   else if (mi_is_crashed(info))
00125     mi_check_print_warning(param,
00126          "Table is marked as crashed");
00127   if (share->state.open_count != (uint) (info->s->global_changed ? 1 : 0))
00128   {
00129     /* Don't count this as a real warning, as check can correct this ! */
00130     uint32_t save=param->warning_printed;
00131     mi_check_print_warning(param,
00132          share->state.open_count==1 ?
00133          "%d client is using or hasn't closed the table properly" :
00134          "%d clients are using or haven't closed the table properly",
00135          share->state.open_count);
00136     /* If this will be fixed by the check, forget the warning */
00137     if (param->testflag & T_UPDATE_STATE)
00138       param->warning_printed=save;
00139   }
00140   return 0;
00141 }
00142 
00143   /* Check delete links */
00144 
00145 int chk_del(MI_CHECK *param, register MI_INFO *info, uint32_t test_flag)
00146 {
00147   register ha_rows i;
00148   uint32_t delete_link_length;
00149   my_off_t empty, next_link, old_link= 0;
00150   char buff[22],buff2[22];
00151 
00152   param->record_checksum=0;
00153   delete_link_length=((info->s->options & HA_OPTION_PACK_RECORD) ? 20 :
00154           info->s->rec_reflength+1);
00155 
00156   if (!(test_flag & T_SILENT))
00157     puts("- check record delete-chain");
00158 
00159   next_link=info->s->state.dellink;
00160   if (info->state->del == 0)
00161   {
00162     if (test_flag & T_VERBOSE)
00163     {
00164       puts("No recordlinks");
00165     }
00166   }
00167   else
00168   {
00169     if (test_flag & T_VERBOSE)
00170       printf("Recordlinks:    ");
00171     empty=0;
00172     for (i= info->state->del ; i > 0L && next_link != HA_OFFSET_ERROR ; i--)
00173     {
00174       if (*killed_ptr(param))
00175         return(1);
00176       if (test_flag & T_VERBOSE)
00177   printf(" %9s",llstr(next_link,buff));
00178       if (next_link >= info->state->data_file_length)
00179   goto wrong;
00180       if (my_pread(info->dfile, (unsigned char*) buff,delete_link_length,
00181        next_link,MYF(MY_NABP)))
00182       {
00183   if (test_flag & T_VERBOSE) puts("");
00184   mi_check_print_error(param,"Can't read delete-link at filepos: %s",
00185         llstr(next_link,buff));
00186   return(1);
00187       }
00188       if (*buff != '\0')
00189       {
00190   if (test_flag & T_VERBOSE) puts("");
00191   mi_check_print_error(param,"Record at pos: %s is not remove-marked",
00192         llstr(next_link,buff));
00193   goto wrong;
00194       }
00195       if (info->s->options & HA_OPTION_PACK_RECORD)
00196       {
00197   my_off_t prev_link=mi_sizekorr(buff+12);
00198   if (empty && prev_link != old_link)
00199   {
00200     if (test_flag & T_VERBOSE) puts("");
00201     mi_check_print_error(param,"Deleted block at %s doesn't point back at previous delete link",llstr(next_link,buff2));
00202     goto wrong;
00203   }
00204   old_link=next_link;
00205   next_link=mi_sizekorr(buff+4);
00206   empty+=mi_uint3korr(buff+1);
00207       }
00208       else
00209       {
00210   param->record_checksum+=(ha_checksum) next_link;
00211   next_link=_mi_rec_pos(info->s,(unsigned char*) buff+1);
00212   empty+=info->s->base.pack_reclength;
00213       }
00214     }
00215     if (test_flag & T_VERBOSE)
00216       puts("\n");
00217     if (empty != info->state->empty)
00218     {
00219       mi_check_print_warning(param,
00220            "Found %s deleted space in delete link chain. Should be %s",
00221            llstr(empty,buff2),
00222            llstr(info->state->empty,buff));
00223     }
00224     if (next_link != HA_OFFSET_ERROR)
00225     {
00226       mi_check_print_error(param,
00227          "Found more than the expected %s deleted rows in delete link chain",
00228          llstr(info->state->del, buff));
00229       goto wrong;
00230     }
00231     if (i != 0)
00232     {
00233       mi_check_print_error(param,
00234          "Found %s deleted rows in delete link chain. Should be %s",
00235          llstr(info->state->del - i, buff2),
00236          llstr(info->state->del, buff));
00237       goto wrong;
00238     }
00239   }
00240   return(0);
00241 
00242 wrong:
00243   param->testflag|=T_RETRY_WITHOUT_QUICK;
00244   if (test_flag & T_VERBOSE) puts("");
00245   mi_check_print_error(param,"record delete-link-chain corrupted");
00246   return(1);
00247 } /* chk_del */
00248 
00249 
00250   /* Check delete links in index file */
00251 
00252 static int check_k_link(MI_CHECK *param, register MI_INFO *info, uint32_t nr)
00253 {
00254   my_off_t next_link;
00255   uint32_t block_size=(nr+1)*MI_MIN_KEY_BLOCK_LENGTH;
00256   ha_rows records;
00257   char llbuff[21], llbuff2[21];
00258   unsigned char *buff;
00259 
00260   if (param->testflag & T_VERBOSE)
00261     printf("block_size %4u:", block_size);
00262 
00263   next_link=info->s->state.key_del[nr];
00264   records= (ha_rows) (info->state->key_file_length / block_size);
00265   while (next_link != HA_OFFSET_ERROR && records > 0)
00266   {
00267     if (*killed_ptr(param))
00268       return(1);
00269     if (param->testflag & T_VERBOSE)
00270       printf("%16s",llstr(next_link,llbuff));
00271 
00272     /* Key blocks must lay within the key file length entirely. */
00273     if (next_link + block_size > info->state->key_file_length)
00274     {
00275       mi_check_print_error(param, "Invalid key block position: %s  "
00276                            "key block size: %u  file_length: %s",
00277                            llstr(next_link, llbuff), block_size,
00278                            llstr(info->state->key_file_length, llbuff2));
00279       return(1);
00280     }
00281 
00282     /* Key blocks must be aligned at MI_MIN_KEY_BLOCK_LENGTH. */
00283     if (next_link & (MI_MIN_KEY_BLOCK_LENGTH - 1))
00284     {
00285       mi_check_print_error(param, "Mis-aligned key block: %s  "
00286                            "minimum key block length: %u",
00287                            llstr(next_link, llbuff), MI_MIN_KEY_BLOCK_LENGTH);
00288       return(1);
00289     }
00290 
00291     /*
00292       Read the key block with MI_MIN_KEY_BLOCK_LENGTH to find next link.
00293       If the key cache block size is smaller than block_size, we can so
00294       avoid unecessary eviction of cache block.
00295     */
00296     if (!(buff=key_cache_read(info->s->getKeyCache(),
00297                               info->s->kfile, next_link, DFLT_INIT_HITS,
00298                               (unsigned char*) info->buff, MI_MIN_KEY_BLOCK_LENGTH,
00299                               MI_MIN_KEY_BLOCK_LENGTH, 1)))
00300     {
00301       mi_check_print_error(param, "key cache read error for block: %s",
00302          llstr(next_link,llbuff));
00303       return(1);
00304     }
00305     next_link=mi_sizekorr(buff);
00306     records--;
00307     param->key_file_blocks+=block_size;
00308   }
00309   if (param->testflag & T_VERBOSE)
00310   {
00311     if (next_link != HA_OFFSET_ERROR)
00312       printf("%16s\n",llstr(next_link,llbuff));
00313     else
00314       puts("");
00315   }
00316   return (next_link != HA_OFFSET_ERROR);
00317 } /* check_k_link */
00318 
00319 
00320   /* Check sizes of files */
00321 
00322 int chk_size(MI_CHECK *param, register MI_INFO *info)
00323 {
00324   int error=0;
00325   register my_off_t skr,size;
00326   char buff[22],buff2[22];
00327 
00328   if (!(param->testflag & T_SILENT)) puts("- check file-size");
00329 
00330   /* The following is needed if called externally (not from myisamchk) */
00331   flush_key_blocks(info->s->getKeyCache(),
00332        info->s->kfile, FLUSH_FORCE_WRITE);
00333 
00334   size= lseek(info->s->kfile, 0, SEEK_END);
00335   if ((skr=(my_off_t) info->state->key_file_length) != size)
00336   {
00337     /* Don't give error if file generated by myisampack */
00338     if (skr > size && mi_is_any_key_active(info->s->state.key_map))
00339     {
00340       error=1;
00341       mi_check_print_error(param,
00342          "Size of indexfile is: %-8s        Should be: %s",
00343          llstr(size,buff), llstr(skr,buff2));
00344     }
00345     else
00346       mi_check_print_warning(param,
00347            "Size of indexfile is: %-8s      Should be: %s",
00348            llstr(size,buff), llstr(skr,buff2));
00349   }
00350   if (!(param->testflag & T_VERY_SILENT) &&
00351       ! (info->s->options & HA_OPTION_COMPRESS_RECORD) &&
00352       uint64_t2double(info->state->key_file_length) >
00353       uint64_t2double(info->s->base.margin_key_file_length)*0.9)
00354     mi_check_print_warning(param,"Keyfile is almost full, %10s of %10s used",
00355          llstr(info->state->key_file_length,buff),
00356          llstr(info->s->base.max_key_file_length-1,buff));
00357 
00358   size=lseek(info->dfile,0L,SEEK_END);
00359   skr=(my_off_t) info->state->data_file_length;
00360   if (info->s->options & HA_OPTION_COMPRESS_RECORD)
00361     skr+= MEMMAP_EXTRA_MARGIN;
00362 #ifdef USE_RELOC
00363   if (info->data_file_type == STATIC_RECORD &&
00364       skr < (my_off_t) info->s->base.reloc*info->s->base.min_pack_length)
00365     skr=(my_off_t) info->s->base.reloc*info->s->base.min_pack_length;
00366 #endif
00367   if (skr != size)
00368   {
00369     info->state->data_file_length=size; /* Skip other errors */
00370     if (skr > size && skr != size + MEMMAP_EXTRA_MARGIN)
00371     {
00372       error=1;
00373       mi_check_print_error(param,"Size of datafile is: %-9s         Should be: %s",
00374         llstr(size,buff), llstr(skr,buff2));
00375       param->testflag|=T_RETRY_WITHOUT_QUICK;
00376     }
00377     else
00378     {
00379       mi_check_print_warning(param,
00380            "Size of datafile is: %-9s       Should be: %s",
00381            llstr(size,buff), llstr(skr,buff2));
00382     }
00383   }
00384   if (!(param->testflag & T_VERY_SILENT) &&
00385       !(info->s->options & HA_OPTION_COMPRESS_RECORD) &&
00386       uint64_t2double(info->state->data_file_length) >
00387       (uint64_t2double(info->s->base.max_data_file_length)*0.9))
00388     mi_check_print_warning(param, "Datafile is almost full, %10s of %10s used",
00389          llstr(info->state->data_file_length,buff),
00390          llstr(info->s->base.max_data_file_length-1,buff2));
00391   return(error);
00392 } /* chk_size */
00393 
00394 
00395   /* Check keys */
00396 
00397 int chk_key(MI_CHECK *param, register MI_INFO *info)
00398 {
00399   uint32_t key,found_keys=0,full_text_keys=0,result=0;
00400   ha_rows keys;
00401   ha_checksum old_record_checksum,init_checksum;
00402   my_off_t all_keydata,all_totaldata,key_totlength,length;
00403   ulong   *rec_per_key_part;
00404   MYISAM_SHARE *share=info->s;
00405   MI_KEYDEF *keyinfo;
00406   char buff[22],buff2[22];
00407 
00408   if (!(param->testflag & T_SILENT))
00409     puts("- check key delete-chain");
00410 
00411   param->key_file_blocks=info->s->base.keystart;
00412   for (key=0 ; key < info->s->state.header.max_block_size_index ; key++)
00413     if (check_k_link(param,info,key))
00414     {
00415       if (param->testflag & T_VERBOSE) puts("");
00416       mi_check_print_error(param,"key delete-link-chain corrupted");
00417       return(-1);
00418     }
00419 
00420   if (!(param->testflag & T_SILENT)) puts("- check index reference");
00421 
00422   all_keydata=all_totaldata=key_totlength=0;
00423   old_record_checksum=0;
00424   init_checksum=param->record_checksum;
00425   if (!(share->options &
00426   (HA_OPTION_PACK_RECORD | HA_OPTION_COMPRESS_RECORD)))
00427     old_record_checksum=calc_checksum(info->state->records+info->state->del-1)*
00428       share->base.pack_reclength;
00429   rec_per_key_part= param->rec_per_key_part;
00430   for (key= 0,keyinfo= &share->keyinfo[0]; key < share->base.keys ;
00431        rec_per_key_part+=keyinfo->keysegs, key++, keyinfo++)
00432   {
00433     param->key_crc[key]=0;
00434     if (! mi_is_key_active(share->state.key_map, key))
00435     {
00436       /* Remember old statistics for key */
00437       assert(rec_per_key_part >= param->rec_per_key_part);
00438       memcpy(rec_per_key_part,
00439        (share->state.rec_per_key_part +
00440               (rec_per_key_part - param->rec_per_key_part)),
00441        keyinfo->keysegs*sizeof(*rec_per_key_part));
00442       continue;
00443     }
00444     found_keys++;
00445 
00446     param->record_checksum=init_checksum;
00447 
00448     memset(&param->unique_count, 0, sizeof(param->unique_count));
00449     memset(&param->notnull_count, 0, sizeof(param->notnull_count));
00450 
00451     if ((!(param->testflag & T_SILENT)))
00452       printf ("- check data record references index: %d\n",key+1);
00453     if (share->state.key_root[key] == HA_OFFSET_ERROR && (info->state->records == 0))
00454       goto do_stat;
00455     if (!_mi_fetch_keypage(info,keyinfo,share->state.key_root[key],
00456                            DFLT_INIT_HITS,info->buff,0))
00457     {
00458       mi_check_print_error(param,"Can't read indexpage from filepos: %s",
00459       llstr(share->state.key_root[key],buff));
00460       if (!(param->testflag & T_INFO))
00461   return(-1);
00462       result= UINT32_MAX;
00463       continue;
00464     }
00465     param->key_file_blocks+=keyinfo->block_length;
00466     keys=0;
00467     param->keydata=param->totaldata=0;
00468     param->key_blocks=0;
00469     param->max_level=0;
00470     if (chk_index(param,info,keyinfo,share->state.key_root[key],info->buff,
00471       &keys, param->key_crc+key,1))
00472       return(-1);
00473     if(!(0))
00474     {
00475       if (keys != info->state->records)
00476       {
00477   mi_check_print_error(param,"Found %s keys of %s",llstr(keys,buff),
00478         llstr(info->state->records,buff2));
00479   if (!(param->testflag & T_INFO))
00480   return(-1);
00481   result= UINT32_MAX;
00482   continue;
00483       }
00484       if (found_keys - full_text_keys == 1 &&
00485     ((share->options &
00486       (HA_OPTION_PACK_RECORD | HA_OPTION_COMPRESS_RECORD)) ||
00487      (param->testflag & T_DONT_CHECK_CHECKSUM)))
00488   old_record_checksum=param->record_checksum;
00489       else if (old_record_checksum != param->record_checksum)
00490       {
00491   if (key)
00492     mi_check_print_error(param,"Key %u doesn't point at same records that key 1",
00493           key+1);
00494   else
00495     mi_check_print_error(param,"Key 1 doesn't point at all records");
00496   if (!(param->testflag & T_INFO))
00497     return(-1);
00498   result= UINT32_MAX;
00499   continue;
00500       }
00501     }
00502     if ((uint) share->base.auto_key -1 == key)
00503     {
00504       /* Check that auto_increment key is bigger than max key value */
00505       uint64_t auto_increment;
00506       info->lastinx=key;
00507       _mi_read_key_record(info, 0L, info->rec_buff);
00508       auto_increment= retrieve_auto_increment(info, info->rec_buff);
00509       if (auto_increment > info->s->state.auto_increment)
00510       {
00511         mi_check_print_warning(param, "Auto-increment value: %s is smaller "
00512                                "than max used value: %s",
00513                                llstr(info->s->state.auto_increment,buff2),
00514                                llstr(auto_increment, buff));
00515       }
00516       if (param->testflag & T_AUTO_INC)
00517       {
00518         set_if_bigger(info->s->state.auto_increment,
00519                       auto_increment);
00520         set_if_bigger(info->s->state.auto_increment,
00521                       param->auto_increment_value);
00522       }
00523 
00524       /* Check that there isn't a row with auto_increment = 0 in the table */
00525       mi_extra(info,HA_EXTRA_KEYREAD,0);
00526       memset(info->lastkey, 0, keyinfo->seg->length);
00527       if (!mi_rkey(info, info->rec_buff, key, (const unsigned char*) info->lastkey,
00528        (key_part_map)1, HA_READ_KEY_EXACT))
00529       {
00530   /* Don't count this as a real warning, as myisamchk can't correct it */
00531   uint32_t save=param->warning_printed;
00532         mi_check_print_warning(param, "Found row where the auto_increment "
00533                                "column has the value 0");
00534   param->warning_printed=save;
00535       }
00536       mi_extra(info,HA_EXTRA_NO_KEYREAD,0);
00537     }
00538 
00539     length=(my_off_t) isam_key_length(info,keyinfo)*keys + param->key_blocks*2;
00540     if (param->testflag & T_INFO && param->totaldata != 0L && keys != 0L)
00541       printf("Key: %2d:  Keyblocks used: %3d%%  Packed: %4d%%  Max levels: %2d\n",
00542        key+1,
00543        (int) (my_off_t2double(param->keydata)*100.0/my_off_t2double(param->totaldata)),
00544        (int) ((my_off_t2double(length) - my_off_t2double(param->keydata))*100.0/
00545         my_off_t2double(length)),
00546        param->max_level);
00547     all_keydata+=param->keydata; all_totaldata+=param->totaldata; key_totlength+=length;
00548 
00549 do_stat:
00550     if (param->testflag & T_STATISTICS)
00551       update_key_parts(keyinfo, rec_per_key_part, param->unique_count,
00552                        param->stats_method == MI_STATS_METHOD_IGNORE_NULLS?
00553                        param->notnull_count: NULL,
00554                        (uint64_t)info->state->records);
00555   }
00556   if (param->testflag & T_INFO)
00557   {
00558     if (all_totaldata != 0L && found_keys > 0)
00559       printf("Total:    Keyblocks used: %3d%%  Packed: %4d%%\n\n",
00560        (int) (my_off_t2double(all_keydata)*100.0/
00561         my_off_t2double(all_totaldata)),
00562        (int) ((my_off_t2double(key_totlength) -
00563          my_off_t2double(all_keydata))*100.0/
00564          my_off_t2double(key_totlength)));
00565     else if (all_totaldata != 0L && mi_is_any_key_active(share->state.key_map))
00566       puts("");
00567   }
00568   if (param->key_file_blocks != info->state->key_file_length &&
00569       param->keys_in_use != ~(uint64_t) 0)
00570     mi_check_print_warning(param, "Some data are unreferenced in keyfile");
00571   if (found_keys != full_text_keys)
00572     param->record_checksum=old_record_checksum-init_checksum; /* Remove delete links */
00573   else
00574     param->record_checksum=0;
00575   return(result);
00576 } /* chk_key */
00577 
00578 
00579 static int chk_index_down(MI_CHECK *param, MI_INFO *info, MI_KEYDEF *keyinfo,
00580                      my_off_t page, unsigned char *buff, ha_rows *keys,
00581                      ha_checksum *key_checksum, uint32_t level)
00582 {
00583   char llbuff[22],llbuff2[22];
00584 
00585   /* Key blocks must lay within the key file length entirely. */
00586   if (page + keyinfo->block_length > info->state->key_file_length)
00587   {
00588     /* Give it a chance to fit in the real file size. */
00589     my_off_t max_length= lseek(info->s->kfile, 0, SEEK_END);
00590     mi_check_print_error(param, "Invalid key block position: %s  "
00591                          "key block size: %u  file_length: %s",
00592                          llstr(page, llbuff), keyinfo->block_length,
00593                          llstr(info->state->key_file_length, llbuff2));
00594     if (page + keyinfo->block_length > max_length)
00595       goto err;
00596     /* Fix the remebered key file length. */
00597     info->state->key_file_length= (max_length &
00598                                    ~ (my_off_t) (keyinfo->block_length - 1));
00599   }
00600 
00601   /* Key blocks must be aligned at MI_MIN_KEY_BLOCK_LENGTH. */
00602   if (page & (MI_MIN_KEY_BLOCK_LENGTH - 1))
00603   {
00604     mi_check_print_error(param, "Mis-aligned key block: %s  "
00605                          "minimum key block length: %u",
00606                          llstr(page, llbuff), MI_MIN_KEY_BLOCK_LENGTH);
00607     goto err;
00608   }
00609 
00610   if (!_mi_fetch_keypage(info,keyinfo,page, DFLT_INIT_HITS,buff,0))
00611   {
00612     mi_check_print_error(param,"Can't read key from filepos: %s",
00613         llstr(page,llbuff));
00614     goto err;
00615   }
00616   param->key_file_blocks+=keyinfo->block_length;
00617   if (chk_index(param,info,keyinfo,page,buff,keys,key_checksum,level))
00618     goto err;
00619 
00620   return(0);
00621 
00622 err:
00623   return(1);
00624 }
00625 
00626 
00627 /*
00628   "Ignore NULLs" statistics collection method: process first index tuple.
00629 
00630   SYNOPSIS
00631     mi_collect_stats_nonulls_first()
00632       keyseg   IN     Array of key part descriptions
00633       notnull  INOUT  Array, notnull[i] = (number of {keypart1...keypart_i}
00634                                            tuples that don't contain NULLs)
00635       key      IN     Key values tuple
00636 
00637   DESCRIPTION
00638     Process the first index tuple - find out which prefix tuples don't
00639     contain NULLs, and update the array of notnull counters accordingly.
00640 */
00641 
00642 static
00643 void mi_collect_stats_nonulls_first(HA_KEYSEG *keyseg, uint64_t *notnull,
00644                                     unsigned char *key)
00645 {
00646   uint32_t first_null, kp;
00647   first_null= ha_find_null(keyseg, key) - keyseg;
00648   /*
00649     All prefix tuples that don't include keypart_{first_null} are not-null
00650     tuples (and all others aren't), increment counters for them.
00651   */
00652   for (kp= 0; kp < first_null; kp++)
00653     notnull[kp]++;
00654 }
00655 
00656 
00657 /*
00658   "Ignore NULLs" statistics collection method: process next index tuple.
00659 
00660   SYNOPSIS
00661     mi_collect_stats_nonulls_next()
00662       keyseg   IN     Array of key part descriptions
00663       notnull  INOUT  Array, notnull[i] = (number of {keypart1...keypart_i}
00664                                            tuples that don't contain NULLs)
00665       prev_key IN     Previous key values tuple
00666       last_key IN     Next key values tuple
00667 
00668   DESCRIPTION
00669     Process the next index tuple:
00670     1. Find out which prefix tuples of last_key don't contain NULLs, and
00671        update the array of notnull counters accordingly.
00672     2. Find the first keypart number where the prev_key and last_key tuples
00673        are different(A), or last_key has NULL value(B), and return it, so the
00674        caller can count number of unique tuples for each key prefix. We don't
00675        need (B) to be counted, and that is compensated back in
00676        update_key_parts().
00677 
00678   RETURN
00679     1 + number of first keypart where values differ or last_key tuple has NULL
00680 */
00681 
00682 static
00683 int mi_collect_stats_nonulls_next(HA_KEYSEG *keyseg, uint64_t *notnull,
00684                                   unsigned char *prev_key, unsigned char *last_key)
00685 {
00686   uint32_t diffs[2];
00687   uint32_t first_null_seg, kp;
00688   HA_KEYSEG *seg;
00689 
00690   /*
00691      Find the first keypart where values are different or either of them is
00692      NULL. We get results in diffs array:
00693      diffs[0]= 1 + number of first different keypart
00694      diffs[1]=offset: (last_key + diffs[1]) points to first value in
00695                       last_key that is NULL or different from corresponding
00696                       value in prev_key.
00697   */
00698   ha_key_cmp(keyseg, prev_key, last_key, USE_WHOLE_KEY,
00699              SEARCH_FIND | SEARCH_NULL_ARE_NOT_EQUAL, diffs);
00700   seg= keyseg + diffs[0] - 1;
00701 
00702   /* Find first NULL in last_key */
00703   first_null_seg= ha_find_null(seg, last_key + diffs[1]) - keyseg;
00704   for (kp= 0; kp < first_null_seg; kp++)
00705     notnull[kp]++;
00706 
00707   /*
00708     Return 1+ number of first key part where values differ. Don't care if
00709     these were NULLs and not .... We compensate for that in
00710     update_key_parts.
00711   */
00712   return diffs[0];
00713 }
00714 
00715 
00716   /* Check if index is ok */
00717 
00718 static int chk_index(MI_CHECK *param, MI_INFO *info, MI_KEYDEF *keyinfo,
00719          my_off_t page, unsigned char *buff, ha_rows *keys,
00720          ha_checksum *key_checksum, uint32_t level)
00721 {
00722   int flag;
00723   uint32_t used_length,comp_flag,nod_flag,key_length=0;
00724   unsigned char key[HA_MAX_POSSIBLE_KEY_BUFF],*temp_buff,*keypos,*old_keypos,*endpos;
00725   my_off_t next_page,record;
00726   char llbuff[22];
00727   uint32_t diff_pos[2];
00728 
00729   if (!(temp_buff=(unsigned char*) malloc(keyinfo->block_length)))
00730   {
00731     mi_check_print_error(param,"Not enough memory for keyblock");
00732     return(-1);
00733   }
00734 
00735   if (keyinfo->flag & HA_NOSAME)
00736     comp_flag=SEARCH_FIND | SEARCH_UPDATE;  /* Not real duplicates */
00737   else
00738     comp_flag=SEARCH_SAME;      /* Keys in positionorder */
00739   nod_flag=mi_test_if_nod(buff);
00740   used_length=mi_getint(buff);
00741   keypos=buff+2+nod_flag;
00742   endpos=buff+used_length;
00743 
00744   param->keydata+=used_length; param->totaldata+=keyinfo->block_length; /* INFO */
00745   param->key_blocks++;
00746   if (level > param->max_level)
00747     param->max_level=level;
00748 
00749   if (used_length > keyinfo->block_length)
00750   {
00751     mi_check_print_error(param,"Wrong pageinfo at page: %s",
00752        llstr(page,llbuff));
00753     goto err;
00754   }
00755   for ( ;; )
00756   {
00757     if (*killed_ptr(param))
00758       goto err;
00759     memcpy(info->lastkey,key,key_length);
00760     info->lastkey_length=key_length;
00761     if (nod_flag)
00762     {
00763       next_page=_mi_kpos(nod_flag,keypos);
00764       if (chk_index_down(param,info,keyinfo,next_page,
00765                          temp_buff,keys,key_checksum,level+1))
00766   goto err;
00767     }
00768     old_keypos=keypos;
00769     if (keypos >= endpos ||
00770   (key_length=(*keyinfo->get_key)(keyinfo,nod_flag,&keypos,key)) == 0)
00771       break;
00772     assert(key_length <= sizeof(key));
00773     if (keypos > endpos)
00774     {
00775       mi_check_print_error(param,"Wrong key block length at page: %s",llstr(page,llbuff));
00776       goto err;
00777     }
00778     if ((*keys)++ &&
00779   (flag=ha_key_cmp(keyinfo->seg,info->lastkey,key,key_length,
00780        comp_flag, diff_pos)) >=0)
00781     {
00782       if (comp_flag & SEARCH_FIND && flag == 0)
00783   mi_check_print_error(param,"Found duplicated key at page %s",llstr(page,llbuff));
00784       else
00785   mi_check_print_error(param,"Key in wrong position at page %s",llstr(page,llbuff));
00786       goto err;
00787     }
00788     if (param->testflag & T_STATISTICS)
00789     {
00790       if (*keys != 1L)        /* not first_key */
00791       {
00792         if (param->stats_method == MI_STATS_METHOD_NULLS_NOT_EQUAL)
00793           ha_key_cmp(keyinfo->seg,info->lastkey,key,USE_WHOLE_KEY,
00794                      SEARCH_FIND | SEARCH_NULL_ARE_NOT_EQUAL,
00795                      diff_pos);
00796         else if (param->stats_method == MI_STATS_METHOD_IGNORE_NULLS)
00797         {
00798           diff_pos[0]= mi_collect_stats_nonulls_next(keyinfo->seg,
00799                                                   param->notnull_count,
00800                                                   info->lastkey, key);
00801         }
00802   param->unique_count[diff_pos[0]-1]++;
00803       }
00804       else
00805       {
00806         if (param->stats_method == MI_STATS_METHOD_IGNORE_NULLS)
00807           mi_collect_stats_nonulls_first(keyinfo->seg, param->notnull_count,
00808                                          key);
00809       }
00810     }
00811     (*key_checksum)+= mi_byte_checksum((unsigned char*) key,
00812                key_length- info->s->rec_reflength);
00813     record= _mi_dpos(info,0,key+key_length);
00814     if (record >= info->state->data_file_length)
00815     {
00816       mi_check_print_error(param,"Found key at page %s that points to record outside datafile",llstr(page,llbuff));
00817       goto err;
00818     }
00819     param->record_checksum+=(ha_checksum) record;
00820   }
00821   if (keypos != endpos)
00822   {
00823     mi_check_print_error(param,"Keyblock size at page %s is not correct.  Block length: %d  key length: %d",
00824                 llstr(page,llbuff), used_length, (keypos - buff));
00825     goto err;
00826   }
00827   free(temp_buff);
00828   return(0);
00829  err:
00830   free(temp_buff);
00831   return(1);
00832 } /* chk_index */
00833 
00834 
00835   /* Calculate a checksum of 1+2+3+4...N = N*(N+1)/2 without overflow */
00836 
00837 static ha_checksum calc_checksum(ha_rows count)
00838 {
00839   uint64_t sum,a,b;
00840 
00841   sum=0;
00842   a=count; b=count+1;
00843   if (a & 1)
00844     b>>=1;
00845   else
00846     a>>=1;
00847   while (b)
00848   {
00849     if (b & 1)
00850       sum+=a;
00851     a<<=1; b>>=1;
00852   }
00853   return((ha_checksum) sum);
00854 } /* calc_checksum */
00855 
00856 
00857   /* Calc length of key in normal isam */
00858 
00859 static uint32_t isam_key_length(MI_INFO *info, register MI_KEYDEF *keyinfo)
00860 {
00861   uint32_t length;
00862   HA_KEYSEG *keyseg;
00863 
00864   length= info->s->rec_reflength;
00865   for (keyseg=keyinfo->seg ; keyseg->type ; keyseg++)
00866     length+= keyseg->length;
00867 
00868   return(length);
00869 } /* key_length */
00870 
00871 
00872   /* Check that record-link is ok */
00873 
00874 int chk_data_link(MI_CHECK *param, MI_INFO *info,int extend)
00875 {
00876   int error,got_error,flag;
00877   uint  key, left_length= 0, b_type,field;
00878   ha_rows records, del_blocks;
00879   my_off_t used, empty, pos, splits, start_recpos= 0,
00880            del_length, link_used, start_block;
00881   unsigned char *record= NULL, *to= NULL;
00882   char llbuff[22],llbuff2[22],llbuff3[22];
00883   ha_checksum intern_record_checksum;
00884   ha_checksum key_checksum[HA_MAX_POSSIBLE_KEY];
00885   bool static_row_size;
00886   MI_KEYDEF *keyinfo;
00887   MI_BLOCK_INFO block_info;
00888 
00889   if (!(param->testflag & T_SILENT))
00890   {
00891     if (extend)
00892       puts("- check records and index references");
00893     else
00894       puts("- check record links");
00895   }
00896 
00897   if (!mi_alloc_rec_buff(info, SIZE_MAX, &record))
00898   {
00899     mi_check_print_error(param,"Not enough memory for record");
00900     return(-1);
00901   }
00902   records=del_blocks=0;
00903   used=link_used=splits=del_length=0;
00904   intern_record_checksum=param->glob_crc=0;
00905   got_error=error=0;
00906   empty=info->s->pack.header_length;
00907 
00908   /* Check how to calculate checksum of rows */
00909   static_row_size=1;
00910   if (info->s->data_file_type == COMPRESSED_RECORD)
00911   {
00912     for (field=0 ; field < info->s->base.fields ; field++)
00913     {
00914       if (info->s->rec[field].base_type == FIELD_BLOB ||
00915     info->s->rec[field].base_type == FIELD_VARCHAR)
00916       {
00917   static_row_size=0;
00918   break;
00919       }
00920     }
00921   }
00922 
00923   pos=my_b_tell(&param->read_cache);
00924   memset(key_checksum, 0, info->s->base.keys * sizeof(key_checksum[0]));
00925   while (pos < info->state->data_file_length)
00926   {
00927     if (*killed_ptr(param))
00928       goto err2;
00929     switch (info->s->data_file_type) {
00930     case STATIC_RECORD:
00931       if (my_b_read(&param->read_cache,(unsigned char*) record,
00932         info->s->base.pack_reclength))
00933   goto err;
00934       start_recpos=pos;
00935       pos+=info->s->base.pack_reclength;
00936       splits++;
00937       if (*record == '\0')
00938       {
00939   del_blocks++;
00940   del_length+=info->s->base.pack_reclength;
00941   continue;         /* Record removed */
00942       }
00943       param->glob_crc+= mi_static_checksum(info,record);
00944       used+=info->s->base.pack_reclength;
00945       break;
00946     case DYNAMIC_RECORD:
00947       flag=block_info.second_read=0;
00948       block_info.next_filepos=pos;
00949       do
00950       {
00951   if (_mi_read_cache(&param->read_cache,(unsigned char*) block_info.header,
00952          (start_block=block_info.next_filepos),
00953          sizeof(block_info.header),
00954          (flag ? 0 : READING_NEXT) | READING_HEADER))
00955     goto err;
00956   if (start_block & (MI_DYN_ALIGN_SIZE-1))
00957   {
00958     mi_check_print_error(param,"Wrong aligned block at %s",
00959              llstr(start_block,llbuff));
00960     goto err2;
00961   }
00962   b_type=_mi_get_block_info(&block_info,-1,start_block);
00963   if (b_type & (BLOCK_DELETED | BLOCK_ERROR | BLOCK_SYNC_ERROR |
00964           BLOCK_FATAL_ERROR))
00965   {
00966     if (b_type & BLOCK_SYNC_ERROR)
00967     {
00968       if (flag)
00969       {
00970         mi_check_print_error(param,"Unexpected byte: %d at link: %s",
00971         (int) block_info.header[0],
00972         llstr(start_block,llbuff));
00973         goto err2;
00974       }
00975       pos=block_info.filepos+block_info.block_len;
00976       goto next;
00977     }
00978     if (b_type & BLOCK_DELETED)
00979     {
00980       if (block_info.block_len < info->s->base.min_block_length)
00981       {
00982         mi_check_print_error(param,
00983            "Deleted block with impossible length %lu at %s",
00984            block_info.block_len,llstr(pos,llbuff));
00985         goto err2;
00986       }
00987       if ((block_info.next_filepos != HA_OFFSET_ERROR &&
00988      block_info.next_filepos >= info->state->data_file_length) ||
00989     (block_info.prev_filepos != HA_OFFSET_ERROR &&
00990      block_info.prev_filepos >= info->state->data_file_length))
00991       {
00992         mi_check_print_error(param,"Delete link points outside datafile at %s",
00993         llstr(pos,llbuff));
00994         goto err2;
00995       }
00996       del_blocks++;
00997       del_length+=block_info.block_len;
00998       pos=block_info.filepos+block_info.block_len;
00999       splits++;
01000       goto next;
01001     }
01002     mi_check_print_error(param,"Wrong bytesec: %d-%d-%d at linkstart: %s",
01003              block_info.header[0],block_info.header[1],
01004              block_info.header[2],
01005              llstr(start_block,llbuff));
01006     goto err2;
01007   }
01008   if (info->state->data_file_length < block_info.filepos+
01009       block_info.block_len)
01010   {
01011     mi_check_print_error(param,
01012              "Recordlink that points outside datafile at %s",
01013              llstr(pos,llbuff));
01014     got_error=1;
01015     break;
01016   }
01017   splits++;
01018   if (!flag++)        /* First block */
01019   {
01020     start_recpos=pos;
01021     pos=block_info.filepos+block_info.block_len;
01022     if (block_info.rec_len > (uint) info->s->base.max_pack_length)
01023     {
01024       mi_check_print_error(param,"Found too long record (%lu) at %s",
01025          (ulong) block_info.rec_len,
01026          llstr(start_recpos,llbuff));
01027       got_error=1;
01028       break;
01029     }
01030     if (info->s->base.blobs)
01031     {
01032       if (!(to= mi_alloc_rec_buff(info, block_info.rec_len,
01033           &info->rec_buff)))
01034       {
01035         mi_check_print_error(param,
01036            "Not enough memory (%lu) for blob at %s",
01037            (ulong) block_info.rec_len,
01038            llstr(start_recpos,llbuff));
01039         got_error=1;
01040         break;
01041       }
01042     }
01043     else
01044       to= info->rec_buff;
01045     left_length=block_info.rec_len;
01046   }
01047   if (left_length < block_info.data_len)
01048   {
01049     mi_check_print_error(param,"Found too long record (%lu) at %s",
01050              (ulong) block_info.data_len,
01051              llstr(start_recpos,llbuff));
01052     got_error=1;
01053     break;
01054   }
01055   if (_mi_read_cache(&param->read_cache,(unsigned char*) to,block_info.filepos,
01056          (uint) block_info.data_len,
01057          flag == 1 ? READING_NEXT : 0))
01058     goto err;
01059   to+=block_info.data_len;
01060   link_used+= block_info.filepos-start_block;
01061   used+= block_info.filepos - start_block + block_info.data_len;
01062   empty+=block_info.block_len-block_info.data_len;
01063   left_length-=block_info.data_len;
01064   if (left_length)
01065   {
01066     if (b_type & BLOCK_LAST)
01067     {
01068       mi_check_print_error(param,
01069          "Wrong record length %s of %s at %s",
01070          llstr(block_info.rec_len-left_length,llbuff),
01071          llstr(block_info.rec_len, llbuff2),
01072          llstr(start_recpos,llbuff3));
01073       got_error=1;
01074       break;
01075     }
01076     if (info->state->data_file_length < block_info.next_filepos)
01077     {
01078       mi_check_print_error(param,
01079          "Found next-recordlink that points outside datafile at %s",
01080          llstr(block_info.filepos,llbuff));
01081       got_error=1;
01082       break;
01083     }
01084   }
01085       } while (left_length);
01086       if (! got_error)
01087       {
01088   if (_mi_rec_unpack(info,record,info->rec_buff,block_info.rec_len) ==
01089       MY_FILE_ERROR)
01090   {
01091     mi_check_print_error(param,"Found wrong record at %s",
01092              llstr(start_recpos,llbuff));
01093     got_error=1;
01094   }
01095   else
01096   {
01097     info->checksum=mi_checksum(info,record);
01098     if (param->testflag & (T_EXTEND | T_MEDIUM | T_VERBOSE))
01099     {
01100       if (_mi_rec_check(info,record, info->rec_buff,block_info.rec_len,
01101                               test(info->s->calc_checksum)))
01102       {
01103         mi_check_print_error(param,"Found wrong packed record at %s",
01104         llstr(start_recpos,llbuff));
01105         got_error=1;
01106       }
01107     }
01108     if (!got_error)
01109       param->glob_crc+= info->checksum;
01110   }
01111       }
01112       else if (!flag)
01113   pos=block_info.filepos+block_info.block_len;
01114       break;
01115     case COMPRESSED_RECORD:
01116     case BLOCK_RECORD:
01117       assert(0);                                /* Impossible */
01118     } /* switch */
01119     if (! got_error)
01120     {
01121       intern_record_checksum+=(ha_checksum) start_recpos;
01122       records++;
01123       if (param->testflag & T_WRITE_LOOP && records % WRITE_COUNT == 0)
01124       {
01125   printf("%s\r", llstr(records,llbuff)); fflush(stdout);
01126       }
01127 
01128       /* Check if keys match the record */
01129 
01130       for (key=0,keyinfo= info->s->keyinfo; key < info->s->base.keys;
01131      key++,keyinfo++)
01132       {
01133         if (mi_is_key_active(info->s->state.key_map, key))
01134   {
01135     {
01136       uint32_t key_length=_mi_make_key(info,key,info->lastkey,record,
01137            start_recpos);
01138       if (extend)
01139       {
01140         /* We don't need to lock the key tree here as we don't allow
01141      concurrent threads when running myisamchk
01142         */
01143               int search_result=
01144                 _mi_search(info,keyinfo,info->lastkey,key_length,
01145                            SEARCH_SAME, info->s->state.key_root[key]);
01146               if (search_result)
01147               {
01148                 mi_check_print_error(param,"Record at: %10s  "
01149                                      "Can't find key for index: %2d",
01150                                      llstr(start_recpos,llbuff),key+1);
01151                 if (error++ > MAXERR || !(param->testflag & T_VERBOSE))
01152                   goto err2;
01153               }
01154       }
01155       else
01156         key_checksum[key]+=mi_byte_checksum((unsigned char*) info->lastkey,
01157               key_length);
01158     }
01159   }
01160       }
01161     }
01162     else
01163     {
01164       got_error=0;
01165       if (error++ > MAXERR || !(param->testflag & T_VERBOSE))
01166   goto err2;
01167     }
01168   next:;        /* Next record */
01169   }
01170   if (param->testflag & T_WRITE_LOOP)
01171   {
01172     fputs("          \r",stdout); fflush(stdout);
01173   }
01174   if (records != info->state->records)
01175   {
01176     mi_check_print_error(param,"Record-count is not ok; is %-10s   Should be: %s",
01177     llstr(records,llbuff), llstr(info->state->records,llbuff2));
01178     error=1;
01179   }
01180   else if (param->record_checksum &&
01181      param->record_checksum != intern_record_checksum)
01182   {
01183     mi_check_print_error(param,
01184        "Keypointers and record positions doesn't match");
01185     error=1;
01186   }
01187   else if (param->glob_crc != info->state->checksum &&
01188      (info->s->options &
01189       (HA_OPTION_COMPRESS_RECORD)))
01190   {
01191     mi_check_print_warning(param,
01192          "Record checksum is not the same as checksum stored in the index file\n");
01193     error=1;
01194   }
01195   else if (!extend)
01196   {
01197     for (key=0 ; key < info->s->base.keys;  key++)
01198     {
01199       if (key_checksum[key] != param->key_crc[key])
01200       {
01201   mi_check_print_error(param,"Checksum for key: %2d doesn't match checksum for records",
01202         key+1);
01203   error=1;
01204       }
01205     }
01206   }
01207 
01208   if (del_length != info->state->empty)
01209   {
01210     mi_check_print_warning(param,
01211          "Found %s deleted space.   Should be %s",
01212          llstr(del_length,llbuff2),
01213          llstr(info->state->empty,llbuff));
01214   }
01215   if (used+empty+del_length != info->state->data_file_length)
01216   {
01217     mi_check_print_warning(param,
01218          "Found %s record-data and %s unused data and %s deleted-data",
01219          llstr(used,llbuff),llstr(empty,llbuff2),
01220          llstr(del_length,llbuff3));
01221     mi_check_print_warning(param,
01222          "Total %s, Should be: %s",
01223          llstr((used+empty+del_length),llbuff),
01224          llstr(info->state->data_file_length,llbuff2));
01225   }
01226   if (del_blocks != info->state->del)
01227   {
01228     mi_check_print_warning(param,
01229          "Found %10s deleted blocks       Should be: %s",
01230          llstr(del_blocks,llbuff),
01231          llstr(info->state->del,llbuff2));
01232   }
01233   if (splits != info->s->state.split)
01234   {
01235     mi_check_print_warning(param,
01236          "Found %10s parts                Should be: %s parts",
01237          llstr(splits,llbuff),
01238          llstr(info->s->state.split,llbuff2));
01239   }
01240   if (param->testflag & T_INFO)
01241   {
01242     if (param->warning_printed || param->error_printed)
01243       puts("");
01244     if (used != 0 && ! param->error_printed)
01245     {
01246       printf("Records:%18s    M.recordlength:%9lu   Packed:%14.0f%%\n",
01247        llstr(records,llbuff), (long)((used-link_used)/records),
01248        (info->s->base.blobs ? 0.0 :
01249         (uint64_t2double((uint64_t) info->s->base.reclength*records)-
01250          my_off_t2double(used))/
01251         uint64_t2double((uint64_t) info->s->base.reclength*records)*100.0));
01252       printf("Recordspace used:%9.0f%%   Empty space:%12d%%  Blocks/Record: %6.2f\n",
01253        (uint64_t2double(used-link_used)/uint64_t2double(used-link_used+empty)*100.0),
01254        (!records ? 100 : (int) (uint64_t2double(del_length+empty)/
01255               my_off_t2double(used)*100.0)),
01256        uint64_t2double(splits - del_blocks) / records);
01257     }
01258     printf("Record blocks:%12s    Delete blocks:%10s\n",
01259      llstr(splits-del_blocks,llbuff),llstr(del_blocks,llbuff2));
01260     printf("Record data:  %12s    Deleted data: %10s\n",
01261      llstr(used-link_used,llbuff),llstr(del_length,llbuff2));
01262     printf("Lost space:   %12s    Linkdata:     %10s\n",
01263      llstr(empty,llbuff),llstr(link_used,llbuff2));
01264   }
01265   free(mi_get_rec_buff_ptr(info, record));
01266   return (error);
01267  err:
01268   mi_check_print_error(param,"got error: %d when reading datafile at record: %s",errno, llstr(records,llbuff));
01269  err2:
01270   free(mi_get_rec_buff_ptr(info, record));
01271   param->testflag|=T_RETRY_WITHOUT_QUICK;
01272   return(1);
01273 } /* chk_data_link */
01274 
01275 
01329 static int mi_drop_all_indexes(MI_CHECK *param, MI_INFO *info, bool force)
01330 {
01331   MYISAM_SHARE *share= info->s;
01332   MI_STATE_INFO *state= &share->state;
01333   uint32_t i;
01334   int error;
01335 
01336   /*
01337     If any of the disabled indexes has a key block assigned, we must
01338     drop and recreate all indexes to avoid losing index blocks.
01339 
01340     If we want to recreate disabled indexes only _and_ all of these
01341     indexes are empty, we don't need to recreate the existing indexes.
01342   */
01343   if (!force && (param->testflag & T_CREATE_MISSING_KEYS))
01344   {
01345     for (i= 0; i < share->base.keys; i++)
01346     {
01347       if ((state->key_root[i] != HA_OFFSET_ERROR) &&
01348           !mi_is_key_active(state->key_map, i))
01349       {
01350         /*
01351           This index has at least one key block and it is disabled.
01352           We would lose its block(s) if would just recreate it.
01353           So we need to drop and recreate all indexes.
01354         */
01355         break;
01356       }
01357     }
01358     if (i >= share->base.keys)
01359     {
01360       /*
01361         All of the disabled indexes are empty. We can just recreate them.
01362         Flush dirty blocks of this index file from key cache and remove
01363         all blocks of this index file from key cache.
01364       */
01365       error= flush_key_blocks(share->getKeyCache(), share->kfile,
01366                               FLUSH_FORCE_WRITE);
01367       goto end;
01368     }
01369     /*
01370       We do now drop all indexes and declare them disabled. With the
01371       T_CREATE_MISSING_KEYS flag, mi_repair*() will recreate all
01372       disabled indexes and enable them.
01373     */
01374     mi_clear_all_keys_active(state->key_map);
01375   }
01376 
01377   /* Remove all key blocks of this index file from key cache. */
01378   if ((error= flush_key_blocks(share->getKeyCache(), share->kfile,
01379                                FLUSH_IGNORE_CHANGED)))
01380     goto end;
01381 
01382   /* Clear index root block pointers. */
01383   for (i= 0; i < share->base.keys; i++)
01384     state->key_root[i]= HA_OFFSET_ERROR;
01385 
01386   /* Clear the delete chains. */
01387   for (i= 0; i < state->header.max_block_size_index; i++)
01388     state->key_del[i]= HA_OFFSET_ERROR;
01389 
01390   /* Reset index file length to end of index file header. */
01391   info->state->key_file_length= share->base.keystart;
01392 
01393   /* error= 0; set by last (error= flush_key_bocks()). */
01394 
01395  end:
01396   return(error);
01397 }
01398 
01399 
01400   /* Recover old table by reading each record and writing all keys */
01401   /* Save new datafile-name in temp_filename */
01402 
01403 int mi_repair(MI_CHECK *param, register MI_INFO *info,
01404         char * name, int rep_quick)
01405 {
01406   int error,got_error;
01407   ha_rows start_records,new_header_length;
01408   my_off_t del;
01409   int new_file;
01410   MYISAM_SHARE *share=info->s;
01411   char llbuff[22],llbuff2[22];
01412   SORT_INFO sort_info;
01413   MI_SORT_PARAM sort_param;
01414 
01415   memset(&sort_info, 0, sizeof(sort_info));
01416   memset(&sort_param, 0, sizeof(sort_param));
01417   start_records=info->state->records;
01418   new_header_length=(param->testflag & T_UNPACK) ? 0L :
01419     share->pack.header_length;
01420   got_error=1;
01421   new_file= -1;
01422   sort_param.sort_info=&sort_info;
01423 
01424   if (!(param->testflag & T_SILENT))
01425   {
01426     printf("- recovering (with keycache) MyISAM-table '%s'\n",name);
01427     printf("Data records: %s\n", llstr(info->state->records,llbuff));
01428   }
01429   param->testflag|=T_REP; /* for easy checking */
01430 
01431   if (info->s->options & (HA_OPTION_COMPRESS_RECORD))
01432     param->testflag|=T_CALC_CHECKSUM;
01433 
01434   if (!param->using_global_keycache)
01435     assert(0);
01436 
01437   if (param->read_cache.init_io_cache(info->dfile, (uint) param->read_buffer_length, READ_CACHE,share->pack.header_length,1,MYF(MY_WME)))
01438   {
01439     memset(&info->rec_cache, 0, sizeof(info->rec_cache));
01440     goto err;
01441   }
01442   if (not rep_quick)
01443   {
01444     if (info->rec_cache.init_io_cache(-1, (uint) param->write_buffer_length, WRITE_CACHE, new_header_length, 1, MYF(MY_WME | MY_WAIT_IF_FULL)))
01445     {
01446       goto err;
01447     }
01448   }
01449   info->opt_flag|=WRITE_CACHE_USED;
01450   if (!mi_alloc_rec_buff(info, SIZE_MAX, &sort_param.record) ||
01451       !mi_alloc_rec_buff(info, SIZE_MAX, &sort_param.rec_buff))
01452   {
01453     mi_check_print_error(param, "Not enough memory for extra record");
01454     goto err;
01455   }
01456 
01457   if (!rep_quick)
01458   {
01459     /* Get real path for data file */
01460     if ((new_file=my_create(internal::fn_format(param->temp_filename,
01461                                       share->data_file_name, "",
01462                                       DATA_TMP_EXT, 2+4),
01463                             0,param->tmpfile_createflag,
01464                             MYF(0))) < 0)
01465     {
01466       mi_check_print_error(param,"Can't create new tempfile: '%s'",
01467                            param->temp_filename);
01468       goto err;
01469     }
01470     if (new_header_length &&
01471         filecopy(param,new_file,info->dfile,0L,new_header_length,
01472                  "datafile-header"))
01473       goto err;
01474     info->s->state.dellink= HA_OFFSET_ERROR;
01475     info->rec_cache.file=new_file;
01476     if (param->testflag & T_UNPACK)
01477     {
01478       share->options&= ~HA_OPTION_COMPRESS_RECORD;
01479       mi_int2store(share->state.header.options,share->options);
01480     }
01481   }
01482   sort_info.info=info;
01483   sort_info.param = param;
01484   sort_param.read_cache=param->read_cache;
01485   sort_param.pos=sort_param.max_pos=share->pack.header_length;
01486   sort_param.filepos=new_header_length;
01487   param->read_cache.end_of_file=sort_info.filelength=
01488     lseek(info->dfile,0L,SEEK_END);
01489   sort_info.dupp=0;
01490   sort_param.fix_datafile= (bool) (! rep_quick);
01491   sort_param.master=1;
01492   sort_info.max_records= ~(ha_rows) 0;
01493 
01494   set_data_file_type(&sort_info, share);
01495   del=info->state->del;
01496   info->state->records=info->state->del=share->state.split=0;
01497   info->state->empty=0;
01498   param->glob_crc=0;
01499   if (param->testflag & T_CALC_CHECKSUM)
01500     sort_param.calc_checksum= 1;
01501 
01502   info->update= (short) (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED);
01503 
01504   /* This function always recreates all enabled indexes. */
01505   if (param->testflag & T_CREATE_MISSING_KEYS)
01506     mi_set_all_keys_active(share->state.key_map, share->base.keys);
01507   mi_drop_all_indexes(param, info, true);
01508 
01509   lock_memory(param);     /* Everything is alloced */
01510 
01511   /* Re-create all keys, which are set in key_map. */
01512   while (!(error=sort_get_next_record(&sort_param)))
01513   {
01514     if (writekeys(&sort_param))
01515     {
01516       if (errno != HA_ERR_FOUND_DUPP_KEY)
01517   goto err;
01518       mi_check_print_info(param,"Duplicate key %2d for record at %10s against new record at %10s",
01519         info->errkey+1,
01520         llstr(sort_param.start_recpos,llbuff),
01521         llstr(info->dupp_key_pos,llbuff2));
01522       if (param->testflag & T_VERBOSE)
01523       {
01524   _mi_make_key(info,(uint) info->errkey,info->lastkey,
01525                      sort_param.record,0L);
01526       }
01527       sort_info.dupp++;
01528       if ((param->testflag & (T_FORCE_UNIQUENESS|T_QUICK)) == T_QUICK)
01529       {
01530         param->testflag|=T_RETRY_WITHOUT_QUICK;
01531   param->error_printed=1;
01532   goto err;
01533       }
01534       continue;
01535     }
01536     if (sort_write_record(&sort_param))
01537       goto err;
01538   }
01539   if (error > 0 || write_data_suffix(&sort_info, (bool)!rep_quick) ||
01540       flush_io_cache(&info->rec_cache) || param->read_cache.error < 0)
01541     goto err;
01542 
01543   if (param->testflag & T_WRITE_LOOP)
01544   {
01545     fputs("          \r",stdout); fflush(stdout);
01546   }
01547   if (ftruncate(share->kfile, info->state->key_file_length))
01548   {
01549     mi_check_print_warning(param,
01550          "Can't change size of indexfile, error: %d",
01551          errno);
01552     goto err;
01553   }
01554 
01555   if (rep_quick && del+sort_info.dupp != info->state->del)
01556   {
01557     mi_check_print_error(param,"Couldn't fix table with quick recovery: Found wrong number of deleted records");
01558     mi_check_print_error(param,"Run recovery again without -q");
01559     got_error=1;
01560     param->retry_repair=1;
01561     param->testflag|=T_RETRY_WITHOUT_QUICK;
01562     goto err;
01563   }
01564   if (param->testflag & T_SAFE_REPAIR)
01565   {
01566     /* Don't repair if we loosed more than one row */
01567     if (info->state->records+1 < start_records)
01568     {
01569       info->state->records=start_records;
01570       got_error=1;
01571       goto err;
01572     }
01573   }
01574 
01575   if (!rep_quick)
01576   {
01577     internal::my_close(info->dfile,MYF(0));
01578     info->dfile=new_file;
01579     info->state->data_file_length=sort_param.filepos;
01580     share->state.version=(ulong) time((time_t*) 0); /* Force reopen */
01581   }
01582   else
01583   {
01584     info->state->data_file_length=sort_param.max_pos;
01585   }
01586   if (param->testflag & T_CALC_CHECKSUM)
01587     info->state->checksum=param->glob_crc;
01588 
01589   if (!(param->testflag & T_SILENT))
01590   {
01591     if (start_records != info->state->records)
01592       printf("Data records: %s\n", llstr(info->state->records,llbuff));
01593     if (sort_info.dupp)
01594       mi_check_print_warning(param,
01595            "%s records have been removed",
01596            llstr(sort_info.dupp,llbuff));
01597   }
01598 
01599   got_error=0;
01600   /* If invoked by external program that uses thr_lock */
01601   if (&share->state.state != info->state)
01602     memcpy( &share->state.state, info->state, sizeof(*info->state));
01603 
01604 err:
01605   if (!got_error)
01606   {
01607     /* Replace the actual file with the temporary file */
01608     if (new_file >= 0)
01609     {
01610       internal::my_close(new_file,MYF(0));
01611       info->dfile=new_file= -1;
01612       if (change_to_newfile(share->data_file_name,MI_NAME_DEXT,
01613           DATA_TMP_EXT, share->base.raid_chunks,
01614           (param->testflag & T_BACKUP_DATA ?
01615            MYF(MY_REDEL_MAKE_BACKUP): MYF(0))) ||
01616     mi_open_datafile(info,share,-1))
01617   got_error=1;
01618     }
01619   }
01620   if (got_error)
01621   {
01622     if (! param->error_printed)
01623       mi_check_print_error(param,"%d for record at pos %s",errno,
01624       llstr(sort_param.start_recpos,llbuff));
01625     if (new_file >= 0)
01626     {
01627       internal::my_close(new_file,MYF(0));
01628       my_delete(param->temp_filename, MYF(MY_WME));
01629       info->rec_cache.file=-1; /* don't flush data to new_file, it's closed */
01630     }
01631     mi_mark_crashed_on_repair(info);
01632   }
01633 
01634   void * rec_buff_ptr= NULL;
01635   rec_buff_ptr= mi_get_rec_buff_ptr(info, sort_param.rec_buff);
01636   if (rec_buff_ptr != NULL)
01637     free(rec_buff_ptr);
01638   rec_buff_ptr= mi_get_rec_buff_ptr(info, sort_param.record);
01639   if (rec_buff_ptr != NULL)
01640     free(rec_buff_ptr);
01641   rec_buff_ptr= NULL;
01642 
01643   free(sort_info.buff);
01644   param->read_cache.end_io_cache();
01645   info->opt_flag&= ~(READ_CACHE_USED | WRITE_CACHE_USED);
01646   info->rec_cache.end_io_cache();
01647   got_error|= flush_blocks(param, share->getKeyCache(), share->kfile);
01648   if (not got_error && param->testflag & T_UNPACK)
01649   {
01650     share->state.header.options[0]&= (unsigned char) ~HA_OPTION_COMPRESS_RECORD;
01651     share->pack.header_length=0;
01652     share->data_file_type=sort_info.new_data_file_type;
01653   }
01654   share->state.changed|= (STATE_NOT_OPTIMIZED_KEYS | STATE_NOT_SORTED_PAGES |
01655         STATE_NOT_ANALYZED);
01656   return(got_error);
01657 }
01658 
01659 
01660 /* Uppate keyfile when doing repair */
01661 
01662 static int writekeys(MI_SORT_PARAM *sort_param)
01663 {
01664   register uint32_t i;
01665   unsigned char    *key;
01666   MI_INFO  *info=   sort_param->sort_info->info;
01667   unsigned char    *buff=   sort_param->record;
01668   my_off_t filepos= sort_param->filepos;
01669 
01670   key=info->lastkey+info->s->base.max_key_length;
01671   for (i=0 ; i < info->s->base.keys ; i++)
01672   {
01673     if (mi_is_key_active(info->s->state.key_map, i))
01674     {
01675       {
01676   uint32_t key_length=_mi_make_key(info,i,key,buff,filepos);
01677   if (_mi_ck_write(info,i,key,key_length))
01678     goto err;
01679       }
01680     }
01681   }
01682   return(0);
01683 
01684  err:
01685   if (errno == HA_ERR_FOUND_DUPP_KEY)
01686   {
01687     info->errkey=(int) i;     /* This key was found */
01688     while ( i-- > 0 )
01689     {
01690       if (mi_is_key_active(info->s->state.key_map, i))
01691       {
01692   {
01693     uint32_t key_length=_mi_make_key(info,i,key,buff,filepos);
01694     if (_mi_ck_delete(info,i,key,key_length))
01695       break;
01696   }
01697       }
01698     }
01699   }
01700   /* Remove checksum that was added to glob_crc in sort_get_next_record */
01701   if (sort_param->calc_checksum)
01702     sort_param->sort_info->param->glob_crc-= info->checksum;
01703   return(-1);
01704 } /* writekeys */
01705 
01706 
01707   /* Change all key-pointers that points to a records */
01708 
01709 int movepoint(register MI_INFO *info, unsigned char *record, my_off_t oldpos,
01710         my_off_t newpos, uint32_t prot_key)
01711 {
01712   register uint32_t i;
01713   unsigned char *key;
01714   uint32_t key_length;
01715 
01716   key=info->lastkey+info->s->base.max_key_length;
01717   for (i=0 ; i < info->s->base.keys; i++)
01718   {
01719     if (i != prot_key && mi_is_key_active(info->s->state.key_map, i))
01720     {
01721       key_length=_mi_make_key(info,i,key,record,oldpos);
01722       if (info->s->keyinfo[i].flag & HA_NOSAME)
01723       {         /* Change pointer direct */
01724   uint32_t nod_flag;
01725   MI_KEYDEF *keyinfo;
01726   keyinfo=info->s->keyinfo+i;
01727   if (_mi_search(info,keyinfo,key,USE_WHOLE_KEY,
01728            (uint) (SEARCH_SAME | SEARCH_SAVE_BUFF),
01729            info->s->state.key_root[i]))
01730     return(-1);
01731   nod_flag=mi_test_if_nod(info->buff);
01732   _mi_dpointer(info,info->int_keypos-nod_flag-
01733          info->s->rec_reflength,newpos);
01734   if (_mi_write_keypage(info,keyinfo,info->last_keypage,
01735                               DFLT_INIT_HITS,info->buff))
01736     return(-1);
01737       }
01738       else
01739       {         /* Change old key to new */
01740   if (_mi_ck_delete(info,i,key,key_length))
01741     return(-1);
01742   key_length=_mi_make_key(info,i,key,record,newpos);
01743   if (_mi_ck_write(info,i,key,key_length))
01744     return(-1);
01745       }
01746     }
01747   }
01748   return(0);
01749 } /* movepoint */
01750 
01751 
01752   /* Tell system that we want all memory for our cache */
01753 
01754 void lock_memory(MI_CHECK *)
01755 {
01756 } /* lock_memory */
01757 
01758 
01759   /* Flush all changed blocks to disk */
01760 
01761 int flush_blocks(MI_CHECK *param, KEY_CACHE *key_cache, int file)
01762 {
01763   if (flush_key_blocks(key_cache, file, FLUSH_RELEASE))
01764   {
01765     mi_check_print_error(param,"%d when trying to write bufferts",errno);
01766     return(1);
01767   }
01768   if (!param->using_global_keycache)
01769     end_key_cache(key_cache,1);
01770   return 0;
01771 } /* flush_blocks */
01772 
01773 
01774   /* Sort index for more efficent reads */
01775 
01776 int mi_sort_index(MI_CHECK *param, register MI_INFO *info, char * name)
01777 {
01778   register uint32_t key;
01779   register MI_KEYDEF *keyinfo;
01780   int new_file;
01781   my_off_t index_pos[HA_MAX_POSSIBLE_KEY];
01782   uint32_t r_locks,w_locks;
01783   int old_lock;
01784   MYISAM_SHARE *share=info->s;
01785   MI_STATE_INFO old_state;
01786 
01787   /* cannot sort index files with R-tree indexes */
01788   for (key= 0,keyinfo= &share->keyinfo[0]; key < share->base.keys ;
01789        key++,keyinfo++)
01790 
01791   if (!(param->testflag & T_SILENT))
01792     printf("- Sorting index for MyISAM-table '%s'\n",name);
01793 
01794   /* Get real path for index file */
01795   internal::fn_format(param->temp_filename,name,"", MI_NAME_IEXT,2+4+32);
01796   if ((new_file=my_create(internal::fn_format(param->temp_filename,param->temp_filename,
01797             "", INDEX_TMP_EXT,2+4),
01798         0,param->tmpfile_createflag,MYF(0))) <= 0)
01799   {
01800     mi_check_print_error(param,"Can't create new tempfile: '%s'",
01801        param->temp_filename);
01802     return(-1);
01803   }
01804   if (filecopy(param, new_file,share->kfile,0L,
01805          (ulong) share->base.keystart, "headerblock"))
01806     goto err;
01807 
01808   param->new_file_pos=share->base.keystart;
01809   for (key= 0,keyinfo= &share->keyinfo[0]; key < share->base.keys ;
01810        key++,keyinfo++)
01811   {
01812     if (! mi_is_key_active(info->s->state.key_map, key))
01813       continue;
01814 
01815     if (share->state.key_root[key] != HA_OFFSET_ERROR)
01816     {
01817       index_pos[key]=param->new_file_pos; /* Write first block here */
01818       if (sort_one_index(param,info,keyinfo,share->state.key_root[key],
01819        new_file))
01820   goto err;
01821     }
01822     else
01823       index_pos[key]= HA_OFFSET_ERROR;    /* No blocks */
01824   }
01825 
01826   /* Flush key cache for this file if we are calling this outside myisamchk */
01827   flush_key_blocks(share->getKeyCache(), share->kfile, FLUSH_IGNORE_CHANGED);
01828 
01829   share->state.version=(ulong) time((time_t*) 0);
01830   old_state= share->state;      /* save state if not stored */
01831   r_locks=   share->r_locks;
01832   w_locks=   share->w_locks;
01833   old_lock=  info->lock_type;
01834 
01835   /* Put same locks as old file */
01836   share->r_locks= share->w_locks= share->tot_locks= 0;
01837   (void) _mi_writeinfo(info,WRITEINFO_UPDATE_KEYFILE);
01838   internal::my_close(share->kfile,MYF(MY_WME));
01839   share->kfile = -1;
01840   internal::my_close(new_file,MYF(MY_WME));
01841   if (change_to_newfile(share->index_file_name,MI_NAME_IEXT,INDEX_TMP_EXT,0,
01842       MYF(0)) ||
01843       mi_open_keyfile(share))
01844     goto err2;
01845   info->lock_type= F_UNLCK;     /* Force mi_readinfo to lock */
01846   _mi_readinfo(info,F_WRLCK,0);     /* Will lock the table */
01847   info->lock_type=  old_lock;
01848   share->r_locks=   r_locks;
01849   share->w_locks=   w_locks;
01850   share->tot_locks= r_locks+w_locks;
01851   share->state=     old_state;      /* Restore old state */
01852 
01853   info->state->key_file_length=param->new_file_pos;
01854   info->update= (short) (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED);
01855   for (key=0 ; key < info->s->base.keys ; key++)
01856     info->s->state.key_root[key]=index_pos[key];
01857   for (key=0 ; key < info->s->state.header.max_block_size_index ; key++)
01858     info->s->state.key_del[key]=  HA_OFFSET_ERROR;
01859 
01860   info->s->state.changed&= ~STATE_NOT_SORTED_PAGES;
01861   return(0);
01862 
01863 err:
01864   internal::my_close(new_file,MYF(MY_WME));
01865 err2:
01866   my_delete(param->temp_filename,MYF(MY_WME));
01867   return(-1);
01868 } /* mi_sort_index */
01869 
01870 
01871    /* Sort records recursive using one index */
01872 
01873 static int sort_one_index(MI_CHECK *param, MI_INFO *info, MI_KEYDEF *keyinfo,
01874         my_off_t pagepos, int new_file)
01875 {
01876   uint32_t length,nod_flag,used_length, key_length;
01877   unsigned char *buff,*keypos,*endpos;
01878   unsigned char key[HA_MAX_POSSIBLE_KEY_BUFF];
01879   my_off_t new_page_pos,next_page;
01880   char llbuff[22];
01881 
01882   new_page_pos=param->new_file_pos;
01883   param->new_file_pos+=keyinfo->block_length;
01884 
01885   if (!(buff=(unsigned char*) malloc(keyinfo->block_length)))
01886   {
01887     mi_check_print_error(param,"Not enough memory for key block");
01888     return(-1);
01889   }
01890   if (!_mi_fetch_keypage(info,keyinfo,pagepos,DFLT_INIT_HITS,buff,0))
01891   {
01892     mi_check_print_error(param,"Can't read key block from filepos: %s",
01893     llstr(pagepos,llbuff));
01894     goto err;
01895   }
01896   if ((nod_flag=mi_test_if_nod(buff)))
01897   {
01898     used_length=mi_getint(buff);
01899     keypos=buff+2+nod_flag;
01900     endpos=buff+used_length;
01901     for ( ;; )
01902     {
01903       if (nod_flag)
01904       {
01905   next_page=_mi_kpos(nod_flag,keypos);
01906   _mi_kpointer(info,keypos-nod_flag,param->new_file_pos); /* Save new pos */
01907   if (sort_one_index(param,info,keyinfo,next_page,new_file))
01908   {
01909     goto err;
01910   }
01911       }
01912       if (keypos >= endpos ||
01913     (key_length=(*keyinfo->get_key)(keyinfo,nod_flag,&keypos,key)) == 0)
01914   break;
01915       assert(keypos <= endpos);
01916     }
01917   }
01918 
01919   /* Fill block with zero and write it to the new index file */
01920   length=mi_getint(buff);
01921   memset(buff+length, 0, keyinfo->block_length-length);
01922   if (my_pwrite(new_file,(unsigned char*) buff,(uint) keyinfo->block_length,
01923     new_page_pos,MYF(MY_NABP | MY_WAIT_IF_FULL)))
01924   {
01925     mi_check_print_error(param,"Can't write indexblock, error: %d",errno);
01926     goto err;
01927   }
01928   free(buff);
01929   return(0);
01930 err:
01931   free(buff);
01932   return(1);
01933 } /* sort_one_index */
01934 
01935 
01936   /*
01937     Let temporary file replace old file.
01938     This assumes that the new file was created in the same
01939     directory as given by realpath(filename).
01940     This will ensure that any symlinks that are used will still work.
01941     Copy stats from old file to new file, deletes orignal and
01942     changes new file name to old file name
01943   */
01944 
01945 int change_to_newfile(const char * filename, const char * old_ext,
01946           const char * new_ext,
01947           uint32_t raid_chunks,
01948           myf MyFlags)
01949 {
01950   (void)raid_chunks;
01951   char old_filename[FN_REFLEN],new_filename[FN_REFLEN];
01952   /* Get real path to filename */
01953   (void) internal::fn_format(old_filename,filename,"",old_ext,2+4+32);
01954   return my_redel(old_filename,
01955       internal::fn_format(new_filename,old_filename,"",new_ext,2+4),
01956       MYF(MY_WME | MY_LINK_WARNING | MyFlags));
01957 } /* change_to_newfile */
01958 
01959 
01960 
01961   /* Copy a block between two files */
01962 
01963 int filecopy(MI_CHECK *param, int to,int from,my_off_t start,
01964        my_off_t length, const char *type)
01965 {
01966   char tmp_buff[IO_SIZE],*buff;
01967   ulong buff_length;
01968 
01969   buff_length=(ulong) min(param->write_buffer_length, (size_t)length);
01970   if (!(buff=(char *)malloc(buff_length)))
01971   {
01972     buff=tmp_buff; buff_length=IO_SIZE;
01973   }
01974 
01975   lseek(from,start,SEEK_SET);
01976   while (length > buff_length)
01977   {
01978     if (my_read(from,(unsigned char*) buff,buff_length,MYF(MY_NABP)) ||
01979   my_write(to,(unsigned char*) buff,buff_length,param->myf_rw))
01980       goto err;
01981     length-= buff_length;
01982   }
01983   if (my_read(from,(unsigned char*) buff,(uint) length,MYF(MY_NABP)) ||
01984       my_write(to,(unsigned char*) buff,(uint) length,param->myf_rw))
01985     goto err;
01986   if (buff != tmp_buff)
01987     free(buff);
01988   return(0);
01989 err:
01990   if (buff != tmp_buff)
01991     free(buff);
01992   mi_check_print_error(param,"Can't copy %s to tempfile, error %d",
01993            type,errno);
01994   return(1);
01995 }
01996 
01997 
01998 /*
01999   Repair table or given index using sorting
02000 
02001   SYNOPSIS
02002     mi_repair_by_sort()
02003     param   Repair parameters
02004     info    MyISAM handler to repair
02005     name    Name of table (for warnings)
02006     rep_quick   set to <> 0 if we should not change data file
02007 
02008   RESULT
02009     0 ok
02010     <>0 Error
02011 */
02012 
02013 int mi_repair_by_sort(MI_CHECK *param, register MI_INFO *info,
02014           const char * name, int rep_quick)
02015 {
02016   int got_error;
02017   uint32_t i;
02018   ulong length;
02019   ha_rows start_records;
02020   my_off_t new_header_length,del;
02021   int new_file;
02022   MI_SORT_PARAM sort_param;
02023   MYISAM_SHARE *share=info->s;
02024   HA_KEYSEG *keyseg;
02025   ulong   *rec_per_key_part;
02026   char llbuff[22];
02027   SORT_INFO sort_info;
02028   uint64_t key_map= 0;
02029 
02030   start_records=info->state->records;
02031   got_error=1;
02032   new_file= -1;
02033   new_header_length=(param->testflag & T_UNPACK) ? 0 :
02034     share->pack.header_length;
02035   if (!(param->testflag & T_SILENT))
02036   {
02037     printf("- recovering (with sort) MyISAM-table '%s'\n",name);
02038     printf("Data records: %s\n", llstr(start_records,llbuff));
02039   }
02040   param->testflag|=T_REP; /* for easy checking */
02041 
02042   if (info->s->options & (HA_OPTION_COMPRESS_RECORD))
02043     param->testflag|=T_CALC_CHECKSUM;
02044 
02045   memset(&sort_info, 0, sizeof(sort_info));
02046   memset(&sort_param, 0, sizeof(sort_param));
02047   if (!(sort_info.key_block=
02048         alloc_key_blocks(param, (uint) param->sort_key_blocks, share->base.max_key_block_length))
02049       || param->read_cache.init_io_cache(info->dfile, (uint) param->read_buffer_length, READ_CACHE,share->pack.header_length,1,MYF(MY_WME))
02050       || (! rep_quick && info->rec_cache.init_io_cache(info->dfile, (uint) param->write_buffer_length, WRITE_CACHE,new_header_length,1, MYF(MY_WME | MY_WAIT_IF_FULL) & param->myf_rw)))
02051   {
02052     goto err;
02053   }
02054   sort_info.key_block_end=sort_info.key_block+param->sort_key_blocks;
02055   info->opt_flag|=WRITE_CACHE_USED;
02056   info->rec_cache.file=info->dfile;   /* for sort_delete_record */
02057 
02058   if (!mi_alloc_rec_buff(info, SIZE_MAX, &sort_param.record) ||
02059       !mi_alloc_rec_buff(info, SIZE_MAX, &sort_param.rec_buff))
02060   {
02061     mi_check_print_error(param, "Not enough memory for extra record");
02062     goto err;
02063   }
02064   if (!rep_quick)
02065   {
02066     /* Get real path for data file */
02067     if ((new_file=my_create(internal::fn_format(param->temp_filename,
02068                                       share->data_file_name, "",
02069                                       DATA_TMP_EXT, 2+4),
02070                             0,param->tmpfile_createflag,
02071                             MYF(0))) < 0)
02072     {
02073       mi_check_print_error(param,"Can't create new tempfile: '%s'",
02074                            param->temp_filename);
02075       goto err;
02076     }
02077     if (new_header_length &&
02078         filecopy(param, new_file,info->dfile,0L,new_header_length,
02079      "datafile-header"))
02080       goto err;
02081     if (param->testflag & T_UNPACK)
02082     {
02083       share->options&= ~HA_OPTION_COMPRESS_RECORD;
02084       mi_int2store(share->state.header.options,share->options);
02085     }
02086     share->state.dellink= HA_OFFSET_ERROR;
02087     info->rec_cache.file=new_file;
02088   }
02089 
02090   info->update= (short) (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED);
02091 
02092   /* Optionally drop indexes and optionally modify the key_map. */
02093   mi_drop_all_indexes(param, info, false);
02094   key_map= share->state.key_map;
02095   if (param->testflag & T_CREATE_MISSING_KEYS)
02096   {
02097     /* Invert the copied key_map to recreate all disabled indexes. */
02098     key_map= ~key_map;
02099   }
02100 
02101   sort_info.info=info;
02102   sort_info.param = param;
02103 
02104   set_data_file_type(&sort_info, share);
02105   sort_param.filepos=new_header_length;
02106   sort_info.dupp=0;
02107   sort_info.buff=0;
02108   param->read_cache.end_of_file=sort_info.filelength=
02109     lseek(param->read_cache.file,0L,SEEK_END);
02110 
02111   sort_param.wordlist=NULL;
02112 
02113   if (share->data_file_type == DYNAMIC_RECORD)
02114     length=max(share->base.min_pack_length+1,share->base.min_block_length);
02115   else if (share->data_file_type == COMPRESSED_RECORD)
02116     length=share->base.min_block_length;
02117   else
02118     length=share->base.pack_reclength;
02119   sort_info.max_records=
02120     ((param->testflag & T_CREATE_MISSING_KEYS) ? info->state->records :
02121      (ha_rows) (sort_info.filelength/length+1));
02122   sort_param.key_cmp=sort_key_cmp;
02123   sort_param.lock_in_memory=lock_memory;
02124   sort_param.sort_info=&sort_info;
02125   sort_param.fix_datafile= (bool) (! rep_quick);
02126   sort_param.master =1;
02127 
02128   del=info->state->del;
02129   param->glob_crc=0;
02130   if (param->testflag & T_CALC_CHECKSUM)
02131     sort_param.calc_checksum= 1;
02132 
02133   rec_per_key_part= param->rec_per_key_part;
02134   for (sort_param.key=0 ; sort_param.key < share->base.keys ;
02135        rec_per_key_part+=sort_param.keyinfo->keysegs, sort_param.key++)
02136   {
02137     sort_param.read_cache=param->read_cache;
02138     sort_param.keyinfo=share->keyinfo+sort_param.key;
02139     sort_param.seg=sort_param.keyinfo->seg;
02140     /*
02141       Skip this index if it is marked disabled in the copied
02142       (and possibly inverted) key_map.
02143     */
02144     if (! mi_is_key_active(key_map, sort_param.key))
02145     {
02146       /* Remember old statistics for key */
02147       assert(rec_per_key_part >= param->rec_per_key_part);
02148       memcpy(rec_per_key_part,
02149        (share->state.rec_per_key_part +
02150               (rec_per_key_part - param->rec_per_key_part)),
02151        sort_param.keyinfo->keysegs*sizeof(*rec_per_key_part));
02152       continue;
02153     }
02154 
02155     if ((!(param->testflag & T_SILENT)))
02156       printf ("- Fixing index %d\n",sort_param.key+1);
02157     sort_param.max_pos=sort_param.pos=share->pack.header_length;
02158     keyseg=sort_param.seg;
02159     memset(sort_param.unique, 0, sizeof(sort_param.unique));
02160     sort_param.key_length=share->rec_reflength;
02161     for (i=0 ; keyseg[i].type != HA_KEYTYPE_END; i++)
02162     {
02163       sort_param.key_length+=keyseg[i].length;
02164       if (keyseg[i].flag & HA_SPACE_PACK)
02165   sort_param.key_length+=get_pack_length(keyseg[i].length);
02166       if (keyseg[i].flag & (HA_BLOB_PART | HA_VAR_LENGTH_PART))
02167   sort_param.key_length+=2 + test(keyseg[i].length >= 127);
02168       if (keyseg[i].flag & HA_NULL_PART)
02169   sort_param.key_length++;
02170     }
02171     info->state->records=info->state->del=share->state.split=0;
02172     info->state->empty=0;
02173 
02174     {
02175       sort_param.key_read=sort_key_read;
02176       sort_param.key_write=sort_key_write;
02177     }
02178 
02179     if (_create_index_by_sort(&sort_param,
02180             (bool) (!(param->testflag & T_VERBOSE)),
02181             (uint) param->sort_buffer_length))
02182     {
02183       param->retry_repair=1;
02184       goto err;
02185     }
02186     /* No need to calculate checksum again. */
02187     sort_param.calc_checksum= 0;
02188     sort_param.wordroot.free_root(MYF(0));
02189 
02190     /* Set for next loop */
02191     sort_info.max_records= (ha_rows) info->state->records;
02192 
02193     if (param->testflag & T_STATISTICS)
02194       update_key_parts(sort_param.keyinfo, rec_per_key_part, sort_param.unique,
02195                        param->stats_method == MI_STATS_METHOD_IGNORE_NULLS?
02196                        sort_param.notnull: NULL,
02197                        (uint64_t) info->state->records);
02198     /* Enable this index in the permanent (not the copied) key_map. */
02199     mi_set_key_active(share->state.key_map, sort_param.key);
02200 
02201     if (sort_param.fix_datafile)
02202     {
02203       param->read_cache.end_of_file=sort_param.filepos;
02204       if (write_data_suffix(&sort_info, 1) || info->rec_cache.end_io_cache())
02205       {
02206   goto err;
02207       }
02208       if (param->testflag & T_SAFE_REPAIR)
02209       {
02210   /* Don't repair if we loosed more than one row */
02211   if (info->state->records+1 < start_records)
02212   {
02213     info->state->records=start_records;
02214     goto err;
02215   }
02216       }
02217       share->state.state.data_file_length = info->state->data_file_length=
02218   sort_param.filepos;
02219       /* Only whole records */
02220       share->state.version=(ulong) time((time_t*) 0);
02221       internal::my_close(info->dfile,MYF(0));
02222       info->dfile=new_file;
02223       share->data_file_type=sort_info.new_data_file_type;
02224       share->pack.header_length=(ulong) new_header_length;
02225       sort_param.fix_datafile=0;
02226     }
02227     else
02228       info->state->data_file_length=sort_param.max_pos;
02229 
02230     param->read_cache.file=info->dfile;   /* re-init read cache */
02231     param->read_cache.reinit_io_cache(READ_CACHE,share->pack.header_length, 1,1);
02232   }
02233 
02234   if (param->testflag & T_WRITE_LOOP)
02235   {
02236     fputs("          \r",stdout); fflush(stdout);
02237   }
02238 
02239   if (rep_quick && del+sort_info.dupp != info->state->del)
02240   {
02241     mi_check_print_error(param,"Couldn't fix table with quick recovery: Found wrong number of deleted records");
02242     mi_check_print_error(param,"Run recovery again without -q");
02243     got_error=1;
02244     param->retry_repair=1;
02245     param->testflag|=T_RETRY_WITHOUT_QUICK;
02246     goto err;
02247   }
02248 
02249   if (rep_quick & T_FORCE_UNIQUENESS)
02250   {
02251     my_off_t skr=info->state->data_file_length+
02252       (share->options & HA_OPTION_COMPRESS_RECORD ?
02253        MEMMAP_EXTRA_MARGIN : 0);
02254 #ifdef USE_RELOC
02255     if (share->data_file_type == STATIC_RECORD &&
02256   skr < share->base.reloc*share->base.min_pack_length)
02257       skr=share->base.reloc*share->base.min_pack_length;
02258 #endif
02259     if (skr != sort_info.filelength && !info->s->base.raid_type)
02260       if (ftruncate(info->dfile, skr))
02261   mi_check_print_warning(param,
02262              "Can't change size of datafile,  error: %d",
02263              errno);
02264   }
02265   if (param->testflag & T_CALC_CHECKSUM)
02266     info->state->checksum=param->glob_crc;
02267 
02268   if (ftruncate(share->kfile, info->state->key_file_length))
02269     mi_check_print_warning(param,
02270          "Can't change size of indexfile, error: %d",
02271          errno);
02272 
02273   if (!(param->testflag & T_SILENT))
02274   {
02275     if (start_records != info->state->records)
02276       printf("Data records: %s\n", llstr(info->state->records,llbuff));
02277     if (sort_info.dupp)
02278       mi_check_print_warning(param,
02279            "%s records have been removed",
02280            llstr(sort_info.dupp,llbuff));
02281   }
02282   got_error=0;
02283 
02284   if (&share->state.state != info->state)
02285     memcpy( &share->state.state, info->state, sizeof(*info->state));
02286 
02287 err:
02288   got_error|= flush_blocks(param, share->getKeyCache(), share->kfile);
02289   info->rec_cache.end_io_cache();
02290   if (!got_error)
02291   {
02292     /* Replace the actual file with the temporary file */
02293     if (new_file >= 0)
02294     {
02295       internal::my_close(new_file,MYF(0));
02296       info->dfile=new_file= -1;
02297       if (change_to_newfile(share->data_file_name,MI_NAME_DEXT,
02298           DATA_TMP_EXT, share->base.raid_chunks,
02299           (param->testflag & T_BACKUP_DATA ?
02300            MYF(MY_REDEL_MAKE_BACKUP): MYF(0))) ||
02301     mi_open_datafile(info,share,-1))
02302   got_error=1;
02303     }
02304   }
02305   if (got_error)
02306   {
02307     if (! param->error_printed)
02308       mi_check_print_error(param,"%d when fixing table",errno);
02309     if (new_file >= 0)
02310     {
02311       internal::my_close(new_file,MYF(0));
02312       my_delete(param->temp_filename, MYF(MY_WME));
02313       if (info->dfile == new_file)
02314         info->dfile= -1;
02315     }
02316     mi_mark_crashed_on_repair(info);
02317   }
02318   else if (key_map == share->state.key_map)
02319     share->state.changed&= ~STATE_NOT_OPTIMIZED_KEYS;
02320   share->state.changed|=STATE_NOT_SORTED_PAGES;
02321 
02322   void * rec_buff_ptr= NULL;
02323   rec_buff_ptr= mi_get_rec_buff_ptr(info, sort_param.rec_buff);
02324   if (rec_buff_ptr != NULL)
02325     free(rec_buff_ptr);
02326   rec_buff_ptr= mi_get_rec_buff_ptr(info, sort_param.record);
02327   if (rec_buff_ptr != NULL)
02328     free(rec_buff_ptr);
02329   rec_buff_ptr= NULL;
02330 
02331   free((unsigned char*) sort_info.key_block);
02332   free(sort_info.buff);
02333   param->read_cache.end_io_cache();
02334   info->opt_flag&= ~(READ_CACHE_USED | WRITE_CACHE_USED);
02335   if (!got_error && (param->testflag & T_UNPACK))
02336   {
02337     share->state.header.options[0]&= (unsigned char) ~HA_OPTION_COMPRESS_RECORD;
02338     share->pack.header_length=0;
02339   }
02340   return(got_error);
02341 }
02342 
02343   /* Read next record and return next key */
02344 
02345 int sort_key_read(MI_SORT_PARAM *sort_param, void *key)
02346 {
02347   int error;
02348   SORT_INFO *sort_info=sort_param->sort_info;
02349   MI_INFO *info=sort_info->info;
02350 
02351   if ((error=sort_get_next_record(sort_param)))
02352     return(error);
02353   if (info->state->records == sort_info->max_records)
02354   {
02355     mi_check_print_error(sort_info->param,
02356        "Key %d - Found too many records; Can't continue",
02357                          sort_param->key+1);
02358     return(1);
02359   }
02360   sort_param->real_key_length=
02361     (info->s->rec_reflength+
02362      _mi_make_key(info, sort_param->key, (unsigned char*) key,
02363       sort_param->record, sort_param->filepos));
02364 #ifdef HAVE_VALGRIND
02365   memset((unsigned char *)key+sort_param->real_key_length, 0,
02366          (sort_param->key_length-sort_param->real_key_length));
02367 #endif
02368   return(sort_write_record(sort_param));
02369 } /* sort_key_read */
02370 
02371 
02372 /*
02373   Read next record from file using parameters in sort_info.
02374 
02375   SYNOPSIS
02376     sort_get_next_record()
02377       sort_param                Information about and for the sort process
02378 
02379   NOTE
02380 
02381     Dynamic Records With Non-Quick Parallel Repair
02382 
02383       For non-quick parallel repair we use a synchronized read/write
02384       cache. This means that one thread is the master who fixes the data
02385       file by reading each record from the old data file and writing it
02386       to the new data file. By doing this the records in the new data
02387       file are written contiguously. Whenever the write buffer is full,
02388       it is copied to the read buffer. The slaves read from the read
02389       buffer, which is not associated with a file. Thus read_cache.file
02390       is -1. When using _mi_read_cache(), the slaves must always set
02391       flag to READING_NEXT so that the function never tries to read from
02392       file. This is safe because the records are contiguous. There is no
02393       need to read outside the cache. This condition is evaluated in the
02394       variable 'parallel_flag' for quick reference. read_cache.file must
02395       be >= 0 in every other case.
02396 
02397   RETURN
02398     -1          end of file
02399     0           ok
02400     > 0         error
02401 */
02402 
02403 int sort_get_next_record(MI_SORT_PARAM *sort_param)
02404 {
02405   int searching;
02406   int parallel_flag;
02407   uint32_t found_record,b_type,left_length;
02408   my_off_t pos;
02409   unsigned char *to= NULL;
02410   MI_BLOCK_INFO block_info;
02411   SORT_INFO *sort_info=sort_param->sort_info;
02412   MI_CHECK *param=sort_info->param;
02413   MI_INFO *info=sort_info->info;
02414   MYISAM_SHARE *share=info->s;
02415   char llbuff[22],llbuff2[22];
02416 
02417   if (*killed_ptr(param))
02418     return(1);
02419 
02420   switch (share->data_file_type) {
02421   case STATIC_RECORD:
02422     for (;;)
02423     {
02424       if (my_b_read(&sort_param->read_cache,sort_param->record,
02425         share->base.pack_reclength))
02426       {
02427   if (sort_param->read_cache.error)
02428     param->out_flag |= O_DATA_LOST;
02429         param->retry_repair=1;
02430         param->testflag|=T_RETRY_WITHOUT_QUICK;
02431   return(-1);
02432       }
02433       sort_param->start_recpos=sort_param->pos;
02434       if (!sort_param->fix_datafile)
02435       {
02436   sort_param->filepos=sort_param->pos;
02437         if (sort_param->master)
02438     share->state.split++;
02439       }
02440       sort_param->max_pos=(sort_param->pos+=share->base.pack_reclength);
02441       if (*sort_param->record)
02442       {
02443   if (sort_param->calc_checksum)
02444     param->glob_crc+= (info->checksum=
02445            mi_static_checksum(info,sort_param->record));
02446   return(0);
02447       }
02448       if (!sort_param->fix_datafile && sort_param->master)
02449       {
02450   info->state->del++;
02451   info->state->empty+=share->base.pack_reclength;
02452       }
02453     }
02454   case DYNAMIC_RECORD:
02455     pos= sort_param->pos;
02456     searching= (sort_param->fix_datafile && (param->testflag & T_EXTEND));
02457     parallel_flag= (sort_param->read_cache.file < 0) ? READING_NEXT : 0;
02458     for (;;)
02459     {
02460       found_record=block_info.second_read= 0;
02461       left_length=1;
02462       if (searching)
02463       {
02464   pos=MY_ALIGN(pos,MI_DYN_ALIGN_SIZE);
02465         param->testflag|=T_RETRY_WITHOUT_QUICK;
02466   sort_param->start_recpos=pos;
02467       }
02468       do
02469       {
02470   if (pos > sort_param->max_pos)
02471     sort_param->max_pos=pos;
02472   if (pos & (MI_DYN_ALIGN_SIZE-1))
02473   {
02474     if ((param->testflag & T_VERBOSE) || searching == 0)
02475       mi_check_print_info(param,"Wrong aligned block at %s",
02476         llstr(pos,llbuff));
02477     if (searching)
02478       goto try_next;
02479   }
02480   if (found_record && pos == param->search_after_block)
02481     mi_check_print_info(param,"Block: %s used by record at %s",
02482          llstr(param->search_after_block,llbuff),
02483          llstr(sort_param->start_recpos,llbuff2));
02484   if (_mi_read_cache(&sort_param->read_cache,
02485                            (unsigned char*) block_info.header,pos,
02486          MI_BLOCK_INFO_HEADER_LENGTH,
02487          (! found_record ? READING_NEXT : 0) |
02488                            parallel_flag | READING_HEADER))
02489   {
02490     if (found_record)
02491     {
02492       mi_check_print_info(param,
02493         "Can't read whole record at %s (errno: %d)",
02494         llstr(sort_param->start_recpos,llbuff),errno);
02495       goto try_next;
02496     }
02497     return(-1);
02498   }
02499   if (searching && ! sort_param->fix_datafile)
02500   {
02501     param->error_printed=1;
02502           param->retry_repair=1;
02503           param->testflag|=T_RETRY_WITHOUT_QUICK;
02504     return(1);  /* Something wrong with data */
02505   }
02506   b_type=_mi_get_block_info(&block_info,-1,pos);
02507   if ((b_type & (BLOCK_ERROR | BLOCK_FATAL_ERROR)) ||
02508      ((b_type & BLOCK_FIRST) &&
02509        (block_info.rec_len < (uint) share->base.min_pack_length ||
02510         block_info.rec_len > (uint) share->base.max_pack_length)))
02511   {
02512     uint32_t i;
02513     if (param->testflag & T_VERBOSE || searching == 0)
02514       mi_check_print_info(param,
02515         "Wrong bytesec: %3d-%3d-%3d at %10s; Skipped",
02516            block_info.header[0],block_info.header[1],
02517            block_info.header[2],llstr(pos,llbuff));
02518     if (found_record)
02519       goto try_next;
02520     block_info.second_read=0;
02521     searching=1;
02522     /* Search after block in read header string */
02523     for (i=MI_DYN_ALIGN_SIZE ;
02524          i < MI_BLOCK_INFO_HEADER_LENGTH ;
02525          i+= MI_DYN_ALIGN_SIZE)
02526       if (block_info.header[i] >= 1 &&
02527     block_info.header[i] <= MI_MAX_DYN_HEADER_BYTE)
02528         break;
02529     pos+=(ulong) i;
02530     sort_param->start_recpos=pos;
02531     continue;
02532   }
02533   if (b_type & BLOCK_DELETED)
02534   {
02535     bool error=0;
02536     if (block_info.block_len+ (uint) (block_info.filepos-pos) <
02537         share->base.min_block_length)
02538     {
02539       if (!searching)
02540         mi_check_print_info(param,
02541           "Deleted block with impossible length %u at %s",
02542           block_info.block_len,llstr(pos,llbuff));
02543       error=1;
02544     }
02545     else
02546     {
02547       if ((block_info.next_filepos != HA_OFFSET_ERROR &&
02548      block_info.next_filepos >=
02549      info->state->data_file_length) ||
02550     (block_info.prev_filepos != HA_OFFSET_ERROR &&
02551      block_info.prev_filepos >= info->state->data_file_length))
02552       {
02553         if (!searching)
02554     mi_check_print_info(param,
02555             "Delete link points outside datafile at %s",
02556             llstr(pos,llbuff));
02557         error=1;
02558       }
02559     }
02560     if (error)
02561     {
02562       if (found_record)
02563         goto try_next;
02564       searching=1;
02565       pos+= MI_DYN_ALIGN_SIZE;
02566       sort_param->start_recpos=pos;
02567       block_info.second_read=0;
02568       continue;
02569     }
02570   }
02571   else
02572   {
02573     if (block_info.block_len+ (uint) (block_info.filepos-pos) <
02574         share->base.min_block_length ||
02575         block_info.block_len > (uint) share->base.max_pack_length+
02576         MI_SPLIT_LENGTH)
02577     {
02578       if (!searching)
02579         mi_check_print_info(param,
02580           "Found block with impossible length %u at %s; Skipped",
02581           block_info.block_len+ (uint) (block_info.filepos-pos),
02582           llstr(pos,llbuff));
02583       if (found_record)
02584         goto try_next;
02585       searching=1;
02586       pos+= MI_DYN_ALIGN_SIZE;
02587       sort_param->start_recpos=pos;
02588       block_info.second_read=0;
02589       continue;
02590     }
02591   }
02592   if (b_type & (BLOCK_DELETED | BLOCK_SYNC_ERROR))
02593   {
02594           if (!sort_param->fix_datafile && sort_param->master &&
02595               (b_type & BLOCK_DELETED))
02596     {
02597       info->state->empty+=block_info.block_len;
02598       info->state->del++;
02599       share->state.split++;
02600     }
02601     if (found_record)
02602       goto try_next;
02603     if (searching)
02604     {
02605       pos+=MI_DYN_ALIGN_SIZE;
02606       sort_param->start_recpos=pos;
02607     }
02608     else
02609       pos=block_info.filepos+block_info.block_len;
02610     block_info.second_read=0;
02611     continue;
02612   }
02613 
02614   if (!sort_param->fix_datafile && sort_param->master)
02615     share->state.split++;
02616   if (! found_record++)
02617   {
02618     sort_param->find_length=left_length=block_info.rec_len;
02619     sort_param->start_recpos=pos;
02620     if (!sort_param->fix_datafile)
02621       sort_param->filepos=sort_param->start_recpos;
02622     if (sort_param->fix_datafile && (param->testflag & T_EXTEND))
02623       sort_param->pos=block_info.filepos+1;
02624     else
02625       sort_param->pos=block_info.filepos+block_info.block_len;
02626     if (share->base.blobs)
02627     {
02628       if (!(to=mi_alloc_rec_buff(info,block_info.rec_len,
02629                &(sort_param->rec_buff))))
02630       {
02631         if (param->max_record_length >= block_info.rec_len)
02632         {
02633     mi_check_print_error(param,"Not enough memory for blob at %s (need %lu)",
02634              llstr(sort_param->start_recpos,llbuff),
02635              (ulong) block_info.rec_len);
02636     return(1);
02637         }
02638         else
02639         {
02640     mi_check_print_info(param,"Not enough memory for blob at %s (need %lu); Row skipped",
02641             llstr(sort_param->start_recpos,llbuff),
02642             (ulong) block_info.rec_len);
02643     goto try_next;
02644         }
02645       }
02646     }
02647     else
02648       to= sort_param->rec_buff;
02649   }
02650   if (left_length < block_info.data_len || ! block_info.data_len)
02651   {
02652     mi_check_print_info(param,
02653             "Found block with too small length at %s; Skipped",
02654             llstr(sort_param->start_recpos,llbuff));
02655     goto try_next;
02656   }
02657   if (block_info.filepos + block_info.data_len >
02658       sort_param->read_cache.end_of_file)
02659   {
02660     mi_check_print_info(param,
02661             "Found block that points outside data file at %s",
02662             llstr(sort_param->start_recpos,llbuff));
02663     goto try_next;
02664   }
02665         /*
02666           Copy information that is already read. Avoid accessing data
02667           below the cache start. This could happen if the header
02668           streched over the end of the previous buffer contents.
02669         */
02670         {
02671           uint32_t header_len= (uint) (block_info.filepos - pos);
02672           uint32_t prefetch_len= (MI_BLOCK_INFO_HEADER_LENGTH - header_len);
02673 
02674           if (prefetch_len > block_info.data_len)
02675             prefetch_len= block_info.data_len;
02676           if (prefetch_len)
02677           {
02678             memcpy(to, block_info.header + header_len, prefetch_len);
02679             block_info.filepos+= prefetch_len;
02680             block_info.data_len-= prefetch_len;
02681             left_length-= prefetch_len;
02682             to+= prefetch_len;
02683           }
02684         }
02685         if (block_info.data_len &&
02686             _mi_read_cache(&sort_param->read_cache,to,block_info.filepos,
02687                            block_info.data_len,
02688                            (found_record == 1 ? READING_NEXT : 0) |
02689                            parallel_flag))
02690   {
02691     mi_check_print_info(param,
02692             "Read error for block at: %s (error: %d); Skipped",
02693             llstr(block_info.filepos,llbuff),errno);
02694     goto try_next;
02695   }
02696   left_length-=block_info.data_len;
02697   to+=block_info.data_len;
02698   pos=block_info.next_filepos;
02699   if (pos == HA_OFFSET_ERROR && left_length)
02700   {
02701     mi_check_print_info(param,"Wrong block with wrong total length starting at %s",
02702             llstr(sort_param->start_recpos,llbuff));
02703     goto try_next;
02704   }
02705   if (pos + MI_BLOCK_INFO_HEADER_LENGTH > sort_param->read_cache.end_of_file)
02706   {
02707     mi_check_print_info(param,"Found link that points at %s (outside data file) at %s",
02708             llstr(pos,llbuff2),
02709             llstr(sort_param->start_recpos,llbuff));
02710     goto try_next;
02711   }
02712       } while (left_length);
02713 
02714       if (_mi_rec_unpack(info,sort_param->record,sort_param->rec_buff,
02715        sort_param->find_length) != MY_FILE_ERROR)
02716       {
02717   if (sort_param->read_cache.error < 0)
02718     return(1);
02719   if (sort_param->calc_checksum)
02720     info->checksum= mi_checksum(info, sort_param->record);
02721   if ((param->testflag & (T_EXTEND | T_REP)) || searching)
02722   {
02723     if (_mi_rec_check(info, sort_param->record, sort_param->rec_buff,
02724                             sort_param->find_length,
02725                             (param->testflag & T_QUICK) &&
02726                             sort_param->calc_checksum &&
02727                             test(info->s->calc_checksum)))
02728     {
02729       mi_check_print_info(param,"Found wrong packed record at %s",
02730         llstr(sort_param->start_recpos,llbuff));
02731       goto try_next;
02732     }
02733   }
02734   if (sort_param->calc_checksum)
02735     param->glob_crc+= info->checksum;
02736   return(0);
02737       }
02738       if (!searching)
02739         mi_check_print_info(param,"Key %d - Found wrong stored record at %s",
02740                             sort_param->key+1,
02741                             llstr(sort_param->start_recpos,llbuff));
02742     try_next:
02743       pos=(sort_param->start_recpos+=MI_DYN_ALIGN_SIZE);
02744       searching=1;
02745     }
02746   case COMPRESSED_RECORD:
02747   case BLOCK_RECORD:
02748     assert(0);                                  /* Impossible */
02749   }
02750   return(1);                               /* Impossible */
02751 }
02752 
02753 
02754 /*
02755   Write record to new file.
02756 
02757   SYNOPSIS
02758     sort_write_record()
02759       sort_param                Sort parameters.
02760 
02761   NOTE
02762     This is only called by a master thread if parallel repair is used.
02763 
02764   RETURN
02765     0           OK
02766     1           Error
02767 */
02768 
02769 int sort_write_record(MI_SORT_PARAM *sort_param)
02770 {
02771   int flag;
02772   ulong block_length,reclength;
02773   unsigned char *from;
02774   SORT_INFO *sort_info=sort_param->sort_info;
02775   MI_CHECK *param=sort_info->param;
02776   MI_INFO *info=sort_info->info;
02777   MYISAM_SHARE *share=info->s;
02778 
02779   if (sort_param->fix_datafile)
02780   {
02781     switch (sort_info->new_data_file_type) {
02782     case STATIC_RECORD:
02783       if (my_b_write(&info->rec_cache,sort_param->record,
02784          share->base.pack_reclength))
02785       {
02786   mi_check_print_error(param,"%d when writing to datafile",errno);
02787   return(1);
02788       }
02789       sort_param->filepos+=share->base.pack_reclength;
02790       info->s->state.split++;
02791       /* sort_info->param->glob_crc+=mi_static_checksum(info, sort_param->record); */
02792       break;
02793     case DYNAMIC_RECORD:
02794       if (! info->blobs)
02795   from=sort_param->rec_buff;
02796       else
02797       {
02798   /* must be sure that local buffer is big enough */
02799   reclength=info->s->base.pack_reclength+
02800     _my_calc_total_blob_length(info,sort_param->record)+
02801     ALIGN_SIZE(MI_MAX_DYN_BLOCK_HEADER)+MI_SPLIT_LENGTH+
02802     MI_DYN_DELETE_BLOCK_HEADER;
02803   if (sort_info->buff_length < reclength)
02804   {
02805           void *tmpptr= NULL;
02806     tmpptr= realloc(sort_info->buff, reclength);
02807           if(tmpptr)
02808           {
02809       sort_info->buff_length=reclength;
02810             sort_info->buff= (unsigned char *)tmpptr;
02811           }
02812           else
02813           {
02814             mi_check_print_error(param,"Could not realloc() sort_info->buff "
02815                                  " to %ul bytes", reclength);
02816             return(1);
02817           }
02818   }
02819   from= sort_info->buff+ALIGN_SIZE(MI_MAX_DYN_BLOCK_HEADER);
02820       }
02821       /* We can use info->checksum here as only one thread calls this. */
02822       info->checksum=mi_checksum(info,sort_param->record);
02823       reclength=_mi_rec_pack(info,from,sort_param->record);
02824       flag=0;
02825       /* sort_info->param->glob_crc+=info->checksum; */
02826 
02827       do
02828       {
02829   block_length=reclength+ 3 + test(reclength >= (65520-3));
02830   if (block_length < share->base.min_block_length)
02831     block_length=share->base.min_block_length;
02832   info->update|=HA_STATE_WRITE_AT_END;
02833   block_length=MY_ALIGN(block_length,MI_DYN_ALIGN_SIZE);
02834   if (block_length > MI_MAX_BLOCK_LENGTH)
02835     block_length=MI_MAX_BLOCK_LENGTH;
02836   if (_mi_write_part_record(info,0L,block_length,
02837           sort_param->filepos+block_length,
02838           &from,&reclength,&flag))
02839   {
02840     mi_check_print_error(param,"%d when writing to datafile",errno);
02841     return(1);
02842   }
02843   sort_param->filepos+=block_length;
02844   info->s->state.split++;
02845       } while (reclength);
02846       /* sort_info->param->glob_crc+=info->checksum; */
02847       break;
02848     case COMPRESSED_RECORD:
02849     case BLOCK_RECORD:
02850       assert(0);                                  /* Impossible */
02851     }
02852   }
02853   if (sort_param->master)
02854   {
02855     info->state->records++;
02856     if ((param->testflag & T_WRITE_LOOP) &&
02857         (info->state->records % WRITE_COUNT) == 0)
02858     {
02859       char llbuff[22];
02860       printf("%s\r", llstr(info->state->records,llbuff));
02861       fflush(stdout);
02862     }
02863   }
02864   return(0);
02865 } /* sort_write_record */
02866 
02867 
02868   /* Compare two keys from _create_index_by_sort */
02869 
02870 int sort_key_cmp(MI_SORT_PARAM *sort_param, const void *a, const void *b)
02871 {
02872   uint32_t not_used[2];
02873   return (ha_key_cmp(sort_param->seg, *((unsigned char* const *) a), *((unsigned char* const *) b),
02874          USE_WHOLE_KEY, SEARCH_SAME, not_used));
02875 } /* sort_key_cmp */
02876 
02877 
02878 int sort_key_write(MI_SORT_PARAM *sort_param, const void *a)
02879 {
02880   uint32_t diff_pos[2];
02881   char llbuff[22],llbuff2[22];
02882   SORT_INFO *sort_info=sort_param->sort_info;
02883   MI_CHECK *param= sort_info->param;
02884   int cmp;
02885 
02886   if (sort_info->key_block->inited)
02887   {
02888     cmp=ha_key_cmp(sort_param->seg,sort_info->key_block->lastkey,
02889        (unsigned char*) a, USE_WHOLE_KEY,SEARCH_FIND | SEARCH_UPDATE,
02890        diff_pos);
02891     if (param->stats_method == MI_STATS_METHOD_NULLS_NOT_EQUAL)
02892       ha_key_cmp(sort_param->seg,sort_info->key_block->lastkey,
02893                  (unsigned char*) a, USE_WHOLE_KEY,
02894                  SEARCH_FIND | SEARCH_NULL_ARE_NOT_EQUAL, diff_pos);
02895     else if (param->stats_method == MI_STATS_METHOD_IGNORE_NULLS)
02896     {
02897       diff_pos[0]= mi_collect_stats_nonulls_next(sort_param->seg,
02898                                                  sort_param->notnull,
02899                                                  sort_info->key_block->lastkey,
02900                                                  (unsigned char*)a);
02901     }
02902     sort_param->unique[diff_pos[0]-1]++;
02903   }
02904   else
02905   {
02906     cmp= -1;
02907     if (param->stats_method == MI_STATS_METHOD_IGNORE_NULLS)
02908       mi_collect_stats_nonulls_first(sort_param->seg, sort_param->notnull,
02909                                      (unsigned char*)a);
02910   }
02911   if ((sort_param->keyinfo->flag & HA_NOSAME) && cmp == 0)
02912   {
02913     sort_info->dupp++;
02914     sort_info->info->lastpos=get_record_for_key(sort_info->info,
02915             sort_param->keyinfo,
02916             (unsigned char*) a);
02917     mi_check_print_warning(param,
02918          "Duplicate key for record at %10s against record at %10s",
02919          llstr(sort_info->info->lastpos,llbuff),
02920          llstr(get_record_for_key(sort_info->info,
02921                 sort_param->keyinfo,
02922                 sort_info->key_block->
02923                 lastkey),
02924          llbuff2));
02925     param->testflag|=T_RETRY_WITHOUT_QUICK;
02926     return (sort_delete_record(sort_param));
02927   }
02928   return (sort_insert_key(sort_param,sort_info->key_block,
02929         (unsigned char*) a, HA_OFFSET_ERROR));
02930 } /* sort_key_write */
02931 
02932 
02933   /* get pointer to record from a key */
02934 
02935 my_off_t get_record_for_key(MI_INFO *info, MI_KEYDEF *keyinfo,
02936                             unsigned char *key) {
02937   return _mi_dpos(info,0,key+_mi_keylength(keyinfo,key));
02938 } /* get_record_for_key */
02939 
02940 
02941   /* Insert a key in sort-key-blocks */
02942 
02943 int sort_insert_key(MI_SORT_PARAM *sort_param,
02944                     register SORT_KEY_BLOCKS *key_block, unsigned char *key,
02945                     my_off_t prev_block)
02946 {
02947   uint32_t a_length,t_length,nod_flag;
02948   my_off_t filepos,key_file_length;
02949   unsigned char *anc_buff,*lastkey;
02950   MI_KEY_PARAM s_temp;
02951   MI_INFO *info;
02952   MI_KEYDEF *keyinfo=sort_param->keyinfo;
02953   SORT_INFO *sort_info= sort_param->sort_info;
02954   MI_CHECK *param=sort_info->param;
02955 
02956   anc_buff=key_block->buff;
02957   info=sort_info->info;
02958   lastkey=key_block->lastkey;
02959   nod_flag= (key_block == sort_info->key_block ? 0 :
02960        info->s->base.key_reflength);
02961 
02962   if (!key_block->inited)
02963   {
02964     key_block->inited=1;
02965     if (key_block == sort_info->key_block_end)
02966     {
02967       mi_check_print_error(param,"To many key-block-levels; Try increasing sort_key_blocks");
02968       return(1);
02969     }
02970     a_length=2+nod_flag;
02971     key_block->end_pos=anc_buff+2;
02972     lastkey=0;          /* No previous key in block */
02973   }
02974   else
02975     a_length=mi_getint(anc_buff);
02976 
02977   /* Save pointer to previous block */
02978   if (nod_flag)
02979     _mi_kpointer(info,key_block->end_pos,prev_block);
02980 
02981   t_length=(*keyinfo->pack_key)(keyinfo,nod_flag,
02982         (unsigned char*) 0,lastkey,lastkey,key,
02983          &s_temp);
02984   (*keyinfo->store_key)(keyinfo, key_block->end_pos+nod_flag,&s_temp);
02985   a_length+=t_length;
02986   mi_putint(anc_buff,a_length,nod_flag);
02987   key_block->end_pos+=t_length;
02988   if (a_length <= keyinfo->block_length)
02989   {
02990     _mi_move_key(keyinfo,key_block->lastkey,key);
02991     key_block->last_length=a_length-t_length;
02992     return(0);
02993   }
02994 
02995   /* Fill block with end-zero and write filled block */
02996   mi_putint(anc_buff,key_block->last_length,nod_flag);
02997   memset(anc_buff+key_block->last_length, 0,
02998          keyinfo->block_length - key_block->last_length);
02999   key_file_length=info->state->key_file_length;
03000   if ((filepos=_mi_new(info,keyinfo,DFLT_INIT_HITS)) == HA_OFFSET_ERROR)
03001     return(1);
03002 
03003   /* If we read the page from the key cache, we have to write it back to it */
03004   if (key_file_length == info->state->key_file_length)
03005   {
03006     if (_mi_write_keypage(info, keyinfo, filepos, DFLT_INIT_HITS, anc_buff))
03007       return(1);
03008   }
03009   else if (my_pwrite(info->s->kfile,(unsigned char*) anc_buff,
03010          (uint) keyinfo->block_length,filepos, param->myf_rw))
03011     return(1);
03012 
03013   /* Write separator-key to block in next level */
03014   if (sort_insert_key(sort_param,key_block+1,key_block->lastkey,filepos))
03015     return(1);
03016 
03017   /* clear old block and write new key in it */
03018   key_block->inited=0;
03019   return(sort_insert_key(sort_param, key_block,key,prev_block));
03020 } /* sort_insert_key */
03021 
03022 
03023   /* Delete record when we found a duplicated key */
03024 
03025 int sort_delete_record(MI_SORT_PARAM *sort_param)
03026 {
03027   uint32_t i;
03028   int old_file,error;
03029   unsigned char *key;
03030   SORT_INFO *sort_info=sort_param->sort_info;
03031   MI_CHECK *param=sort_info->param;
03032   MI_INFO *info=sort_info->info;
03033 
03034   if ((param->testflag & (T_FORCE_UNIQUENESS|T_QUICK)) == T_QUICK)
03035   {
03036     mi_check_print_error(param,
03037        "Quick-recover aborted; Run recovery without switch -q or with switch -qq");
03038     return(1);
03039   }
03040   if (info->s->options & HA_OPTION_COMPRESS_RECORD)
03041   {
03042     mi_check_print_error(param,
03043        "Recover aborted; Can't run standard recovery on compressed tables with errors in data-file. Use switch 'myisamchk --safe-recover' to fix it\n",stderr);;
03044     return(1);
03045   }
03046 
03047   old_file=info->dfile;
03048   info->dfile=info->rec_cache.file;
03049   if (sort_info->current_key)
03050   {
03051     key=info->lastkey+info->s->base.max_key_length;
03052     if ((error=(*info->s->read_rnd)(info,sort_param->record,info->lastpos,0)) &&
03053   error != HA_ERR_RECORD_DELETED)
03054     {
03055       mi_check_print_error(param,"Can't read record to be removed");
03056       info->dfile=old_file;
03057       return(1);
03058     }
03059 
03060     for (i=0 ; i < sort_info->current_key ; i++)
03061     {
03062       uint32_t key_length=_mi_make_key(info,i,key,sort_param->record,info->lastpos);
03063       if (_mi_ck_delete(info,i,key,key_length))
03064       {
03065   mi_check_print_error(param,"Can't delete key %d from record to be removed",i+1);
03066   info->dfile=old_file;
03067   return(1);
03068       }
03069     }
03070     if (sort_param->calc_checksum)
03071       param->glob_crc-=(*info->s->calc_checksum)(info, sort_param->record);
03072   }
03073   error=flush_io_cache(&info->rec_cache) || (*info->s->delete_record)(info);
03074   info->dfile=old_file;       /* restore actual value */
03075   info->state->records--;
03076   return(error);
03077 } /* sort_delete_record */
03078 
03079   /* Fix all pending blocks and flush everything to disk */
03080 
03081 int flush_pending_blocks(MI_SORT_PARAM *sort_param)
03082 {
03083   uint32_t nod_flag,length;
03084   my_off_t filepos,key_file_length;
03085   SORT_KEY_BLOCKS *key_block;
03086   SORT_INFO *sort_info= sort_param->sort_info;
03087   myf myf_rw=sort_info->param->myf_rw;
03088   MI_INFO *info=sort_info->info;
03089   MI_KEYDEF *keyinfo=sort_param->keyinfo;
03090 
03091   filepos= HA_OFFSET_ERROR;     /* if empty file */
03092   nod_flag=0;
03093   for (key_block=sort_info->key_block ; key_block->inited ; key_block++)
03094   {
03095     key_block->inited=0;
03096     length=mi_getint(key_block->buff);
03097     if (nod_flag)
03098       _mi_kpointer(info,key_block->end_pos,filepos);
03099     key_file_length=info->state->key_file_length;
03100     memset(key_block->buff+length, 0, keyinfo->block_length-length);
03101     if ((filepos=_mi_new(info,keyinfo,DFLT_INIT_HITS)) == HA_OFFSET_ERROR)
03102       return(1);
03103 
03104     /* If we read the page from the key cache, we have to write it back */
03105     if (key_file_length == info->state->key_file_length)
03106     {
03107       if (_mi_write_keypage(info, keyinfo, filepos,
03108                             DFLT_INIT_HITS, key_block->buff))
03109   return(1);
03110     }
03111     else if (my_pwrite(info->s->kfile,(unsigned char*) key_block->buff,
03112            (uint) keyinfo->block_length,filepos, myf_rw))
03113       return(1);
03114     nod_flag=1;
03115   }
03116   info->s->state.key_root[sort_param->key]=filepos; /* Last is root for tree */
03117   return(0);
03118 } /* flush_pending_blocks */
03119 
03120   /* alloc space and pointers for key_blocks */
03121 
03122 static SORT_KEY_BLOCKS *alloc_key_blocks(MI_CHECK *param, uint32_t blocks,
03123                                          uint32_t buffer_length)
03124 {
03125   register uint32_t i;
03126   SORT_KEY_BLOCKS *block;
03127 
03128   if (!(block=(SORT_KEY_BLOCKS*) malloc((sizeof(SORT_KEY_BLOCKS)+
03129                                         buffer_length+IO_SIZE)*blocks)))
03130   {
03131     mi_check_print_error(param,"Not enough memory for sort-key-blocks");
03132     return(0);
03133   }
03134   for (i=0 ; i < blocks ; i++)
03135   {
03136     block[i].inited=0;
03137     block[i].buff=(unsigned char*) (block+blocks)+(buffer_length+IO_SIZE)*i;
03138   }
03139   return(block);
03140 } /* alloc_key_blocks */
03141 
03142 
03143   /* Check if file is almost full */
03144 
03145 int test_if_almost_full(MI_INFO *info)
03146 {
03147   if (info->s->options & HA_OPTION_COMPRESS_RECORD)
03148     return 0;
03149   return (my_off_t)(lseek(info->s->kfile, 0L, SEEK_END) / 10 * 9) >
03150          (my_off_t) info->s->base.max_key_file_length ||
03151          (my_off_t)(lseek(info->dfile, 0L, SEEK_END) / 10 * 9) >
03152          (my_off_t) info->s->base.max_data_file_length;
03153 }
03154 
03155 
03156   /* write suffix to data file if neaded */
03157 
03158 int write_data_suffix(SORT_INFO *sort_info, bool fix_datafile)
03159 {
03160   MI_INFO *info=sort_info->info;
03161 
03162   if (info->s->options & HA_OPTION_COMPRESS_RECORD && fix_datafile)
03163   {
03164     unsigned char buff[MEMMAP_EXTRA_MARGIN];
03165     memset(buff, 0, sizeof(buff));
03166     if (my_b_write(&info->rec_cache,buff,sizeof(buff)))
03167     {
03168       mi_check_print_error(sort_info->param,
03169          "%d when writing to datafile",errno);
03170       return 1;
03171     }
03172     sort_info->param->read_cache.end_of_file+=sizeof(buff);
03173   }
03174   return 0;
03175 }
03176 
03177   /* Update state and myisamchk_time of indexfile */
03178 
03179 int update_state_info(MI_CHECK *param, MI_INFO *info,uint32_t update)
03180 {
03181   MYISAM_SHARE *share=info->s;
03182 
03183   if (update & UPDATE_OPEN_COUNT)
03184   {
03185     share->state.open_count=0;
03186     share->global_changed=0;
03187   }
03188   if (update & UPDATE_STAT)
03189   {
03190     uint32_t i, key_parts= mi_uint2korr(share->state.header.key_parts);
03191     share->state.rec_per_key_rows=info->state->records;
03192     share->state.changed&= ~STATE_NOT_ANALYZED;
03193     if (info->state->records)
03194     {
03195       for (i=0; i<key_parts; i++)
03196       {
03197         if (!(share->state.rec_per_key_part[i]=param->rec_per_key_part[i]))
03198           share->state.changed|= STATE_NOT_ANALYZED;
03199       }
03200     }
03201   }
03202   if (update & (UPDATE_STAT | UPDATE_SORT | UPDATE_TIME | UPDATE_AUTO_INC))
03203   {
03204     if (update & UPDATE_TIME)
03205     {
03206       share->state.check_time= (long) time((time_t*) 0);
03207       if (!share->state.create_time)
03208   share->state.create_time=share->state.check_time;
03209     }
03210     /*
03211       When tables are locked we haven't synched the share state and the
03212       real state for a while so we better do it here before synching
03213       the share state to disk. Only when table is write locked is it
03214       necessary to perform this synch.
03215     */
03216     if (info->lock_type == F_WRLCK)
03217       share->state.state= *info->state;
03218     if (mi_state_info_write(share->kfile,&share->state,1+2))
03219       goto err;
03220     share->changed=0;
03221   }
03222   {           /* Force update of status */
03223     int error;
03224     uint32_t r_locks=share->r_locks,w_locks=share->w_locks;
03225     share->r_locks= share->w_locks= share->tot_locks= 0;
03226     error=_mi_writeinfo(info,WRITEINFO_NO_UNLOCK);
03227     share->r_locks=r_locks;
03228     share->w_locks=w_locks;
03229     share->tot_locks=r_locks+w_locks;
03230     if (!error)
03231       return 0;
03232   }
03233 err:
03234   mi_check_print_error(param,"%d when updating keyfile",errno);
03235   return 1;
03236 }
03237 
03238   /*
03239     Update auto increment value for a table
03240     When setting the 'repair_only' flag we only want to change the
03241     old auto_increment value if its wrong (smaller than some given key).
03242     The reason is that we shouldn't change the auto_increment value
03243     for a table without good reason when only doing a repair; If the
03244     user have inserted and deleted rows, the auto_increment value
03245     may be bigger than the biggest current row and this is ok.
03246 
03247     If repair_only is not set, we will update the flag to the value in
03248     param->auto_increment is bigger than the biggest key.
03249   */
03250 
03251 void update_auto_increment_key(MI_CHECK *param, MI_INFO *info,
03252              bool repair_only)
03253 {
03254   unsigned char *record= 0;
03255 
03256   if (!info->s->base.auto_key ||
03257       ! mi_is_key_active(info->s->state.key_map, info->s->base.auto_key - 1))
03258   {
03259     if (!(param->testflag & T_VERY_SILENT))
03260       mi_check_print_info(param,
03261         "Table: %s doesn't have an auto increment key\n",
03262         param->isam_file_name);
03263     return;
03264   }
03265   if (!(param->testflag & T_SILENT) &&
03266       !(param->testflag & T_REP))
03267     printf("Updating MyISAM file: %s\n", param->isam_file_name);
03268   /*
03269     We have to use an allocated buffer instead of info->rec_buff as
03270     _mi_put_key_in_record() may use info->rec_buff
03271   */
03272   if (!mi_alloc_rec_buff(info, SIZE_MAX, &record))
03273   {
03274     mi_check_print_error(param,"Not enough memory for extra record");
03275     return;
03276   }
03277 
03278   mi_extra(info,HA_EXTRA_KEYREAD,0);
03279   if (mi_rlast(info, record, info->s->base.auto_key-1))
03280   {
03281     if (errno != HA_ERR_END_OF_FILE)
03282     {
03283       mi_extra(info,HA_EXTRA_NO_KEYREAD,0);
03284       free(mi_get_rec_buff_ptr(info, record));
03285       mi_check_print_error(param,"%d when reading last record",errno);
03286       return;
03287     }
03288     if (!repair_only)
03289       info->s->state.auto_increment=param->auto_increment_value;
03290   }
03291   else
03292   {
03293     uint64_t auto_increment= retrieve_auto_increment(info, record);
03294     set_if_bigger(info->s->state.auto_increment,auto_increment);
03295     if (!repair_only)
03296       set_if_bigger(info->s->state.auto_increment, param->auto_increment_value);
03297   }
03298   mi_extra(info,HA_EXTRA_NO_KEYREAD,0);
03299   free(mi_get_rec_buff_ptr(info, record));
03300   update_state_info(param, info, UPDATE_AUTO_INC);
03301   return;
03302 }
03303 
03304 
03305 /*
03306   Update statistics for each part of an index
03307 
03308   SYNOPSIS
03309     update_key_parts()
03310       keyinfo           IN  Index information (only key->keysegs used)
03311       rec_per_key_part  OUT Store statistics here
03312       unique            IN  Array of (#distinct tuples)
03313       notnull_tuples    IN  Array of (#tuples), or NULL
03314       records               Number of records in the table
03315 
03316   DESCRIPTION
03317     This function is called produce index statistics values from unique and
03318     notnull_tuples arrays after these arrays were produced with sequential
03319     index scan (the scan is done in two places: chk_index() and
03320     sort_key_write()).
03321 
03322     This function handles all 3 index statistics collection methods.
03323 
03324     Unique is an array:
03325       unique[0]= (#different values of {keypart1}) - 1
03326       unique[1]= (#different values of {keypart1,keypart2} tuple)-unique[0]-1
03327       ...
03328 
03329     For MI_STATS_METHOD_IGNORE_NULLS method, notnull_tuples is an array too:
03330       notnull_tuples[0]= (#of {keypart1} tuples such that keypart1 is not NULL)
03331       notnull_tuples[1]= (#of {keypart1,keypart2} tuples such that all
03332                           keypart{i} are not NULL)
03333       ...
03334     For all other statistics collection methods notnull_tuples==NULL.
03335 
03336     Output is an array:
03337     rec_per_key_part[k] =
03338      = E(#records in the table such that keypart_1=c_1 AND ... AND
03339          keypart_k=c_k for arbitrary constants c_1 ... c_k)
03340 
03341      = {assuming that values have uniform distribution and index contains all
03342         tuples from the domain (or that {c_1, ..., c_k} tuple is choosen from
03343         index tuples}
03344 
03345      = #tuples-in-the-index / #distinct-tuples-in-the-index.
03346 
03347     The #tuples-in-the-index and #distinct-tuples-in-the-index have different
03348     meaning depending on which statistics collection method is used:
03349 
03350     MI_STATS_METHOD_*  how are nulls compared?  which tuples are counted?
03351      NULLS_EQUAL            NULL == NULL           all tuples in table
03352      NULLS_NOT_EQUAL        NULL != NULL           all tuples in table
03353      IGNORE_NULLS               n/a             tuples that don't have NULLs
03354 */
03355 
03356 void update_key_parts(MI_KEYDEF *keyinfo, ulong *rec_per_key_part,
03357                       uint64_t *unique, uint64_t *notnull,
03358                       uint64_t records)
03359 {
03360   uint64_t count=0,tmp, unique_tuples;
03361   uint64_t tuples= records;
03362   uint32_t parts;
03363   for (parts=0 ; parts < keyinfo->keysegs  ; parts++)
03364   {
03365     count+=unique[parts];
03366     unique_tuples= count + 1;
03367     if (notnull)
03368     {
03369       tuples= notnull[parts];
03370       /*
03371         #(unique_tuples not counting tuples with NULLs) =
03372           #(unique_tuples counting tuples with NULLs as different) -
03373           #(tuples with NULLs)
03374       */
03375       unique_tuples -= (records - notnull[parts]);
03376     }
03377 
03378     if (unique_tuples == 0)
03379       tmp= 1;
03380     else if (count == 0)
03381       tmp= tuples; /* 1 unique tuple */
03382     else
03383       tmp= (tuples + unique_tuples/2) / unique_tuples;
03384 
03385     /*
03386       for some weird keys (e.g. FULLTEXT) tmp can be <1 here.
03387       let's ensure it is not
03388     */
03389     if (tmp < 1)
03390       tmp= 1;
03391     if (tmp >= (uint64_t) ~(ulong) 0)
03392       tmp=(uint64_t) ~(ulong) 0;
03393 
03394     *rec_per_key_part=(ulong) tmp;
03395     rec_per_key_part++;
03396   }
03397 }
03398 
03399 
03400 static ha_checksum mi_byte_checksum(const unsigned char *buf, uint32_t length)
03401 {
03402   ha_checksum crc;
03403   const unsigned char *end=buf+length;
03404   for (crc=0; buf != end; buf++)
03405     crc=((crc << 1) + *((unsigned char*) buf)) +
03406       test(crc & (((ha_checksum) 1) << (8*sizeof(ha_checksum)-1)));
03407   return crc;
03408 }
03409 
03410 static bool mi_too_big_key_for_sort(MI_KEYDEF *key, ha_rows rows)
03411 {
03412   uint32_t key_maxlength=key->maxlength;
03413   return (key->flag & (HA_BINARY_PACK_KEY | HA_VAR_LENGTH_KEY) &&
03414     ((uint64_t) rows * key_maxlength >
03415      (uint64_t) MAX_FILE_SIZE));
03416 }
03417 
03418 /*
03419   Deactivate all not unique index that can be recreated fast
03420   These include packed keys on which sorting will use more temporary
03421   space than the max allowed file length or for which the unpacked keys
03422   will take much more space than packed keys.
03423   Note that 'rows' may be zero for the case when we don't know how many
03424   rows we will put into the file.
03425  */
03426 
03427 void mi_disable_non_unique_index(MI_INFO *info, ha_rows rows)
03428 {
03429   MYISAM_SHARE *share=info->s;
03430   MI_KEYDEF    *key=share->keyinfo;
03431   uint32_t          i;
03432 
03433   assert(info->state->records == 0 &&
03434               (!rows || rows >= MI_MIN_ROWS_TO_DISABLE_INDEXES));
03435   for (i=0 ; i < share->base.keys ; i++,key++)
03436   {
03437     if (!(key->flag & (HA_NOSAME | HA_AUTO_KEY)) &&
03438         ! mi_too_big_key_for_sort(key,rows) && info->s->base.auto_key != i+1)
03439     {
03440       mi_clear_key_active(share->state.key_map, i);
03441       info->update|= HA_STATE_CHANGED;
03442     }
03443   }
03444 }
03445 
03446 
03447 /*
03448   Return true if we can use repair by sorting
03449   One can set the force argument to force to use sorting
03450   even if the temporary file would be quite big!
03451 */
03452 
03453 bool mi_test_if_sort_rep(MI_INFO *info, ha_rows rows,
03454           uint64_t key_map, bool force)
03455 {
03456   MYISAM_SHARE *share=info->s;
03457   MI_KEYDEF *key=share->keyinfo;
03458   uint32_t i;
03459 
03460   /*
03461     mi_repair_by_sort only works if we have at least one key. If we don't
03462     have any keys, we should use the normal repair.
03463   */
03464   if (! mi_is_any_key_active(key_map))
03465     return false;       /* Can't use sort */
03466   for (i=0 ; i < share->base.keys ; i++,key++)
03467   {
03468     if (!force && mi_too_big_key_for_sort(key,rows))
03469       return false;
03470   }
03471   return true;
03472 }
03473 
03474 
03475 static void
03476 set_data_file_type(SORT_INFO *sort_info, MYISAM_SHARE *share)
03477 {
03478   if ((sort_info->new_data_file_type=share->data_file_type) ==
03479       COMPRESSED_RECORD && sort_info->param->testflag & T_UNPACK)
03480   {
03481     MYISAM_SHARE tmp;
03482 
03483     if (share->options & HA_OPTION_PACK_RECORD)
03484       sort_info->new_data_file_type = DYNAMIC_RECORD;
03485     else
03486       sort_info->new_data_file_type = STATIC_RECORD;
03487 
03488     /* Set delete_function for sort_delete_record() */
03489     memcpy(&tmp, share, sizeof(*share));
03490     tmp.options= ~HA_OPTION_COMPRESS_RECORD;
03491     mi_setup_functions(&tmp);
03492     share->delete_record=tmp.delete_record;
03493   }
03494 }