00001
00002
00003
00004
00005
00006
00007
00008
00009
00010
00011
00012
00013
00014
00015
00016
00017
00018
00019
00020
00021
00022
00023
00024
00025
00026
00027
00028
00029
00030
00031
00032
00033
00034
00035
00036
00037
00038
00039
00040
00041
00042
00043 #include "myisam_priv.h"
00044 #include <drizzled/internal/m_string.h>
00045 #include <stdarg.h>
00046 #ifdef HAVE_SYS_VADVISE_H
00047 #include <sys/vadvise.h>
00048 #endif
00049 #ifdef HAVE_SYS_TYPES
00050 #include <sys/types.h>
00051 #endif
00052 #ifdef HAVE_SYS_MMAN_H
00053 #include <sys/mman.h>
00054 #endif
00055 #include <drizzled/util/test.h>
00056 #include <drizzled/error.h>
00057
00058 #include <algorithm>
00059
00060 using namespace std;
00061 using namespace drizzled;
00062 using namespace drizzled::internal;
00063
00064
00065 #define my_off_t2double(A) ((double) (my_off_t) (A))
00066
00067
00068
00069 static int check_k_link(MI_CHECK *param, MI_INFO *info,uint32_t nr);
00070 static int chk_index(MI_CHECK *param, MI_INFO *info,MI_KEYDEF *keyinfo,
00071 my_off_t page, unsigned char *buff, ha_rows *keys,
00072 ha_checksum *key_checksum, uint32_t level);
00073 static uint32_t isam_key_length(MI_INFO *info,MI_KEYDEF *keyinfo);
00074 static ha_checksum calc_checksum(ha_rows count);
00075 static int writekeys(MI_SORT_PARAM *sort_param);
00076 static int sort_one_index(MI_CHECK *param, MI_INFO *info,MI_KEYDEF *keyinfo,
00077 my_off_t pagepos, int new_file);
00078 int sort_key_read(MI_SORT_PARAM *sort_param,void *key);
00079 int sort_get_next_record(MI_SORT_PARAM *sort_param);
00080 int sort_key_cmp(MI_SORT_PARAM *sort_param, const void *a,const void *b);
00081 int sort_key_write(MI_SORT_PARAM *sort_param, const void *a);
00082 my_off_t get_record_for_key(MI_INFO *info,MI_KEYDEF *keyinfo,
00083 unsigned char *key);
00084 int sort_insert_key(MI_SORT_PARAM *sort_param,
00085 register SORT_KEY_BLOCKS *key_block,
00086 unsigned char *key, my_off_t prev_block);
00087 int sort_delete_record(MI_SORT_PARAM *sort_param);
00088
00089
00090 static SORT_KEY_BLOCKS *alloc_key_blocks(MI_CHECK *param, uint32_t blocks,
00091 uint32_t buffer_length);
00092 static ha_checksum mi_byte_checksum(const unsigned char *buf, uint32_t length);
00093 static void set_data_file_type(SORT_INFO *sort_info, MYISAM_SHARE *share);
00094
00095 void myisamchk_init(MI_CHECK *param)
00096 {
00097 memset(param, 0, sizeof(*param));
00098 param->opt_follow_links=1;
00099 param->keys_in_use= ~(uint64_t) 0;
00100 param->search_after_block=HA_OFFSET_ERROR;
00101 param->auto_increment_value= 0;
00102 param->use_buffers=USE_BUFFER_INIT;
00103 param->read_buffer_length=READ_BUFFER_INIT;
00104 param->write_buffer_length=READ_BUFFER_INIT;
00105 param->sort_buffer_length=SORT_BUFFER_INIT;
00106 param->sort_key_blocks=BUFFERS_WHEN_SORTING;
00107 param->tmpfile_createflag=O_RDWR | O_TRUNC | O_EXCL;
00108 param->myf_rw=MYF(MY_NABP | MY_WME | MY_WAIT_IF_FULL);
00109 param->start_check_pos=0;
00110 param->max_record_length= INT64_MAX;
00111 param->key_cache_block_size= KEY_CACHE_BLOCK_SIZE;
00112 param->stats_method= MI_STATS_METHOD_NULLS_NOT_EQUAL;
00113 }
00114
00115
00116
00117 int chk_status(MI_CHECK *param, register MI_INFO *info)
00118 {
00119 MYISAM_SHARE *share=info->s;
00120
00121 if (mi_is_crashed_on_repair(info))
00122 mi_check_print_warning(param,
00123 "Table is marked as crashed and last repair failed");
00124 else if (mi_is_crashed(info))
00125 mi_check_print_warning(param,
00126 "Table is marked as crashed");
00127 if (share->state.open_count != (uint) (info->s->global_changed ? 1 : 0))
00128 {
00129
00130 uint32_t save=param->warning_printed;
00131 mi_check_print_warning(param,
00132 share->state.open_count==1 ?
00133 "%d client is using or hasn't closed the table properly" :
00134 "%d clients are using or haven't closed the table properly",
00135 share->state.open_count);
00136
00137 if (param->testflag & T_UPDATE_STATE)
00138 param->warning_printed=save;
00139 }
00140 return 0;
00141 }
00142
00143
00144
00145 int chk_del(MI_CHECK *param, register MI_INFO *info, uint32_t test_flag)
00146 {
00147 register ha_rows i;
00148 uint32_t delete_link_length;
00149 my_off_t empty, next_link, old_link= 0;
00150 char buff[22],buff2[22];
00151
00152 param->record_checksum=0;
00153 delete_link_length=((info->s->options & HA_OPTION_PACK_RECORD) ? 20 :
00154 info->s->rec_reflength+1);
00155
00156 if (!(test_flag & T_SILENT))
00157 puts("- check record delete-chain");
00158
00159 next_link=info->s->state.dellink;
00160 if (info->state->del == 0)
00161 {
00162 if (test_flag & T_VERBOSE)
00163 {
00164 puts("No recordlinks");
00165 }
00166 }
00167 else
00168 {
00169 if (test_flag & T_VERBOSE)
00170 printf("Recordlinks: ");
00171 empty=0;
00172 for (i= info->state->del ; i > 0L && next_link != HA_OFFSET_ERROR ; i--)
00173 {
00174 if (*killed_ptr(param))
00175 return(1);
00176 if (test_flag & T_VERBOSE)
00177 printf(" %9s",llstr(next_link,buff));
00178 if (next_link >= info->state->data_file_length)
00179 goto wrong;
00180 if (my_pread(info->dfile, (unsigned char*) buff,delete_link_length,
00181 next_link,MYF(MY_NABP)))
00182 {
00183 if (test_flag & T_VERBOSE) puts("");
00184 mi_check_print_error(param,"Can't read delete-link at filepos: %s",
00185 llstr(next_link,buff));
00186 return(1);
00187 }
00188 if (*buff != '\0')
00189 {
00190 if (test_flag & T_VERBOSE) puts("");
00191 mi_check_print_error(param,"Record at pos: %s is not remove-marked",
00192 llstr(next_link,buff));
00193 goto wrong;
00194 }
00195 if (info->s->options & HA_OPTION_PACK_RECORD)
00196 {
00197 my_off_t prev_link=mi_sizekorr(buff+12);
00198 if (empty && prev_link != old_link)
00199 {
00200 if (test_flag & T_VERBOSE) puts("");
00201 mi_check_print_error(param,"Deleted block at %s doesn't point back at previous delete link",llstr(next_link,buff2));
00202 goto wrong;
00203 }
00204 old_link=next_link;
00205 next_link=mi_sizekorr(buff+4);
00206 empty+=mi_uint3korr(buff+1);
00207 }
00208 else
00209 {
00210 param->record_checksum+=(ha_checksum) next_link;
00211 next_link=_mi_rec_pos(info->s,(unsigned char*) buff+1);
00212 empty+=info->s->base.pack_reclength;
00213 }
00214 }
00215 if (test_flag & T_VERBOSE)
00216 puts("\n");
00217 if (empty != info->state->empty)
00218 {
00219 mi_check_print_warning(param,
00220 "Found %s deleted space in delete link chain. Should be %s",
00221 llstr(empty,buff2),
00222 llstr(info->state->empty,buff));
00223 }
00224 if (next_link != HA_OFFSET_ERROR)
00225 {
00226 mi_check_print_error(param,
00227 "Found more than the expected %s deleted rows in delete link chain",
00228 llstr(info->state->del, buff));
00229 goto wrong;
00230 }
00231 if (i != 0)
00232 {
00233 mi_check_print_error(param,
00234 "Found %s deleted rows in delete link chain. Should be %s",
00235 llstr(info->state->del - i, buff2),
00236 llstr(info->state->del, buff));
00237 goto wrong;
00238 }
00239 }
00240 return(0);
00241
00242 wrong:
00243 param->testflag|=T_RETRY_WITHOUT_QUICK;
00244 if (test_flag & T_VERBOSE) puts("");
00245 mi_check_print_error(param,"record delete-link-chain corrupted");
00246 return(1);
00247 }
00248
00249
00250
00251
00252 static int check_k_link(MI_CHECK *param, register MI_INFO *info, uint32_t nr)
00253 {
00254 my_off_t next_link;
00255 uint32_t block_size=(nr+1)*MI_MIN_KEY_BLOCK_LENGTH;
00256 ha_rows records;
00257 char llbuff[21], llbuff2[21];
00258 unsigned char *buff;
00259
00260 if (param->testflag & T_VERBOSE)
00261 printf("block_size %4u:", block_size);
00262
00263 next_link=info->s->state.key_del[nr];
00264 records= (ha_rows) (info->state->key_file_length / block_size);
00265 while (next_link != HA_OFFSET_ERROR && records > 0)
00266 {
00267 if (*killed_ptr(param))
00268 return(1);
00269 if (param->testflag & T_VERBOSE)
00270 printf("%16s",llstr(next_link,llbuff));
00271
00272
00273 if (next_link + block_size > info->state->key_file_length)
00274 {
00275 mi_check_print_error(param, "Invalid key block position: %s "
00276 "key block size: %u file_length: %s",
00277 llstr(next_link, llbuff), block_size,
00278 llstr(info->state->key_file_length, llbuff2));
00279 return(1);
00280 }
00281
00282
00283 if (next_link & (MI_MIN_KEY_BLOCK_LENGTH - 1))
00284 {
00285 mi_check_print_error(param, "Mis-aligned key block: %s "
00286 "minimum key block length: %u",
00287 llstr(next_link, llbuff), MI_MIN_KEY_BLOCK_LENGTH);
00288 return(1);
00289 }
00290
00291
00292
00293
00294
00295
00296 if (!(buff=key_cache_read(info->s->getKeyCache(),
00297 info->s->kfile, next_link, DFLT_INIT_HITS,
00298 (unsigned char*) info->buff, MI_MIN_KEY_BLOCK_LENGTH,
00299 MI_MIN_KEY_BLOCK_LENGTH, 1)))
00300 {
00301 mi_check_print_error(param, "key cache read error for block: %s",
00302 llstr(next_link,llbuff));
00303 return(1);
00304 }
00305 next_link=mi_sizekorr(buff);
00306 records--;
00307 param->key_file_blocks+=block_size;
00308 }
00309 if (param->testflag & T_VERBOSE)
00310 {
00311 if (next_link != HA_OFFSET_ERROR)
00312 printf("%16s\n",llstr(next_link,llbuff));
00313 else
00314 puts("");
00315 }
00316 return (next_link != HA_OFFSET_ERROR);
00317 }
00318
00319
00320
00321
00322 int chk_size(MI_CHECK *param, register MI_INFO *info)
00323 {
00324 int error=0;
00325 register my_off_t skr,size;
00326 char buff[22],buff2[22];
00327
00328 if (!(param->testflag & T_SILENT)) puts("- check file-size");
00329
00330
00331 flush_key_blocks(info->s->getKeyCache(),
00332 info->s->kfile, FLUSH_FORCE_WRITE);
00333
00334 size= lseek(info->s->kfile, 0, SEEK_END);
00335 if ((skr=(my_off_t) info->state->key_file_length) != size)
00336 {
00337
00338 if (skr > size && mi_is_any_key_active(info->s->state.key_map))
00339 {
00340 error=1;
00341 mi_check_print_error(param,
00342 "Size of indexfile is: %-8s Should be: %s",
00343 llstr(size,buff), llstr(skr,buff2));
00344 }
00345 else
00346 mi_check_print_warning(param,
00347 "Size of indexfile is: %-8s Should be: %s",
00348 llstr(size,buff), llstr(skr,buff2));
00349 }
00350 if (!(param->testflag & T_VERY_SILENT) &&
00351 ! (info->s->options & HA_OPTION_COMPRESS_RECORD) &&
00352 uint64_t2double(info->state->key_file_length) >
00353 uint64_t2double(info->s->base.margin_key_file_length)*0.9)
00354 mi_check_print_warning(param,"Keyfile is almost full, %10s of %10s used",
00355 llstr(info->state->key_file_length,buff),
00356 llstr(info->s->base.max_key_file_length-1,buff));
00357
00358 size=lseek(info->dfile,0L,SEEK_END);
00359 skr=(my_off_t) info->state->data_file_length;
00360 if (info->s->options & HA_OPTION_COMPRESS_RECORD)
00361 skr+= MEMMAP_EXTRA_MARGIN;
00362 #ifdef USE_RELOC
00363 if (info->data_file_type == STATIC_RECORD &&
00364 skr < (my_off_t) info->s->base.reloc*info->s->base.min_pack_length)
00365 skr=(my_off_t) info->s->base.reloc*info->s->base.min_pack_length;
00366 #endif
00367 if (skr != size)
00368 {
00369 info->state->data_file_length=size;
00370 if (skr > size && skr != size + MEMMAP_EXTRA_MARGIN)
00371 {
00372 error=1;
00373 mi_check_print_error(param,"Size of datafile is: %-9s Should be: %s",
00374 llstr(size,buff), llstr(skr,buff2));
00375 param->testflag|=T_RETRY_WITHOUT_QUICK;
00376 }
00377 else
00378 {
00379 mi_check_print_warning(param,
00380 "Size of datafile is: %-9s Should be: %s",
00381 llstr(size,buff), llstr(skr,buff2));
00382 }
00383 }
00384 if (!(param->testflag & T_VERY_SILENT) &&
00385 !(info->s->options & HA_OPTION_COMPRESS_RECORD) &&
00386 uint64_t2double(info->state->data_file_length) >
00387 (uint64_t2double(info->s->base.max_data_file_length)*0.9))
00388 mi_check_print_warning(param, "Datafile is almost full, %10s of %10s used",
00389 llstr(info->state->data_file_length,buff),
00390 llstr(info->s->base.max_data_file_length-1,buff2));
00391 return(error);
00392 }
00393
00394
00395
00396
00397 int chk_key(MI_CHECK *param, register MI_INFO *info)
00398 {
00399 uint32_t key,found_keys=0,full_text_keys=0,result=0;
00400 ha_rows keys;
00401 ha_checksum old_record_checksum,init_checksum;
00402 my_off_t all_keydata,all_totaldata,key_totlength,length;
00403 ulong *rec_per_key_part;
00404 MYISAM_SHARE *share=info->s;
00405 MI_KEYDEF *keyinfo;
00406 char buff[22],buff2[22];
00407
00408 if (!(param->testflag & T_SILENT))
00409 puts("- check key delete-chain");
00410
00411 param->key_file_blocks=info->s->base.keystart;
00412 for (key=0 ; key < info->s->state.header.max_block_size_index ; key++)
00413 if (check_k_link(param,info,key))
00414 {
00415 if (param->testflag & T_VERBOSE) puts("");
00416 mi_check_print_error(param,"key delete-link-chain corrupted");
00417 return(-1);
00418 }
00419
00420 if (!(param->testflag & T_SILENT)) puts("- check index reference");
00421
00422 all_keydata=all_totaldata=key_totlength=0;
00423 old_record_checksum=0;
00424 init_checksum=param->record_checksum;
00425 if (!(share->options &
00426 (HA_OPTION_PACK_RECORD | HA_OPTION_COMPRESS_RECORD)))
00427 old_record_checksum=calc_checksum(info->state->records+info->state->del-1)*
00428 share->base.pack_reclength;
00429 rec_per_key_part= param->rec_per_key_part;
00430 for (key= 0,keyinfo= &share->keyinfo[0]; key < share->base.keys ;
00431 rec_per_key_part+=keyinfo->keysegs, key++, keyinfo++)
00432 {
00433 param->key_crc[key]=0;
00434 if (! mi_is_key_active(share->state.key_map, key))
00435 {
00436
00437 assert(rec_per_key_part >= param->rec_per_key_part);
00438 memcpy(rec_per_key_part,
00439 (share->state.rec_per_key_part +
00440 (rec_per_key_part - param->rec_per_key_part)),
00441 keyinfo->keysegs*sizeof(*rec_per_key_part));
00442 continue;
00443 }
00444 found_keys++;
00445
00446 param->record_checksum=init_checksum;
00447
00448 memset(¶m->unique_count, 0, sizeof(param->unique_count));
00449 memset(¶m->notnull_count, 0, sizeof(param->notnull_count));
00450
00451 if ((!(param->testflag & T_SILENT)))
00452 printf ("- check data record references index: %d\n",key+1);
00453 if (share->state.key_root[key] == HA_OFFSET_ERROR && (info->state->records == 0))
00454 goto do_stat;
00455 if (!_mi_fetch_keypage(info,keyinfo,share->state.key_root[key],
00456 DFLT_INIT_HITS,info->buff,0))
00457 {
00458 mi_check_print_error(param,"Can't read indexpage from filepos: %s",
00459 llstr(share->state.key_root[key],buff));
00460 if (!(param->testflag & T_INFO))
00461 return(-1);
00462 result= UINT32_MAX;
00463 continue;
00464 }
00465 param->key_file_blocks+=keyinfo->block_length;
00466 keys=0;
00467 param->keydata=param->totaldata=0;
00468 param->key_blocks=0;
00469 param->max_level=0;
00470 if (chk_index(param,info,keyinfo,share->state.key_root[key],info->buff,
00471 &keys, param->key_crc+key,1))
00472 return(-1);
00473 if(!(0))
00474 {
00475 if (keys != info->state->records)
00476 {
00477 mi_check_print_error(param,"Found %s keys of %s",llstr(keys,buff),
00478 llstr(info->state->records,buff2));
00479 if (!(param->testflag & T_INFO))
00480 return(-1);
00481 result= UINT32_MAX;
00482 continue;
00483 }
00484 if (found_keys - full_text_keys == 1 &&
00485 ((share->options &
00486 (HA_OPTION_PACK_RECORD | HA_OPTION_COMPRESS_RECORD)) ||
00487 (param->testflag & T_DONT_CHECK_CHECKSUM)))
00488 old_record_checksum=param->record_checksum;
00489 else if (old_record_checksum != param->record_checksum)
00490 {
00491 if (key)
00492 mi_check_print_error(param,"Key %u doesn't point at same records that key 1",
00493 key+1);
00494 else
00495 mi_check_print_error(param,"Key 1 doesn't point at all records");
00496 if (!(param->testflag & T_INFO))
00497 return(-1);
00498 result= UINT32_MAX;
00499 continue;
00500 }
00501 }
00502 if ((uint) share->base.auto_key -1 == key)
00503 {
00504
00505 uint64_t auto_increment;
00506 info->lastinx=key;
00507 _mi_read_key_record(info, 0L, info->rec_buff);
00508 auto_increment= retrieve_auto_increment(info, info->rec_buff);
00509 if (auto_increment > info->s->state.auto_increment)
00510 {
00511 mi_check_print_warning(param, "Auto-increment value: %s is smaller "
00512 "than max used value: %s",
00513 llstr(info->s->state.auto_increment,buff2),
00514 llstr(auto_increment, buff));
00515 }
00516 if (param->testflag & T_AUTO_INC)
00517 {
00518 set_if_bigger(info->s->state.auto_increment,
00519 auto_increment);
00520 set_if_bigger(info->s->state.auto_increment,
00521 param->auto_increment_value);
00522 }
00523
00524
00525 mi_extra(info,HA_EXTRA_KEYREAD,0);
00526 memset(info->lastkey, 0, keyinfo->seg->length);
00527 if (!mi_rkey(info, info->rec_buff, key, (const unsigned char*) info->lastkey,
00528 (key_part_map)1, HA_READ_KEY_EXACT))
00529 {
00530
00531 uint32_t save=param->warning_printed;
00532 mi_check_print_warning(param, "Found row where the auto_increment "
00533 "column has the value 0");
00534 param->warning_printed=save;
00535 }
00536 mi_extra(info,HA_EXTRA_NO_KEYREAD,0);
00537 }
00538
00539 length=(my_off_t) isam_key_length(info,keyinfo)*keys + param->key_blocks*2;
00540 if (param->testflag & T_INFO && param->totaldata != 0L && keys != 0L)
00541 printf("Key: %2d: Keyblocks used: %3d%% Packed: %4d%% Max levels: %2d\n",
00542 key+1,
00543 (int) (my_off_t2double(param->keydata)*100.0/my_off_t2double(param->totaldata)),
00544 (int) ((my_off_t2double(length) - my_off_t2double(param->keydata))*100.0/
00545 my_off_t2double(length)),
00546 param->max_level);
00547 all_keydata+=param->keydata; all_totaldata+=param->totaldata; key_totlength+=length;
00548
00549 do_stat:
00550 if (param->testflag & T_STATISTICS)
00551 update_key_parts(keyinfo, rec_per_key_part, param->unique_count,
00552 param->stats_method == MI_STATS_METHOD_IGNORE_NULLS?
00553 param->notnull_count: NULL,
00554 (uint64_t)info->state->records);
00555 }
00556 if (param->testflag & T_INFO)
00557 {
00558 if (all_totaldata != 0L && found_keys > 0)
00559 printf("Total: Keyblocks used: %3d%% Packed: %4d%%\n\n",
00560 (int) (my_off_t2double(all_keydata)*100.0/
00561 my_off_t2double(all_totaldata)),
00562 (int) ((my_off_t2double(key_totlength) -
00563 my_off_t2double(all_keydata))*100.0/
00564 my_off_t2double(key_totlength)));
00565 else if (all_totaldata != 0L && mi_is_any_key_active(share->state.key_map))
00566 puts("");
00567 }
00568 if (param->key_file_blocks != info->state->key_file_length &&
00569 param->keys_in_use != ~(uint64_t) 0)
00570 mi_check_print_warning(param, "Some data are unreferenced in keyfile");
00571 if (found_keys != full_text_keys)
00572 param->record_checksum=old_record_checksum-init_checksum;
00573 else
00574 param->record_checksum=0;
00575 return(result);
00576 }
00577
00578
00579 static int chk_index_down(MI_CHECK *param, MI_INFO *info, MI_KEYDEF *keyinfo,
00580 my_off_t page, unsigned char *buff, ha_rows *keys,
00581 ha_checksum *key_checksum, uint32_t level)
00582 {
00583 char llbuff[22],llbuff2[22];
00584
00585
00586 if (page + keyinfo->block_length > info->state->key_file_length)
00587 {
00588
00589 my_off_t max_length= lseek(info->s->kfile, 0, SEEK_END);
00590 mi_check_print_error(param, "Invalid key block position: %s "
00591 "key block size: %u file_length: %s",
00592 llstr(page, llbuff), keyinfo->block_length,
00593 llstr(info->state->key_file_length, llbuff2));
00594 if (page + keyinfo->block_length > max_length)
00595 goto err;
00596
00597 info->state->key_file_length= (max_length &
00598 ~ (my_off_t) (keyinfo->block_length - 1));
00599 }
00600
00601
00602 if (page & (MI_MIN_KEY_BLOCK_LENGTH - 1))
00603 {
00604 mi_check_print_error(param, "Mis-aligned key block: %s "
00605 "minimum key block length: %u",
00606 llstr(page, llbuff), MI_MIN_KEY_BLOCK_LENGTH);
00607 goto err;
00608 }
00609
00610 if (!_mi_fetch_keypage(info,keyinfo,page, DFLT_INIT_HITS,buff,0))
00611 {
00612 mi_check_print_error(param,"Can't read key from filepos: %s",
00613 llstr(page,llbuff));
00614 goto err;
00615 }
00616 param->key_file_blocks+=keyinfo->block_length;
00617 if (chk_index(param,info,keyinfo,page,buff,keys,key_checksum,level))
00618 goto err;
00619
00620 return(0);
00621
00622 err:
00623 return(1);
00624 }
00625
00626
00627
00628
00629
00630
00631
00632
00633
00634
00635
00636
00637
00638
00639
00640
00641
00642 static
00643 void mi_collect_stats_nonulls_first(HA_KEYSEG *keyseg, uint64_t *notnull,
00644 unsigned char *key)
00645 {
00646 uint32_t first_null, kp;
00647 first_null= ha_find_null(keyseg, key) - keyseg;
00648
00649
00650
00651
00652 for (kp= 0; kp < first_null; kp++)
00653 notnull[kp]++;
00654 }
00655
00656
00657
00658
00659
00660
00661
00662
00663
00664
00665
00666
00667
00668
00669
00670
00671
00672
00673
00674
00675
00676
00677
00678
00679
00680
00681
00682 static
00683 int mi_collect_stats_nonulls_next(HA_KEYSEG *keyseg, uint64_t *notnull,
00684 unsigned char *prev_key, unsigned char *last_key)
00685 {
00686 uint32_t diffs[2];
00687 uint32_t first_null_seg, kp;
00688 HA_KEYSEG *seg;
00689
00690
00691
00692
00693
00694
00695
00696
00697
00698 ha_key_cmp(keyseg, prev_key, last_key, USE_WHOLE_KEY,
00699 SEARCH_FIND | SEARCH_NULL_ARE_NOT_EQUAL, diffs);
00700 seg= keyseg + diffs[0] - 1;
00701
00702
00703 first_null_seg= ha_find_null(seg, last_key + diffs[1]) - keyseg;
00704 for (kp= 0; kp < first_null_seg; kp++)
00705 notnull[kp]++;
00706
00707
00708
00709
00710
00711
00712 return diffs[0];
00713 }
00714
00715
00716
00717
00718 static int chk_index(MI_CHECK *param, MI_INFO *info, MI_KEYDEF *keyinfo,
00719 my_off_t page, unsigned char *buff, ha_rows *keys,
00720 ha_checksum *key_checksum, uint32_t level)
00721 {
00722 int flag;
00723 uint32_t used_length,comp_flag,nod_flag,key_length=0;
00724 unsigned char key[HA_MAX_POSSIBLE_KEY_BUFF],*temp_buff,*keypos,*old_keypos,*endpos;
00725 my_off_t next_page,record;
00726 char llbuff[22];
00727 uint32_t diff_pos[2];
00728
00729 if (!(temp_buff=(unsigned char*) malloc(keyinfo->block_length)))
00730 {
00731 mi_check_print_error(param,"Not enough memory for keyblock");
00732 return(-1);
00733 }
00734
00735 if (keyinfo->flag & HA_NOSAME)
00736 comp_flag=SEARCH_FIND | SEARCH_UPDATE;
00737 else
00738 comp_flag=SEARCH_SAME;
00739 nod_flag=mi_test_if_nod(buff);
00740 used_length=mi_getint(buff);
00741 keypos=buff+2+nod_flag;
00742 endpos=buff+used_length;
00743
00744 param->keydata+=used_length; param->totaldata+=keyinfo->block_length;
00745 param->key_blocks++;
00746 if (level > param->max_level)
00747 param->max_level=level;
00748
00749 if (used_length > keyinfo->block_length)
00750 {
00751 mi_check_print_error(param,"Wrong pageinfo at page: %s",
00752 llstr(page,llbuff));
00753 goto err;
00754 }
00755 for ( ;; )
00756 {
00757 if (*killed_ptr(param))
00758 goto err;
00759 memcpy(info->lastkey,key,key_length);
00760 info->lastkey_length=key_length;
00761 if (nod_flag)
00762 {
00763 next_page=_mi_kpos(nod_flag,keypos);
00764 if (chk_index_down(param,info,keyinfo,next_page,
00765 temp_buff,keys,key_checksum,level+1))
00766 goto err;
00767 }
00768 old_keypos=keypos;
00769 if (keypos >= endpos ||
00770 (key_length=(*keyinfo->get_key)(keyinfo,nod_flag,&keypos,key)) == 0)
00771 break;
00772 assert(key_length <= sizeof(key));
00773 if (keypos > endpos)
00774 {
00775 mi_check_print_error(param,"Wrong key block length at page: %s",llstr(page,llbuff));
00776 goto err;
00777 }
00778 if ((*keys)++ &&
00779 (flag=ha_key_cmp(keyinfo->seg,info->lastkey,key,key_length,
00780 comp_flag, diff_pos)) >=0)
00781 {
00782 if (comp_flag & SEARCH_FIND && flag == 0)
00783 mi_check_print_error(param,"Found duplicated key at page %s",llstr(page,llbuff));
00784 else
00785 mi_check_print_error(param,"Key in wrong position at page %s",llstr(page,llbuff));
00786 goto err;
00787 }
00788 if (param->testflag & T_STATISTICS)
00789 {
00790 if (*keys != 1L)
00791 {
00792 if (param->stats_method == MI_STATS_METHOD_NULLS_NOT_EQUAL)
00793 ha_key_cmp(keyinfo->seg,info->lastkey,key,USE_WHOLE_KEY,
00794 SEARCH_FIND | SEARCH_NULL_ARE_NOT_EQUAL,
00795 diff_pos);
00796 else if (param->stats_method == MI_STATS_METHOD_IGNORE_NULLS)
00797 {
00798 diff_pos[0]= mi_collect_stats_nonulls_next(keyinfo->seg,
00799 param->notnull_count,
00800 info->lastkey, key);
00801 }
00802 param->unique_count[diff_pos[0]-1]++;
00803 }
00804 else
00805 {
00806 if (param->stats_method == MI_STATS_METHOD_IGNORE_NULLS)
00807 mi_collect_stats_nonulls_first(keyinfo->seg, param->notnull_count,
00808 key);
00809 }
00810 }
00811 (*key_checksum)+= mi_byte_checksum((unsigned char*) key,
00812 key_length- info->s->rec_reflength);
00813 record= _mi_dpos(info,0,key+key_length);
00814 if (record >= info->state->data_file_length)
00815 {
00816 mi_check_print_error(param,"Found key at page %s that points to record outside datafile",llstr(page,llbuff));
00817 goto err;
00818 }
00819 param->record_checksum+=(ha_checksum) record;
00820 }
00821 if (keypos != endpos)
00822 {
00823 mi_check_print_error(param,"Keyblock size at page %s is not correct. Block length: %d key length: %d",
00824 llstr(page,llbuff), used_length, (keypos - buff));
00825 goto err;
00826 }
00827 free(temp_buff);
00828 return(0);
00829 err:
00830 free(temp_buff);
00831 return(1);
00832 }
00833
00834
00835
00836
00837 static ha_checksum calc_checksum(ha_rows count)
00838 {
00839 uint64_t sum,a,b;
00840
00841 sum=0;
00842 a=count; b=count+1;
00843 if (a & 1)
00844 b>>=1;
00845 else
00846 a>>=1;
00847 while (b)
00848 {
00849 if (b & 1)
00850 sum+=a;
00851 a<<=1; b>>=1;
00852 }
00853 return((ha_checksum) sum);
00854 }
00855
00856
00857
00858
00859 static uint32_t isam_key_length(MI_INFO *info, register MI_KEYDEF *keyinfo)
00860 {
00861 uint32_t length;
00862 HA_KEYSEG *keyseg;
00863
00864 length= info->s->rec_reflength;
00865 for (keyseg=keyinfo->seg ; keyseg->type ; keyseg++)
00866 length+= keyseg->length;
00867
00868 return(length);
00869 }
00870
00871
00872
00873
00874 int chk_data_link(MI_CHECK *param, MI_INFO *info,int extend)
00875 {
00876 int error,got_error,flag;
00877 uint key, left_length= 0, b_type,field;
00878 ha_rows records, del_blocks;
00879 my_off_t used, empty, pos, splits, start_recpos= 0,
00880 del_length, link_used, start_block;
00881 unsigned char *record= NULL, *to= NULL;
00882 char llbuff[22],llbuff2[22],llbuff3[22];
00883 ha_checksum intern_record_checksum;
00884 ha_checksum key_checksum[HA_MAX_POSSIBLE_KEY];
00885 bool static_row_size;
00886 MI_KEYDEF *keyinfo;
00887 MI_BLOCK_INFO block_info;
00888
00889 if (!(param->testflag & T_SILENT))
00890 {
00891 if (extend)
00892 puts("- check records and index references");
00893 else
00894 puts("- check record links");
00895 }
00896
00897 if (!mi_alloc_rec_buff(info, SIZE_MAX, &record))
00898 {
00899 mi_check_print_error(param,"Not enough memory for record");
00900 return(-1);
00901 }
00902 records=del_blocks=0;
00903 used=link_used=splits=del_length=0;
00904 intern_record_checksum=param->glob_crc=0;
00905 got_error=error=0;
00906 empty=info->s->pack.header_length;
00907
00908
00909 static_row_size=1;
00910 if (info->s->data_file_type == COMPRESSED_RECORD)
00911 {
00912 for (field=0 ; field < info->s->base.fields ; field++)
00913 {
00914 if (info->s->rec[field].base_type == FIELD_BLOB ||
00915 info->s->rec[field].base_type == FIELD_VARCHAR)
00916 {
00917 static_row_size=0;
00918 break;
00919 }
00920 }
00921 }
00922
00923 pos=my_b_tell(¶m->read_cache);
00924 memset(key_checksum, 0, info->s->base.keys * sizeof(key_checksum[0]));
00925 while (pos < info->state->data_file_length)
00926 {
00927 if (*killed_ptr(param))
00928 goto err2;
00929 switch (info->s->data_file_type) {
00930 case STATIC_RECORD:
00931 if (my_b_read(¶m->read_cache,(unsigned char*) record,
00932 info->s->base.pack_reclength))
00933 goto err;
00934 start_recpos=pos;
00935 pos+=info->s->base.pack_reclength;
00936 splits++;
00937 if (*record == '\0')
00938 {
00939 del_blocks++;
00940 del_length+=info->s->base.pack_reclength;
00941 continue;
00942 }
00943 param->glob_crc+= mi_static_checksum(info,record);
00944 used+=info->s->base.pack_reclength;
00945 break;
00946 case DYNAMIC_RECORD:
00947 flag=block_info.second_read=0;
00948 block_info.next_filepos=pos;
00949 do
00950 {
00951 if (_mi_read_cache(¶m->read_cache,(unsigned char*) block_info.header,
00952 (start_block=block_info.next_filepos),
00953 sizeof(block_info.header),
00954 (flag ? 0 : READING_NEXT) | READING_HEADER))
00955 goto err;
00956 if (start_block & (MI_DYN_ALIGN_SIZE-1))
00957 {
00958 mi_check_print_error(param,"Wrong aligned block at %s",
00959 llstr(start_block,llbuff));
00960 goto err2;
00961 }
00962 b_type=_mi_get_block_info(&block_info,-1,start_block);
00963 if (b_type & (BLOCK_DELETED | BLOCK_ERROR | BLOCK_SYNC_ERROR |
00964 BLOCK_FATAL_ERROR))
00965 {
00966 if (b_type & BLOCK_SYNC_ERROR)
00967 {
00968 if (flag)
00969 {
00970 mi_check_print_error(param,"Unexpected byte: %d at link: %s",
00971 (int) block_info.header[0],
00972 llstr(start_block,llbuff));
00973 goto err2;
00974 }
00975 pos=block_info.filepos+block_info.block_len;
00976 goto next;
00977 }
00978 if (b_type & BLOCK_DELETED)
00979 {
00980 if (block_info.block_len < info->s->base.min_block_length)
00981 {
00982 mi_check_print_error(param,
00983 "Deleted block with impossible length %lu at %s",
00984 block_info.block_len,llstr(pos,llbuff));
00985 goto err2;
00986 }
00987 if ((block_info.next_filepos != HA_OFFSET_ERROR &&
00988 block_info.next_filepos >= info->state->data_file_length) ||
00989 (block_info.prev_filepos != HA_OFFSET_ERROR &&
00990 block_info.prev_filepos >= info->state->data_file_length))
00991 {
00992 mi_check_print_error(param,"Delete link points outside datafile at %s",
00993 llstr(pos,llbuff));
00994 goto err2;
00995 }
00996 del_blocks++;
00997 del_length+=block_info.block_len;
00998 pos=block_info.filepos+block_info.block_len;
00999 splits++;
01000 goto next;
01001 }
01002 mi_check_print_error(param,"Wrong bytesec: %d-%d-%d at linkstart: %s",
01003 block_info.header[0],block_info.header[1],
01004 block_info.header[2],
01005 llstr(start_block,llbuff));
01006 goto err2;
01007 }
01008 if (info->state->data_file_length < block_info.filepos+
01009 block_info.block_len)
01010 {
01011 mi_check_print_error(param,
01012 "Recordlink that points outside datafile at %s",
01013 llstr(pos,llbuff));
01014 got_error=1;
01015 break;
01016 }
01017 splits++;
01018 if (!flag++)
01019 {
01020 start_recpos=pos;
01021 pos=block_info.filepos+block_info.block_len;
01022 if (block_info.rec_len > (uint) info->s->base.max_pack_length)
01023 {
01024 mi_check_print_error(param,"Found too long record (%lu) at %s",
01025 (ulong) block_info.rec_len,
01026 llstr(start_recpos,llbuff));
01027 got_error=1;
01028 break;
01029 }
01030 if (info->s->base.blobs)
01031 {
01032 if (!(to= mi_alloc_rec_buff(info, block_info.rec_len,
01033 &info->rec_buff)))
01034 {
01035 mi_check_print_error(param,
01036 "Not enough memory (%lu) for blob at %s",
01037 (ulong) block_info.rec_len,
01038 llstr(start_recpos,llbuff));
01039 got_error=1;
01040 break;
01041 }
01042 }
01043 else
01044 to= info->rec_buff;
01045 left_length=block_info.rec_len;
01046 }
01047 if (left_length < block_info.data_len)
01048 {
01049 mi_check_print_error(param,"Found too long record (%lu) at %s",
01050 (ulong) block_info.data_len,
01051 llstr(start_recpos,llbuff));
01052 got_error=1;
01053 break;
01054 }
01055 if (_mi_read_cache(¶m->read_cache,(unsigned char*) to,block_info.filepos,
01056 (uint) block_info.data_len,
01057 flag == 1 ? READING_NEXT : 0))
01058 goto err;
01059 to+=block_info.data_len;
01060 link_used+= block_info.filepos-start_block;
01061 used+= block_info.filepos - start_block + block_info.data_len;
01062 empty+=block_info.block_len-block_info.data_len;
01063 left_length-=block_info.data_len;
01064 if (left_length)
01065 {
01066 if (b_type & BLOCK_LAST)
01067 {
01068 mi_check_print_error(param,
01069 "Wrong record length %s of %s at %s",
01070 llstr(block_info.rec_len-left_length,llbuff),
01071 llstr(block_info.rec_len, llbuff2),
01072 llstr(start_recpos,llbuff3));
01073 got_error=1;
01074 break;
01075 }
01076 if (info->state->data_file_length < block_info.next_filepos)
01077 {
01078 mi_check_print_error(param,
01079 "Found next-recordlink that points outside datafile at %s",
01080 llstr(block_info.filepos,llbuff));
01081 got_error=1;
01082 break;
01083 }
01084 }
01085 } while (left_length);
01086 if (! got_error)
01087 {
01088 if (_mi_rec_unpack(info,record,info->rec_buff,block_info.rec_len) ==
01089 MY_FILE_ERROR)
01090 {
01091 mi_check_print_error(param,"Found wrong record at %s",
01092 llstr(start_recpos,llbuff));
01093 got_error=1;
01094 }
01095 else
01096 {
01097 info->checksum=mi_checksum(info,record);
01098 if (param->testflag & (T_EXTEND | T_MEDIUM | T_VERBOSE))
01099 {
01100 if (_mi_rec_check(info,record, info->rec_buff,block_info.rec_len,
01101 test(info->s->calc_checksum)))
01102 {
01103 mi_check_print_error(param,"Found wrong packed record at %s",
01104 llstr(start_recpos,llbuff));
01105 got_error=1;
01106 }
01107 }
01108 if (!got_error)
01109 param->glob_crc+= info->checksum;
01110 }
01111 }
01112 else if (!flag)
01113 pos=block_info.filepos+block_info.block_len;
01114 break;
01115 case COMPRESSED_RECORD:
01116 case BLOCK_RECORD:
01117 assert(0);
01118 }
01119 if (! got_error)
01120 {
01121 intern_record_checksum+=(ha_checksum) start_recpos;
01122 records++;
01123 if (param->testflag & T_WRITE_LOOP && records % WRITE_COUNT == 0)
01124 {
01125 printf("%s\r", llstr(records,llbuff)); fflush(stdout);
01126 }
01127
01128
01129
01130 for (key=0,keyinfo= info->s->keyinfo; key < info->s->base.keys;
01131 key++,keyinfo++)
01132 {
01133 if (mi_is_key_active(info->s->state.key_map, key))
01134 {
01135 {
01136 uint32_t key_length=_mi_make_key(info,key,info->lastkey,record,
01137 start_recpos);
01138 if (extend)
01139 {
01140
01141
01142
01143 int search_result=
01144 _mi_search(info,keyinfo,info->lastkey,key_length,
01145 SEARCH_SAME, info->s->state.key_root[key]);
01146 if (search_result)
01147 {
01148 mi_check_print_error(param,"Record at: %10s "
01149 "Can't find key for index: %2d",
01150 llstr(start_recpos,llbuff),key+1);
01151 if (error++ > MAXERR || !(param->testflag & T_VERBOSE))
01152 goto err2;
01153 }
01154 }
01155 else
01156 key_checksum[key]+=mi_byte_checksum((unsigned char*) info->lastkey,
01157 key_length);
01158 }
01159 }
01160 }
01161 }
01162 else
01163 {
01164 got_error=0;
01165 if (error++ > MAXERR || !(param->testflag & T_VERBOSE))
01166 goto err2;
01167 }
01168 next:;
01169 }
01170 if (param->testflag & T_WRITE_LOOP)
01171 {
01172 fputs(" \r",stdout); fflush(stdout);
01173 }
01174 if (records != info->state->records)
01175 {
01176 mi_check_print_error(param,"Record-count is not ok; is %-10s Should be: %s",
01177 llstr(records,llbuff), llstr(info->state->records,llbuff2));
01178 error=1;
01179 }
01180 else if (param->record_checksum &&
01181 param->record_checksum != intern_record_checksum)
01182 {
01183 mi_check_print_error(param,
01184 "Keypointers and record positions doesn't match");
01185 error=1;
01186 }
01187 else if (param->glob_crc != info->state->checksum &&
01188 (info->s->options &
01189 (HA_OPTION_COMPRESS_RECORD)))
01190 {
01191 mi_check_print_warning(param,
01192 "Record checksum is not the same as checksum stored in the index file\n");
01193 error=1;
01194 }
01195 else if (!extend)
01196 {
01197 for (key=0 ; key < info->s->base.keys; key++)
01198 {
01199 if (key_checksum[key] != param->key_crc[key])
01200 {
01201 mi_check_print_error(param,"Checksum for key: %2d doesn't match checksum for records",
01202 key+1);
01203 error=1;
01204 }
01205 }
01206 }
01207
01208 if (del_length != info->state->empty)
01209 {
01210 mi_check_print_warning(param,
01211 "Found %s deleted space. Should be %s",
01212 llstr(del_length,llbuff2),
01213 llstr(info->state->empty,llbuff));
01214 }
01215 if (used+empty+del_length != info->state->data_file_length)
01216 {
01217 mi_check_print_warning(param,
01218 "Found %s record-data and %s unused data and %s deleted-data",
01219 llstr(used,llbuff),llstr(empty,llbuff2),
01220 llstr(del_length,llbuff3));
01221 mi_check_print_warning(param,
01222 "Total %s, Should be: %s",
01223 llstr((used+empty+del_length),llbuff),
01224 llstr(info->state->data_file_length,llbuff2));
01225 }
01226 if (del_blocks != info->state->del)
01227 {
01228 mi_check_print_warning(param,
01229 "Found %10s deleted blocks Should be: %s",
01230 llstr(del_blocks,llbuff),
01231 llstr(info->state->del,llbuff2));
01232 }
01233 if (splits != info->s->state.split)
01234 {
01235 mi_check_print_warning(param,
01236 "Found %10s parts Should be: %s parts",
01237 llstr(splits,llbuff),
01238 llstr(info->s->state.split,llbuff2));
01239 }
01240 if (param->testflag & T_INFO)
01241 {
01242 if (param->warning_printed || param->error_printed)
01243 puts("");
01244 if (used != 0 && ! param->error_printed)
01245 {
01246 printf("Records:%18s M.recordlength:%9lu Packed:%14.0f%%\n",
01247 llstr(records,llbuff), (long)((used-link_used)/records),
01248 (info->s->base.blobs ? 0.0 :
01249 (uint64_t2double((uint64_t) info->s->base.reclength*records)-
01250 my_off_t2double(used))/
01251 uint64_t2double((uint64_t) info->s->base.reclength*records)*100.0));
01252 printf("Recordspace used:%9.0f%% Empty space:%12d%% Blocks/Record: %6.2f\n",
01253 (uint64_t2double(used-link_used)/uint64_t2double(used-link_used+empty)*100.0),
01254 (!records ? 100 : (int) (uint64_t2double(del_length+empty)/
01255 my_off_t2double(used)*100.0)),
01256 uint64_t2double(splits - del_blocks) / records);
01257 }
01258 printf("Record blocks:%12s Delete blocks:%10s\n",
01259 llstr(splits-del_blocks,llbuff),llstr(del_blocks,llbuff2));
01260 printf("Record data: %12s Deleted data: %10s\n",
01261 llstr(used-link_used,llbuff),llstr(del_length,llbuff2));
01262 printf("Lost space: %12s Linkdata: %10s\n",
01263 llstr(empty,llbuff),llstr(link_used,llbuff2));
01264 }
01265 free(mi_get_rec_buff_ptr(info, record));
01266 return (error);
01267 err:
01268 mi_check_print_error(param,"got error: %d when reading datafile at record: %s",errno, llstr(records,llbuff));
01269 err2:
01270 free(mi_get_rec_buff_ptr(info, record));
01271 param->testflag|=T_RETRY_WITHOUT_QUICK;
01272 return(1);
01273 }
01274
01275
01329 static int mi_drop_all_indexes(MI_CHECK *param, MI_INFO *info, bool force)
01330 {
01331 MYISAM_SHARE *share= info->s;
01332 MI_STATE_INFO *state= &share->state;
01333 uint32_t i;
01334 int error;
01335
01336
01337
01338
01339
01340
01341
01342
01343 if (!force && (param->testflag & T_CREATE_MISSING_KEYS))
01344 {
01345 for (i= 0; i < share->base.keys; i++)
01346 {
01347 if ((state->key_root[i] != HA_OFFSET_ERROR) &&
01348 !mi_is_key_active(state->key_map, i))
01349 {
01350
01351
01352
01353
01354
01355 break;
01356 }
01357 }
01358 if (i >= share->base.keys)
01359 {
01360
01361
01362
01363
01364
01365 error= flush_key_blocks(share->getKeyCache(), share->kfile,
01366 FLUSH_FORCE_WRITE);
01367 goto end;
01368 }
01369
01370
01371
01372
01373
01374 mi_clear_all_keys_active(state->key_map);
01375 }
01376
01377
01378 if ((error= flush_key_blocks(share->getKeyCache(), share->kfile,
01379 FLUSH_IGNORE_CHANGED)))
01380 goto end;
01381
01382
01383 for (i= 0; i < share->base.keys; i++)
01384 state->key_root[i]= HA_OFFSET_ERROR;
01385
01386
01387 for (i= 0; i < state->header.max_block_size_index; i++)
01388 state->key_del[i]= HA_OFFSET_ERROR;
01389
01390
01391 info->state->key_file_length= share->base.keystart;
01392
01393
01394
01395 end:
01396 return(error);
01397 }
01398
01399
01400
01401
01402
01403 int mi_repair(MI_CHECK *param, register MI_INFO *info,
01404 char * name, int rep_quick)
01405 {
01406 int error,got_error;
01407 ha_rows start_records,new_header_length;
01408 my_off_t del;
01409 int new_file;
01410 MYISAM_SHARE *share=info->s;
01411 char llbuff[22],llbuff2[22];
01412 SORT_INFO sort_info;
01413 MI_SORT_PARAM sort_param;
01414
01415 memset(&sort_info, 0, sizeof(sort_info));
01416 memset(&sort_param, 0, sizeof(sort_param));
01417 start_records=info->state->records;
01418 new_header_length=(param->testflag & T_UNPACK) ? 0L :
01419 share->pack.header_length;
01420 got_error=1;
01421 new_file= -1;
01422 sort_param.sort_info=&sort_info;
01423
01424 if (!(param->testflag & T_SILENT))
01425 {
01426 printf("- recovering (with keycache) MyISAM-table '%s'\n",name);
01427 printf("Data records: %s\n", llstr(info->state->records,llbuff));
01428 }
01429 param->testflag|=T_REP;
01430
01431 if (info->s->options & (HA_OPTION_COMPRESS_RECORD))
01432 param->testflag|=T_CALC_CHECKSUM;
01433
01434 if (!param->using_global_keycache)
01435 assert(0);
01436
01437 if (param->read_cache.init_io_cache(info->dfile, (uint) param->read_buffer_length, READ_CACHE,share->pack.header_length,1,MYF(MY_WME)))
01438 {
01439 memset(&info->rec_cache, 0, sizeof(info->rec_cache));
01440 goto err;
01441 }
01442 if (not rep_quick)
01443 {
01444 if (info->rec_cache.init_io_cache(-1, (uint) param->write_buffer_length, WRITE_CACHE, new_header_length, 1, MYF(MY_WME | MY_WAIT_IF_FULL)))
01445 {
01446 goto err;
01447 }
01448 }
01449 info->opt_flag|=WRITE_CACHE_USED;
01450 if (!mi_alloc_rec_buff(info, SIZE_MAX, &sort_param.record) ||
01451 !mi_alloc_rec_buff(info, SIZE_MAX, &sort_param.rec_buff))
01452 {
01453 mi_check_print_error(param, "Not enough memory for extra record");
01454 goto err;
01455 }
01456
01457 if (!rep_quick)
01458 {
01459
01460 if ((new_file=my_create(internal::fn_format(param->temp_filename,
01461 share->data_file_name, "",
01462 DATA_TMP_EXT, 2+4),
01463 0,param->tmpfile_createflag,
01464 MYF(0))) < 0)
01465 {
01466 mi_check_print_error(param,"Can't create new tempfile: '%s'",
01467 param->temp_filename);
01468 goto err;
01469 }
01470 if (new_header_length &&
01471 filecopy(param,new_file,info->dfile,0L,new_header_length,
01472 "datafile-header"))
01473 goto err;
01474 info->s->state.dellink= HA_OFFSET_ERROR;
01475 info->rec_cache.file=new_file;
01476 if (param->testflag & T_UNPACK)
01477 {
01478 share->options&= ~HA_OPTION_COMPRESS_RECORD;
01479 mi_int2store(share->state.header.options,share->options);
01480 }
01481 }
01482 sort_info.info=info;
01483 sort_info.param = param;
01484 sort_param.read_cache=param->read_cache;
01485 sort_param.pos=sort_param.max_pos=share->pack.header_length;
01486 sort_param.filepos=new_header_length;
01487 param->read_cache.end_of_file=sort_info.filelength=
01488 lseek(info->dfile,0L,SEEK_END);
01489 sort_info.dupp=0;
01490 sort_param.fix_datafile= (bool) (! rep_quick);
01491 sort_param.master=1;
01492 sort_info.max_records= ~(ha_rows) 0;
01493
01494 set_data_file_type(&sort_info, share);
01495 del=info->state->del;
01496 info->state->records=info->state->del=share->state.split=0;
01497 info->state->empty=0;
01498 param->glob_crc=0;
01499 if (param->testflag & T_CALC_CHECKSUM)
01500 sort_param.calc_checksum= 1;
01501
01502 info->update= (short) (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED);
01503
01504
01505 if (param->testflag & T_CREATE_MISSING_KEYS)
01506 mi_set_all_keys_active(share->state.key_map, share->base.keys);
01507 mi_drop_all_indexes(param, info, true);
01508
01509 lock_memory(param);
01510
01511
01512 while (!(error=sort_get_next_record(&sort_param)))
01513 {
01514 if (writekeys(&sort_param))
01515 {
01516 if (errno != HA_ERR_FOUND_DUPP_KEY)
01517 goto err;
01518 mi_check_print_info(param,"Duplicate key %2d for record at %10s against new record at %10s",
01519 info->errkey+1,
01520 llstr(sort_param.start_recpos,llbuff),
01521 llstr(info->dupp_key_pos,llbuff2));
01522 if (param->testflag & T_VERBOSE)
01523 {
01524 _mi_make_key(info,(uint) info->errkey,info->lastkey,
01525 sort_param.record,0L);
01526 }
01527 sort_info.dupp++;
01528 if ((param->testflag & (T_FORCE_UNIQUENESS|T_QUICK)) == T_QUICK)
01529 {
01530 param->testflag|=T_RETRY_WITHOUT_QUICK;
01531 param->error_printed=1;
01532 goto err;
01533 }
01534 continue;
01535 }
01536 if (sort_write_record(&sort_param))
01537 goto err;
01538 }
01539 if (error > 0 || write_data_suffix(&sort_info, (bool)!rep_quick) ||
01540 flush_io_cache(&info->rec_cache) || param->read_cache.error < 0)
01541 goto err;
01542
01543 if (param->testflag & T_WRITE_LOOP)
01544 {
01545 fputs(" \r",stdout); fflush(stdout);
01546 }
01547 if (ftruncate(share->kfile, info->state->key_file_length))
01548 {
01549 mi_check_print_warning(param,
01550 "Can't change size of indexfile, error: %d",
01551 errno);
01552 goto err;
01553 }
01554
01555 if (rep_quick && del+sort_info.dupp != info->state->del)
01556 {
01557 mi_check_print_error(param,"Couldn't fix table with quick recovery: Found wrong number of deleted records");
01558 mi_check_print_error(param,"Run recovery again without -q");
01559 got_error=1;
01560 param->retry_repair=1;
01561 param->testflag|=T_RETRY_WITHOUT_QUICK;
01562 goto err;
01563 }
01564 if (param->testflag & T_SAFE_REPAIR)
01565 {
01566
01567 if (info->state->records+1 < start_records)
01568 {
01569 info->state->records=start_records;
01570 got_error=1;
01571 goto err;
01572 }
01573 }
01574
01575 if (!rep_quick)
01576 {
01577 internal::my_close(info->dfile,MYF(0));
01578 info->dfile=new_file;
01579 info->state->data_file_length=sort_param.filepos;
01580 share->state.version=(ulong) time((time_t*) 0);
01581 }
01582 else
01583 {
01584 info->state->data_file_length=sort_param.max_pos;
01585 }
01586 if (param->testflag & T_CALC_CHECKSUM)
01587 info->state->checksum=param->glob_crc;
01588
01589 if (!(param->testflag & T_SILENT))
01590 {
01591 if (start_records != info->state->records)
01592 printf("Data records: %s\n", llstr(info->state->records,llbuff));
01593 if (sort_info.dupp)
01594 mi_check_print_warning(param,
01595 "%s records have been removed",
01596 llstr(sort_info.dupp,llbuff));
01597 }
01598
01599 got_error=0;
01600
01601 if (&share->state.state != info->state)
01602 memcpy( &share->state.state, info->state, sizeof(*info->state));
01603
01604 err:
01605 if (!got_error)
01606 {
01607
01608 if (new_file >= 0)
01609 {
01610 internal::my_close(new_file,MYF(0));
01611 info->dfile=new_file= -1;
01612 if (change_to_newfile(share->data_file_name,MI_NAME_DEXT,
01613 DATA_TMP_EXT, share->base.raid_chunks,
01614 (param->testflag & T_BACKUP_DATA ?
01615 MYF(MY_REDEL_MAKE_BACKUP): MYF(0))) ||
01616 mi_open_datafile(info,share,-1))
01617 got_error=1;
01618 }
01619 }
01620 if (got_error)
01621 {
01622 if (! param->error_printed)
01623 mi_check_print_error(param,"%d for record at pos %s",errno,
01624 llstr(sort_param.start_recpos,llbuff));
01625 if (new_file >= 0)
01626 {
01627 internal::my_close(new_file,MYF(0));
01628 my_delete(param->temp_filename, MYF(MY_WME));
01629 info->rec_cache.file=-1;
01630 }
01631 mi_mark_crashed_on_repair(info);
01632 }
01633
01634 void * rec_buff_ptr= NULL;
01635 rec_buff_ptr= mi_get_rec_buff_ptr(info, sort_param.rec_buff);
01636 if (rec_buff_ptr != NULL)
01637 free(rec_buff_ptr);
01638 rec_buff_ptr= mi_get_rec_buff_ptr(info, sort_param.record);
01639 if (rec_buff_ptr != NULL)
01640 free(rec_buff_ptr);
01641 rec_buff_ptr= NULL;
01642
01643 free(sort_info.buff);
01644 param->read_cache.end_io_cache();
01645 info->opt_flag&= ~(READ_CACHE_USED | WRITE_CACHE_USED);
01646 info->rec_cache.end_io_cache();
01647 got_error|= flush_blocks(param, share->getKeyCache(), share->kfile);
01648 if (not got_error && param->testflag & T_UNPACK)
01649 {
01650 share->state.header.options[0]&= (unsigned char) ~HA_OPTION_COMPRESS_RECORD;
01651 share->pack.header_length=0;
01652 share->data_file_type=sort_info.new_data_file_type;
01653 }
01654 share->state.changed|= (STATE_NOT_OPTIMIZED_KEYS | STATE_NOT_SORTED_PAGES |
01655 STATE_NOT_ANALYZED);
01656 return(got_error);
01657 }
01658
01659
01660
01661
01662 static int writekeys(MI_SORT_PARAM *sort_param)
01663 {
01664 register uint32_t i;
01665 unsigned char *key;
01666 MI_INFO *info= sort_param->sort_info->info;
01667 unsigned char *buff= sort_param->record;
01668 my_off_t filepos= sort_param->filepos;
01669
01670 key=info->lastkey+info->s->base.max_key_length;
01671 for (i=0 ; i < info->s->base.keys ; i++)
01672 {
01673 if (mi_is_key_active(info->s->state.key_map, i))
01674 {
01675 {
01676 uint32_t key_length=_mi_make_key(info,i,key,buff,filepos);
01677 if (_mi_ck_write(info,i,key,key_length))
01678 goto err;
01679 }
01680 }
01681 }
01682 return(0);
01683
01684 err:
01685 if (errno == HA_ERR_FOUND_DUPP_KEY)
01686 {
01687 info->errkey=(int) i;
01688 while ( i-- > 0 )
01689 {
01690 if (mi_is_key_active(info->s->state.key_map, i))
01691 {
01692 {
01693 uint32_t key_length=_mi_make_key(info,i,key,buff,filepos);
01694 if (_mi_ck_delete(info,i,key,key_length))
01695 break;
01696 }
01697 }
01698 }
01699 }
01700
01701 if (sort_param->calc_checksum)
01702 sort_param->sort_info->param->glob_crc-= info->checksum;
01703 return(-1);
01704 }
01705
01706
01707
01708
01709 int movepoint(register MI_INFO *info, unsigned char *record, my_off_t oldpos,
01710 my_off_t newpos, uint32_t prot_key)
01711 {
01712 register uint32_t i;
01713 unsigned char *key;
01714 uint32_t key_length;
01715
01716 key=info->lastkey+info->s->base.max_key_length;
01717 for (i=0 ; i < info->s->base.keys; i++)
01718 {
01719 if (i != prot_key && mi_is_key_active(info->s->state.key_map, i))
01720 {
01721 key_length=_mi_make_key(info,i,key,record,oldpos);
01722 if (info->s->keyinfo[i].flag & HA_NOSAME)
01723 {
01724 uint32_t nod_flag;
01725 MI_KEYDEF *keyinfo;
01726 keyinfo=info->s->keyinfo+i;
01727 if (_mi_search(info,keyinfo,key,USE_WHOLE_KEY,
01728 (uint) (SEARCH_SAME | SEARCH_SAVE_BUFF),
01729 info->s->state.key_root[i]))
01730 return(-1);
01731 nod_flag=mi_test_if_nod(info->buff);
01732 _mi_dpointer(info,info->int_keypos-nod_flag-
01733 info->s->rec_reflength,newpos);
01734 if (_mi_write_keypage(info,keyinfo,info->last_keypage,
01735 DFLT_INIT_HITS,info->buff))
01736 return(-1);
01737 }
01738 else
01739 {
01740 if (_mi_ck_delete(info,i,key,key_length))
01741 return(-1);
01742 key_length=_mi_make_key(info,i,key,record,newpos);
01743 if (_mi_ck_write(info,i,key,key_length))
01744 return(-1);
01745 }
01746 }
01747 }
01748 return(0);
01749 }
01750
01751
01752
01753
01754 void lock_memory(MI_CHECK *)
01755 {
01756 }
01757
01758
01759
01760
01761 int flush_blocks(MI_CHECK *param, KEY_CACHE *key_cache, int file)
01762 {
01763 if (flush_key_blocks(key_cache, file, FLUSH_RELEASE))
01764 {
01765 mi_check_print_error(param,"%d when trying to write bufferts",errno);
01766 return(1);
01767 }
01768 if (!param->using_global_keycache)
01769 end_key_cache(key_cache,1);
01770 return 0;
01771 }
01772
01773
01774
01775
01776 int mi_sort_index(MI_CHECK *param, register MI_INFO *info, char * name)
01777 {
01778 register uint32_t key;
01779 register MI_KEYDEF *keyinfo;
01780 int new_file;
01781 my_off_t index_pos[HA_MAX_POSSIBLE_KEY];
01782 uint32_t r_locks,w_locks;
01783 int old_lock;
01784 MYISAM_SHARE *share=info->s;
01785 MI_STATE_INFO old_state;
01786
01787
01788 for (key= 0,keyinfo= &share->keyinfo[0]; key < share->base.keys ;
01789 key++,keyinfo++)
01790
01791 if (!(param->testflag & T_SILENT))
01792 printf("- Sorting index for MyISAM-table '%s'\n",name);
01793
01794
01795 internal::fn_format(param->temp_filename,name,"", MI_NAME_IEXT,2+4+32);
01796 if ((new_file=my_create(internal::fn_format(param->temp_filename,param->temp_filename,
01797 "", INDEX_TMP_EXT,2+4),
01798 0,param->tmpfile_createflag,MYF(0))) <= 0)
01799 {
01800 mi_check_print_error(param,"Can't create new tempfile: '%s'",
01801 param->temp_filename);
01802 return(-1);
01803 }
01804 if (filecopy(param, new_file,share->kfile,0L,
01805 (ulong) share->base.keystart, "headerblock"))
01806 goto err;
01807
01808 param->new_file_pos=share->base.keystart;
01809 for (key= 0,keyinfo= &share->keyinfo[0]; key < share->base.keys ;
01810 key++,keyinfo++)
01811 {
01812 if (! mi_is_key_active(info->s->state.key_map, key))
01813 continue;
01814
01815 if (share->state.key_root[key] != HA_OFFSET_ERROR)
01816 {
01817 index_pos[key]=param->new_file_pos;
01818 if (sort_one_index(param,info,keyinfo,share->state.key_root[key],
01819 new_file))
01820 goto err;
01821 }
01822 else
01823 index_pos[key]= HA_OFFSET_ERROR;
01824 }
01825
01826
01827 flush_key_blocks(share->getKeyCache(), share->kfile, FLUSH_IGNORE_CHANGED);
01828
01829 share->state.version=(ulong) time((time_t*) 0);
01830 old_state= share->state;
01831 r_locks= share->r_locks;
01832 w_locks= share->w_locks;
01833 old_lock= info->lock_type;
01834
01835
01836 share->r_locks= share->w_locks= share->tot_locks= 0;
01837 (void) _mi_writeinfo(info,WRITEINFO_UPDATE_KEYFILE);
01838 internal::my_close(share->kfile,MYF(MY_WME));
01839 share->kfile = -1;
01840 internal::my_close(new_file,MYF(MY_WME));
01841 if (change_to_newfile(share->index_file_name,MI_NAME_IEXT,INDEX_TMP_EXT,0,
01842 MYF(0)) ||
01843 mi_open_keyfile(share))
01844 goto err2;
01845 info->lock_type= F_UNLCK;
01846 _mi_readinfo(info,F_WRLCK,0);
01847 info->lock_type= old_lock;
01848 share->r_locks= r_locks;
01849 share->w_locks= w_locks;
01850 share->tot_locks= r_locks+w_locks;
01851 share->state= old_state;
01852
01853 info->state->key_file_length=param->new_file_pos;
01854 info->update= (short) (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED);
01855 for (key=0 ; key < info->s->base.keys ; key++)
01856 info->s->state.key_root[key]=index_pos[key];
01857 for (key=0 ; key < info->s->state.header.max_block_size_index ; key++)
01858 info->s->state.key_del[key]= HA_OFFSET_ERROR;
01859
01860 info->s->state.changed&= ~STATE_NOT_SORTED_PAGES;
01861 return(0);
01862
01863 err:
01864 internal::my_close(new_file,MYF(MY_WME));
01865 err2:
01866 my_delete(param->temp_filename,MYF(MY_WME));
01867 return(-1);
01868 }
01869
01870
01871
01872
01873 static int sort_one_index(MI_CHECK *param, MI_INFO *info, MI_KEYDEF *keyinfo,
01874 my_off_t pagepos, int new_file)
01875 {
01876 uint32_t length,nod_flag,used_length, key_length;
01877 unsigned char *buff,*keypos,*endpos;
01878 unsigned char key[HA_MAX_POSSIBLE_KEY_BUFF];
01879 my_off_t new_page_pos,next_page;
01880 char llbuff[22];
01881
01882 new_page_pos=param->new_file_pos;
01883 param->new_file_pos+=keyinfo->block_length;
01884
01885 if (!(buff=(unsigned char*) malloc(keyinfo->block_length)))
01886 {
01887 mi_check_print_error(param,"Not enough memory for key block");
01888 return(-1);
01889 }
01890 if (!_mi_fetch_keypage(info,keyinfo,pagepos,DFLT_INIT_HITS,buff,0))
01891 {
01892 mi_check_print_error(param,"Can't read key block from filepos: %s",
01893 llstr(pagepos,llbuff));
01894 goto err;
01895 }
01896 if ((nod_flag=mi_test_if_nod(buff)))
01897 {
01898 used_length=mi_getint(buff);
01899 keypos=buff+2+nod_flag;
01900 endpos=buff+used_length;
01901 for ( ;; )
01902 {
01903 if (nod_flag)
01904 {
01905 next_page=_mi_kpos(nod_flag,keypos);
01906 _mi_kpointer(info,keypos-nod_flag,param->new_file_pos);
01907 if (sort_one_index(param,info,keyinfo,next_page,new_file))
01908 {
01909 goto err;
01910 }
01911 }
01912 if (keypos >= endpos ||
01913 (key_length=(*keyinfo->get_key)(keyinfo,nod_flag,&keypos,key)) == 0)
01914 break;
01915 assert(keypos <= endpos);
01916 }
01917 }
01918
01919
01920 length=mi_getint(buff);
01921 memset(buff+length, 0, keyinfo->block_length-length);
01922 if (my_pwrite(new_file,(unsigned char*) buff,(uint) keyinfo->block_length,
01923 new_page_pos,MYF(MY_NABP | MY_WAIT_IF_FULL)))
01924 {
01925 mi_check_print_error(param,"Can't write indexblock, error: %d",errno);
01926 goto err;
01927 }
01928 free(buff);
01929 return(0);
01930 err:
01931 free(buff);
01932 return(1);
01933 }
01934
01935
01936
01937
01938
01939
01940
01941
01942
01943
01944
01945 int change_to_newfile(const char * filename, const char * old_ext,
01946 const char * new_ext,
01947 uint32_t raid_chunks,
01948 myf MyFlags)
01949 {
01950 (void)raid_chunks;
01951 char old_filename[FN_REFLEN],new_filename[FN_REFLEN];
01952
01953 (void) internal::fn_format(old_filename,filename,"",old_ext,2+4+32);
01954 return my_redel(old_filename,
01955 internal::fn_format(new_filename,old_filename,"",new_ext,2+4),
01956 MYF(MY_WME | MY_LINK_WARNING | MyFlags));
01957 }
01958
01959
01960
01961
01962
01963 int filecopy(MI_CHECK *param, int to,int from,my_off_t start,
01964 my_off_t length, const char *type)
01965 {
01966 char tmp_buff[IO_SIZE],*buff;
01967 ulong buff_length;
01968
01969 buff_length=(ulong) min(param->write_buffer_length, (size_t)length);
01970 if (!(buff=(char *)malloc(buff_length)))
01971 {
01972 buff=tmp_buff; buff_length=IO_SIZE;
01973 }
01974
01975 lseek(from,start,SEEK_SET);
01976 while (length > buff_length)
01977 {
01978 if (my_read(from,(unsigned char*) buff,buff_length,MYF(MY_NABP)) ||
01979 my_write(to,(unsigned char*) buff,buff_length,param->myf_rw))
01980 goto err;
01981 length-= buff_length;
01982 }
01983 if (my_read(from,(unsigned char*) buff,(uint) length,MYF(MY_NABP)) ||
01984 my_write(to,(unsigned char*) buff,(uint) length,param->myf_rw))
01985 goto err;
01986 if (buff != tmp_buff)
01987 free(buff);
01988 return(0);
01989 err:
01990 if (buff != tmp_buff)
01991 free(buff);
01992 mi_check_print_error(param,"Can't copy %s to tempfile, error %d",
01993 type,errno);
01994 return(1);
01995 }
01996
01997
01998
01999
02000
02001
02002
02003
02004
02005
02006
02007
02008
02009
02010
02011
02012
02013 int mi_repair_by_sort(MI_CHECK *param, register MI_INFO *info,
02014 const char * name, int rep_quick)
02015 {
02016 int got_error;
02017 uint32_t i;
02018 ulong length;
02019 ha_rows start_records;
02020 my_off_t new_header_length,del;
02021 int new_file;
02022 MI_SORT_PARAM sort_param;
02023 MYISAM_SHARE *share=info->s;
02024 HA_KEYSEG *keyseg;
02025 ulong *rec_per_key_part;
02026 char llbuff[22];
02027 SORT_INFO sort_info;
02028 uint64_t key_map= 0;
02029
02030 start_records=info->state->records;
02031 got_error=1;
02032 new_file= -1;
02033 new_header_length=(param->testflag & T_UNPACK) ? 0 :
02034 share->pack.header_length;
02035 if (!(param->testflag & T_SILENT))
02036 {
02037 printf("- recovering (with sort) MyISAM-table '%s'\n",name);
02038 printf("Data records: %s\n", llstr(start_records,llbuff));
02039 }
02040 param->testflag|=T_REP;
02041
02042 if (info->s->options & (HA_OPTION_COMPRESS_RECORD))
02043 param->testflag|=T_CALC_CHECKSUM;
02044
02045 memset(&sort_info, 0, sizeof(sort_info));
02046 memset(&sort_param, 0, sizeof(sort_param));
02047 if (!(sort_info.key_block=
02048 alloc_key_blocks(param, (uint) param->sort_key_blocks, share->base.max_key_block_length))
02049 || param->read_cache.init_io_cache(info->dfile, (uint) param->read_buffer_length, READ_CACHE,share->pack.header_length,1,MYF(MY_WME))
02050 || (! rep_quick && info->rec_cache.init_io_cache(info->dfile, (uint) param->write_buffer_length, WRITE_CACHE,new_header_length,1, MYF(MY_WME | MY_WAIT_IF_FULL) & param->myf_rw)))
02051 {
02052 goto err;
02053 }
02054 sort_info.key_block_end=sort_info.key_block+param->sort_key_blocks;
02055 info->opt_flag|=WRITE_CACHE_USED;
02056 info->rec_cache.file=info->dfile;
02057
02058 if (!mi_alloc_rec_buff(info, SIZE_MAX, &sort_param.record) ||
02059 !mi_alloc_rec_buff(info, SIZE_MAX, &sort_param.rec_buff))
02060 {
02061 mi_check_print_error(param, "Not enough memory for extra record");
02062 goto err;
02063 }
02064 if (!rep_quick)
02065 {
02066
02067 if ((new_file=my_create(internal::fn_format(param->temp_filename,
02068 share->data_file_name, "",
02069 DATA_TMP_EXT, 2+4),
02070 0,param->tmpfile_createflag,
02071 MYF(0))) < 0)
02072 {
02073 mi_check_print_error(param,"Can't create new tempfile: '%s'",
02074 param->temp_filename);
02075 goto err;
02076 }
02077 if (new_header_length &&
02078 filecopy(param, new_file,info->dfile,0L,new_header_length,
02079 "datafile-header"))
02080 goto err;
02081 if (param->testflag & T_UNPACK)
02082 {
02083 share->options&= ~HA_OPTION_COMPRESS_RECORD;
02084 mi_int2store(share->state.header.options,share->options);
02085 }
02086 share->state.dellink= HA_OFFSET_ERROR;
02087 info->rec_cache.file=new_file;
02088 }
02089
02090 info->update= (short) (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED);
02091
02092
02093 mi_drop_all_indexes(param, info, false);
02094 key_map= share->state.key_map;
02095 if (param->testflag & T_CREATE_MISSING_KEYS)
02096 {
02097
02098 key_map= ~key_map;
02099 }
02100
02101 sort_info.info=info;
02102 sort_info.param = param;
02103
02104 set_data_file_type(&sort_info, share);
02105 sort_param.filepos=new_header_length;
02106 sort_info.dupp=0;
02107 sort_info.buff=0;
02108 param->read_cache.end_of_file=sort_info.filelength=
02109 lseek(param->read_cache.file,0L,SEEK_END);
02110
02111 sort_param.wordlist=NULL;
02112
02113 if (share->data_file_type == DYNAMIC_RECORD)
02114 length=max(share->base.min_pack_length+1,share->base.min_block_length);
02115 else if (share->data_file_type == COMPRESSED_RECORD)
02116 length=share->base.min_block_length;
02117 else
02118 length=share->base.pack_reclength;
02119 sort_info.max_records=
02120 ((param->testflag & T_CREATE_MISSING_KEYS) ? info->state->records :
02121 (ha_rows) (sort_info.filelength/length+1));
02122 sort_param.key_cmp=sort_key_cmp;
02123 sort_param.lock_in_memory=lock_memory;
02124 sort_param.sort_info=&sort_info;
02125 sort_param.fix_datafile= (bool) (! rep_quick);
02126 sort_param.master =1;
02127
02128 del=info->state->del;
02129 param->glob_crc=0;
02130 if (param->testflag & T_CALC_CHECKSUM)
02131 sort_param.calc_checksum= 1;
02132
02133 rec_per_key_part= param->rec_per_key_part;
02134 for (sort_param.key=0 ; sort_param.key < share->base.keys ;
02135 rec_per_key_part+=sort_param.keyinfo->keysegs, sort_param.key++)
02136 {
02137 sort_param.read_cache=param->read_cache;
02138 sort_param.keyinfo=share->keyinfo+sort_param.key;
02139 sort_param.seg=sort_param.keyinfo->seg;
02140
02141
02142
02143
02144 if (! mi_is_key_active(key_map, sort_param.key))
02145 {
02146
02147 assert(rec_per_key_part >= param->rec_per_key_part);
02148 memcpy(rec_per_key_part,
02149 (share->state.rec_per_key_part +
02150 (rec_per_key_part - param->rec_per_key_part)),
02151 sort_param.keyinfo->keysegs*sizeof(*rec_per_key_part));
02152 continue;
02153 }
02154
02155 if ((!(param->testflag & T_SILENT)))
02156 printf ("- Fixing index %d\n",sort_param.key+1);
02157 sort_param.max_pos=sort_param.pos=share->pack.header_length;
02158 keyseg=sort_param.seg;
02159 memset(sort_param.unique, 0, sizeof(sort_param.unique));
02160 sort_param.key_length=share->rec_reflength;
02161 for (i=0 ; keyseg[i].type != HA_KEYTYPE_END; i++)
02162 {
02163 sort_param.key_length+=keyseg[i].length;
02164 if (keyseg[i].flag & HA_SPACE_PACK)
02165 sort_param.key_length+=get_pack_length(keyseg[i].length);
02166 if (keyseg[i].flag & (HA_BLOB_PART | HA_VAR_LENGTH_PART))
02167 sort_param.key_length+=2 + test(keyseg[i].length >= 127);
02168 if (keyseg[i].flag & HA_NULL_PART)
02169 sort_param.key_length++;
02170 }
02171 info->state->records=info->state->del=share->state.split=0;
02172 info->state->empty=0;
02173
02174 {
02175 sort_param.key_read=sort_key_read;
02176 sort_param.key_write=sort_key_write;
02177 }
02178
02179 if (_create_index_by_sort(&sort_param,
02180 (bool) (!(param->testflag & T_VERBOSE)),
02181 (uint) param->sort_buffer_length))
02182 {
02183 param->retry_repair=1;
02184 goto err;
02185 }
02186
02187 sort_param.calc_checksum= 0;
02188 sort_param.wordroot.free_root(MYF(0));
02189
02190
02191 sort_info.max_records= (ha_rows) info->state->records;
02192
02193 if (param->testflag & T_STATISTICS)
02194 update_key_parts(sort_param.keyinfo, rec_per_key_part, sort_param.unique,
02195 param->stats_method == MI_STATS_METHOD_IGNORE_NULLS?
02196 sort_param.notnull: NULL,
02197 (uint64_t) info->state->records);
02198
02199 mi_set_key_active(share->state.key_map, sort_param.key);
02200
02201 if (sort_param.fix_datafile)
02202 {
02203 param->read_cache.end_of_file=sort_param.filepos;
02204 if (write_data_suffix(&sort_info, 1) || info->rec_cache.end_io_cache())
02205 {
02206 goto err;
02207 }
02208 if (param->testflag & T_SAFE_REPAIR)
02209 {
02210
02211 if (info->state->records+1 < start_records)
02212 {
02213 info->state->records=start_records;
02214 goto err;
02215 }
02216 }
02217 share->state.state.data_file_length = info->state->data_file_length=
02218 sort_param.filepos;
02219
02220 share->state.version=(ulong) time((time_t*) 0);
02221 internal::my_close(info->dfile,MYF(0));
02222 info->dfile=new_file;
02223 share->data_file_type=sort_info.new_data_file_type;
02224 share->pack.header_length=(ulong) new_header_length;
02225 sort_param.fix_datafile=0;
02226 }
02227 else
02228 info->state->data_file_length=sort_param.max_pos;
02229
02230 param->read_cache.file=info->dfile;
02231 param->read_cache.reinit_io_cache(READ_CACHE,share->pack.header_length, 1,1);
02232 }
02233
02234 if (param->testflag & T_WRITE_LOOP)
02235 {
02236 fputs(" \r",stdout); fflush(stdout);
02237 }
02238
02239 if (rep_quick && del+sort_info.dupp != info->state->del)
02240 {
02241 mi_check_print_error(param,"Couldn't fix table with quick recovery: Found wrong number of deleted records");
02242 mi_check_print_error(param,"Run recovery again without -q");
02243 got_error=1;
02244 param->retry_repair=1;
02245 param->testflag|=T_RETRY_WITHOUT_QUICK;
02246 goto err;
02247 }
02248
02249 if (rep_quick & T_FORCE_UNIQUENESS)
02250 {
02251 my_off_t skr=info->state->data_file_length+
02252 (share->options & HA_OPTION_COMPRESS_RECORD ?
02253 MEMMAP_EXTRA_MARGIN : 0);
02254 #ifdef USE_RELOC
02255 if (share->data_file_type == STATIC_RECORD &&
02256 skr < share->base.reloc*share->base.min_pack_length)
02257 skr=share->base.reloc*share->base.min_pack_length;
02258 #endif
02259 if (skr != sort_info.filelength && !info->s->base.raid_type)
02260 if (ftruncate(info->dfile, skr))
02261 mi_check_print_warning(param,
02262 "Can't change size of datafile, error: %d",
02263 errno);
02264 }
02265 if (param->testflag & T_CALC_CHECKSUM)
02266 info->state->checksum=param->glob_crc;
02267
02268 if (ftruncate(share->kfile, info->state->key_file_length))
02269 mi_check_print_warning(param,
02270 "Can't change size of indexfile, error: %d",
02271 errno);
02272
02273 if (!(param->testflag & T_SILENT))
02274 {
02275 if (start_records != info->state->records)
02276 printf("Data records: %s\n", llstr(info->state->records,llbuff));
02277 if (sort_info.dupp)
02278 mi_check_print_warning(param,
02279 "%s records have been removed",
02280 llstr(sort_info.dupp,llbuff));
02281 }
02282 got_error=0;
02283
02284 if (&share->state.state != info->state)
02285 memcpy( &share->state.state, info->state, sizeof(*info->state));
02286
02287 err:
02288 got_error|= flush_blocks(param, share->getKeyCache(), share->kfile);
02289 info->rec_cache.end_io_cache();
02290 if (!got_error)
02291 {
02292
02293 if (new_file >= 0)
02294 {
02295 internal::my_close(new_file,MYF(0));
02296 info->dfile=new_file= -1;
02297 if (change_to_newfile(share->data_file_name,MI_NAME_DEXT,
02298 DATA_TMP_EXT, share->base.raid_chunks,
02299 (param->testflag & T_BACKUP_DATA ?
02300 MYF(MY_REDEL_MAKE_BACKUP): MYF(0))) ||
02301 mi_open_datafile(info,share,-1))
02302 got_error=1;
02303 }
02304 }
02305 if (got_error)
02306 {
02307 if (! param->error_printed)
02308 mi_check_print_error(param,"%d when fixing table",errno);
02309 if (new_file >= 0)
02310 {
02311 internal::my_close(new_file,MYF(0));
02312 my_delete(param->temp_filename, MYF(MY_WME));
02313 if (info->dfile == new_file)
02314 info->dfile= -1;
02315 }
02316 mi_mark_crashed_on_repair(info);
02317 }
02318 else if (key_map == share->state.key_map)
02319 share->state.changed&= ~STATE_NOT_OPTIMIZED_KEYS;
02320 share->state.changed|=STATE_NOT_SORTED_PAGES;
02321
02322 void * rec_buff_ptr= NULL;
02323 rec_buff_ptr= mi_get_rec_buff_ptr(info, sort_param.rec_buff);
02324 if (rec_buff_ptr != NULL)
02325 free(rec_buff_ptr);
02326 rec_buff_ptr= mi_get_rec_buff_ptr(info, sort_param.record);
02327 if (rec_buff_ptr != NULL)
02328 free(rec_buff_ptr);
02329 rec_buff_ptr= NULL;
02330
02331 free((unsigned char*) sort_info.key_block);
02332 free(sort_info.buff);
02333 param->read_cache.end_io_cache();
02334 info->opt_flag&= ~(READ_CACHE_USED | WRITE_CACHE_USED);
02335 if (!got_error && (param->testflag & T_UNPACK))
02336 {
02337 share->state.header.options[0]&= (unsigned char) ~HA_OPTION_COMPRESS_RECORD;
02338 share->pack.header_length=0;
02339 }
02340 return(got_error);
02341 }
02342
02343
02344
02345 int sort_key_read(MI_SORT_PARAM *sort_param, void *key)
02346 {
02347 int error;
02348 SORT_INFO *sort_info=sort_param->sort_info;
02349 MI_INFO *info=sort_info->info;
02350
02351 if ((error=sort_get_next_record(sort_param)))
02352 return(error);
02353 if (info->state->records == sort_info->max_records)
02354 {
02355 mi_check_print_error(sort_info->param,
02356 "Key %d - Found too many records; Can't continue",
02357 sort_param->key+1);
02358 return(1);
02359 }
02360 sort_param->real_key_length=
02361 (info->s->rec_reflength+
02362 _mi_make_key(info, sort_param->key, (unsigned char*) key,
02363 sort_param->record, sort_param->filepos));
02364 #ifdef HAVE_VALGRIND
02365 memset((unsigned char *)key+sort_param->real_key_length, 0,
02366 (sort_param->key_length-sort_param->real_key_length));
02367 #endif
02368 return(sort_write_record(sort_param));
02369 }
02370
02371
02372
02373
02374
02375
02376
02377
02378
02379
02380
02381
02382
02383
02384
02385
02386
02387
02388
02389
02390
02391
02392
02393
02394
02395
02396
02397
02398
02399
02400
02401
02402
02403 int sort_get_next_record(MI_SORT_PARAM *sort_param)
02404 {
02405 int searching;
02406 int parallel_flag;
02407 uint32_t found_record,b_type,left_length;
02408 my_off_t pos;
02409 unsigned char *to= NULL;
02410 MI_BLOCK_INFO block_info;
02411 SORT_INFO *sort_info=sort_param->sort_info;
02412 MI_CHECK *param=sort_info->param;
02413 MI_INFO *info=sort_info->info;
02414 MYISAM_SHARE *share=info->s;
02415 char llbuff[22],llbuff2[22];
02416
02417 if (*killed_ptr(param))
02418 return(1);
02419
02420 switch (share->data_file_type) {
02421 case STATIC_RECORD:
02422 for (;;)
02423 {
02424 if (my_b_read(&sort_param->read_cache,sort_param->record,
02425 share->base.pack_reclength))
02426 {
02427 if (sort_param->read_cache.error)
02428 param->out_flag |= O_DATA_LOST;
02429 param->retry_repair=1;
02430 param->testflag|=T_RETRY_WITHOUT_QUICK;
02431 return(-1);
02432 }
02433 sort_param->start_recpos=sort_param->pos;
02434 if (!sort_param->fix_datafile)
02435 {
02436 sort_param->filepos=sort_param->pos;
02437 if (sort_param->master)
02438 share->state.split++;
02439 }
02440 sort_param->max_pos=(sort_param->pos+=share->base.pack_reclength);
02441 if (*sort_param->record)
02442 {
02443 if (sort_param->calc_checksum)
02444 param->glob_crc+= (info->checksum=
02445 mi_static_checksum(info,sort_param->record));
02446 return(0);
02447 }
02448 if (!sort_param->fix_datafile && sort_param->master)
02449 {
02450 info->state->del++;
02451 info->state->empty+=share->base.pack_reclength;
02452 }
02453 }
02454 case DYNAMIC_RECORD:
02455 pos= sort_param->pos;
02456 searching= (sort_param->fix_datafile && (param->testflag & T_EXTEND));
02457 parallel_flag= (sort_param->read_cache.file < 0) ? READING_NEXT : 0;
02458 for (;;)
02459 {
02460 found_record=block_info.second_read= 0;
02461 left_length=1;
02462 if (searching)
02463 {
02464 pos=MY_ALIGN(pos,MI_DYN_ALIGN_SIZE);
02465 param->testflag|=T_RETRY_WITHOUT_QUICK;
02466 sort_param->start_recpos=pos;
02467 }
02468 do
02469 {
02470 if (pos > sort_param->max_pos)
02471 sort_param->max_pos=pos;
02472 if (pos & (MI_DYN_ALIGN_SIZE-1))
02473 {
02474 if ((param->testflag & T_VERBOSE) || searching == 0)
02475 mi_check_print_info(param,"Wrong aligned block at %s",
02476 llstr(pos,llbuff));
02477 if (searching)
02478 goto try_next;
02479 }
02480 if (found_record && pos == param->search_after_block)
02481 mi_check_print_info(param,"Block: %s used by record at %s",
02482 llstr(param->search_after_block,llbuff),
02483 llstr(sort_param->start_recpos,llbuff2));
02484 if (_mi_read_cache(&sort_param->read_cache,
02485 (unsigned char*) block_info.header,pos,
02486 MI_BLOCK_INFO_HEADER_LENGTH,
02487 (! found_record ? READING_NEXT : 0) |
02488 parallel_flag | READING_HEADER))
02489 {
02490 if (found_record)
02491 {
02492 mi_check_print_info(param,
02493 "Can't read whole record at %s (errno: %d)",
02494 llstr(sort_param->start_recpos,llbuff),errno);
02495 goto try_next;
02496 }
02497 return(-1);
02498 }
02499 if (searching && ! sort_param->fix_datafile)
02500 {
02501 param->error_printed=1;
02502 param->retry_repair=1;
02503 param->testflag|=T_RETRY_WITHOUT_QUICK;
02504 return(1);
02505 }
02506 b_type=_mi_get_block_info(&block_info,-1,pos);
02507 if ((b_type & (BLOCK_ERROR | BLOCK_FATAL_ERROR)) ||
02508 ((b_type & BLOCK_FIRST) &&
02509 (block_info.rec_len < (uint) share->base.min_pack_length ||
02510 block_info.rec_len > (uint) share->base.max_pack_length)))
02511 {
02512 uint32_t i;
02513 if (param->testflag & T_VERBOSE || searching == 0)
02514 mi_check_print_info(param,
02515 "Wrong bytesec: %3d-%3d-%3d at %10s; Skipped",
02516 block_info.header[0],block_info.header[1],
02517 block_info.header[2],llstr(pos,llbuff));
02518 if (found_record)
02519 goto try_next;
02520 block_info.second_read=0;
02521 searching=1;
02522
02523 for (i=MI_DYN_ALIGN_SIZE ;
02524 i < MI_BLOCK_INFO_HEADER_LENGTH ;
02525 i+= MI_DYN_ALIGN_SIZE)
02526 if (block_info.header[i] >= 1 &&
02527 block_info.header[i] <= MI_MAX_DYN_HEADER_BYTE)
02528 break;
02529 pos+=(ulong) i;
02530 sort_param->start_recpos=pos;
02531 continue;
02532 }
02533 if (b_type & BLOCK_DELETED)
02534 {
02535 bool error=0;
02536 if (block_info.block_len+ (uint) (block_info.filepos-pos) <
02537 share->base.min_block_length)
02538 {
02539 if (!searching)
02540 mi_check_print_info(param,
02541 "Deleted block with impossible length %u at %s",
02542 block_info.block_len,llstr(pos,llbuff));
02543 error=1;
02544 }
02545 else
02546 {
02547 if ((block_info.next_filepos != HA_OFFSET_ERROR &&
02548 block_info.next_filepos >=
02549 info->state->data_file_length) ||
02550 (block_info.prev_filepos != HA_OFFSET_ERROR &&
02551 block_info.prev_filepos >= info->state->data_file_length))
02552 {
02553 if (!searching)
02554 mi_check_print_info(param,
02555 "Delete link points outside datafile at %s",
02556 llstr(pos,llbuff));
02557 error=1;
02558 }
02559 }
02560 if (error)
02561 {
02562 if (found_record)
02563 goto try_next;
02564 searching=1;
02565 pos+= MI_DYN_ALIGN_SIZE;
02566 sort_param->start_recpos=pos;
02567 block_info.second_read=0;
02568 continue;
02569 }
02570 }
02571 else
02572 {
02573 if (block_info.block_len+ (uint) (block_info.filepos-pos) <
02574 share->base.min_block_length ||
02575 block_info.block_len > (uint) share->base.max_pack_length+
02576 MI_SPLIT_LENGTH)
02577 {
02578 if (!searching)
02579 mi_check_print_info(param,
02580 "Found block with impossible length %u at %s; Skipped",
02581 block_info.block_len+ (uint) (block_info.filepos-pos),
02582 llstr(pos,llbuff));
02583 if (found_record)
02584 goto try_next;
02585 searching=1;
02586 pos+= MI_DYN_ALIGN_SIZE;
02587 sort_param->start_recpos=pos;
02588 block_info.second_read=0;
02589 continue;
02590 }
02591 }
02592 if (b_type & (BLOCK_DELETED | BLOCK_SYNC_ERROR))
02593 {
02594 if (!sort_param->fix_datafile && sort_param->master &&
02595 (b_type & BLOCK_DELETED))
02596 {
02597 info->state->empty+=block_info.block_len;
02598 info->state->del++;
02599 share->state.split++;
02600 }
02601 if (found_record)
02602 goto try_next;
02603 if (searching)
02604 {
02605 pos+=MI_DYN_ALIGN_SIZE;
02606 sort_param->start_recpos=pos;
02607 }
02608 else
02609 pos=block_info.filepos+block_info.block_len;
02610 block_info.second_read=0;
02611 continue;
02612 }
02613
02614 if (!sort_param->fix_datafile && sort_param->master)
02615 share->state.split++;
02616 if (! found_record++)
02617 {
02618 sort_param->find_length=left_length=block_info.rec_len;
02619 sort_param->start_recpos=pos;
02620 if (!sort_param->fix_datafile)
02621 sort_param->filepos=sort_param->start_recpos;
02622 if (sort_param->fix_datafile && (param->testflag & T_EXTEND))
02623 sort_param->pos=block_info.filepos+1;
02624 else
02625 sort_param->pos=block_info.filepos+block_info.block_len;
02626 if (share->base.blobs)
02627 {
02628 if (!(to=mi_alloc_rec_buff(info,block_info.rec_len,
02629 &(sort_param->rec_buff))))
02630 {
02631 if (param->max_record_length >= block_info.rec_len)
02632 {
02633 mi_check_print_error(param,"Not enough memory for blob at %s (need %lu)",
02634 llstr(sort_param->start_recpos,llbuff),
02635 (ulong) block_info.rec_len);
02636 return(1);
02637 }
02638 else
02639 {
02640 mi_check_print_info(param,"Not enough memory for blob at %s (need %lu); Row skipped",
02641 llstr(sort_param->start_recpos,llbuff),
02642 (ulong) block_info.rec_len);
02643 goto try_next;
02644 }
02645 }
02646 }
02647 else
02648 to= sort_param->rec_buff;
02649 }
02650 if (left_length < block_info.data_len || ! block_info.data_len)
02651 {
02652 mi_check_print_info(param,
02653 "Found block with too small length at %s; Skipped",
02654 llstr(sort_param->start_recpos,llbuff));
02655 goto try_next;
02656 }
02657 if (block_info.filepos + block_info.data_len >
02658 sort_param->read_cache.end_of_file)
02659 {
02660 mi_check_print_info(param,
02661 "Found block that points outside data file at %s",
02662 llstr(sort_param->start_recpos,llbuff));
02663 goto try_next;
02664 }
02665
02666
02667
02668
02669
02670 {
02671 uint32_t header_len= (uint) (block_info.filepos - pos);
02672 uint32_t prefetch_len= (MI_BLOCK_INFO_HEADER_LENGTH - header_len);
02673
02674 if (prefetch_len > block_info.data_len)
02675 prefetch_len= block_info.data_len;
02676 if (prefetch_len)
02677 {
02678 memcpy(to, block_info.header + header_len, prefetch_len);
02679 block_info.filepos+= prefetch_len;
02680 block_info.data_len-= prefetch_len;
02681 left_length-= prefetch_len;
02682 to+= prefetch_len;
02683 }
02684 }
02685 if (block_info.data_len &&
02686 _mi_read_cache(&sort_param->read_cache,to,block_info.filepos,
02687 block_info.data_len,
02688 (found_record == 1 ? READING_NEXT : 0) |
02689 parallel_flag))
02690 {
02691 mi_check_print_info(param,
02692 "Read error for block at: %s (error: %d); Skipped",
02693 llstr(block_info.filepos,llbuff),errno);
02694 goto try_next;
02695 }
02696 left_length-=block_info.data_len;
02697 to+=block_info.data_len;
02698 pos=block_info.next_filepos;
02699 if (pos == HA_OFFSET_ERROR && left_length)
02700 {
02701 mi_check_print_info(param,"Wrong block with wrong total length starting at %s",
02702 llstr(sort_param->start_recpos,llbuff));
02703 goto try_next;
02704 }
02705 if (pos + MI_BLOCK_INFO_HEADER_LENGTH > sort_param->read_cache.end_of_file)
02706 {
02707 mi_check_print_info(param,"Found link that points at %s (outside data file) at %s",
02708 llstr(pos,llbuff2),
02709 llstr(sort_param->start_recpos,llbuff));
02710 goto try_next;
02711 }
02712 } while (left_length);
02713
02714 if (_mi_rec_unpack(info,sort_param->record,sort_param->rec_buff,
02715 sort_param->find_length) != MY_FILE_ERROR)
02716 {
02717 if (sort_param->read_cache.error < 0)
02718 return(1);
02719 if (sort_param->calc_checksum)
02720 info->checksum= mi_checksum(info, sort_param->record);
02721 if ((param->testflag & (T_EXTEND | T_REP)) || searching)
02722 {
02723 if (_mi_rec_check(info, sort_param->record, sort_param->rec_buff,
02724 sort_param->find_length,
02725 (param->testflag & T_QUICK) &&
02726 sort_param->calc_checksum &&
02727 test(info->s->calc_checksum)))
02728 {
02729 mi_check_print_info(param,"Found wrong packed record at %s",
02730 llstr(sort_param->start_recpos,llbuff));
02731 goto try_next;
02732 }
02733 }
02734 if (sort_param->calc_checksum)
02735 param->glob_crc+= info->checksum;
02736 return(0);
02737 }
02738 if (!searching)
02739 mi_check_print_info(param,"Key %d - Found wrong stored record at %s",
02740 sort_param->key+1,
02741 llstr(sort_param->start_recpos,llbuff));
02742 try_next:
02743 pos=(sort_param->start_recpos+=MI_DYN_ALIGN_SIZE);
02744 searching=1;
02745 }
02746 case COMPRESSED_RECORD:
02747 case BLOCK_RECORD:
02748 assert(0);
02749 }
02750 return(1);
02751 }
02752
02753
02754
02755
02756
02757
02758
02759
02760
02761
02762
02763
02764
02765
02766
02767
02768
02769 int sort_write_record(MI_SORT_PARAM *sort_param)
02770 {
02771 int flag;
02772 ulong block_length,reclength;
02773 unsigned char *from;
02774 SORT_INFO *sort_info=sort_param->sort_info;
02775 MI_CHECK *param=sort_info->param;
02776 MI_INFO *info=sort_info->info;
02777 MYISAM_SHARE *share=info->s;
02778
02779 if (sort_param->fix_datafile)
02780 {
02781 switch (sort_info->new_data_file_type) {
02782 case STATIC_RECORD:
02783 if (my_b_write(&info->rec_cache,sort_param->record,
02784 share->base.pack_reclength))
02785 {
02786 mi_check_print_error(param,"%d when writing to datafile",errno);
02787 return(1);
02788 }
02789 sort_param->filepos+=share->base.pack_reclength;
02790 info->s->state.split++;
02791
02792 break;
02793 case DYNAMIC_RECORD:
02794 if (! info->blobs)
02795 from=sort_param->rec_buff;
02796 else
02797 {
02798
02799 reclength=info->s->base.pack_reclength+
02800 _my_calc_total_blob_length(info,sort_param->record)+
02801 ALIGN_SIZE(MI_MAX_DYN_BLOCK_HEADER)+MI_SPLIT_LENGTH+
02802 MI_DYN_DELETE_BLOCK_HEADER;
02803 if (sort_info->buff_length < reclength)
02804 {
02805 void *tmpptr= NULL;
02806 tmpptr= realloc(sort_info->buff, reclength);
02807 if(tmpptr)
02808 {
02809 sort_info->buff_length=reclength;
02810 sort_info->buff= (unsigned char *)tmpptr;
02811 }
02812 else
02813 {
02814 mi_check_print_error(param,"Could not realloc() sort_info->buff "
02815 " to %ul bytes", reclength);
02816 return(1);
02817 }
02818 }
02819 from= sort_info->buff+ALIGN_SIZE(MI_MAX_DYN_BLOCK_HEADER);
02820 }
02821
02822 info->checksum=mi_checksum(info,sort_param->record);
02823 reclength=_mi_rec_pack(info,from,sort_param->record);
02824 flag=0;
02825
02826
02827 do
02828 {
02829 block_length=reclength+ 3 + test(reclength >= (65520-3));
02830 if (block_length < share->base.min_block_length)
02831 block_length=share->base.min_block_length;
02832 info->update|=HA_STATE_WRITE_AT_END;
02833 block_length=MY_ALIGN(block_length,MI_DYN_ALIGN_SIZE);
02834 if (block_length > MI_MAX_BLOCK_LENGTH)
02835 block_length=MI_MAX_BLOCK_LENGTH;
02836 if (_mi_write_part_record(info,0L,block_length,
02837 sort_param->filepos+block_length,
02838 &from,&reclength,&flag))
02839 {
02840 mi_check_print_error(param,"%d when writing to datafile",errno);
02841 return(1);
02842 }
02843 sort_param->filepos+=block_length;
02844 info->s->state.split++;
02845 } while (reclength);
02846
02847 break;
02848 case COMPRESSED_RECORD:
02849 case BLOCK_RECORD:
02850 assert(0);
02851 }
02852 }
02853 if (sort_param->master)
02854 {
02855 info->state->records++;
02856 if ((param->testflag & T_WRITE_LOOP) &&
02857 (info->state->records % WRITE_COUNT) == 0)
02858 {
02859 char llbuff[22];
02860 printf("%s\r", llstr(info->state->records,llbuff));
02861 fflush(stdout);
02862 }
02863 }
02864 return(0);
02865 }
02866
02867
02868
02869
02870 int sort_key_cmp(MI_SORT_PARAM *sort_param, const void *a, const void *b)
02871 {
02872 uint32_t not_used[2];
02873 return (ha_key_cmp(sort_param->seg, *((unsigned char* const *) a), *((unsigned char* const *) b),
02874 USE_WHOLE_KEY, SEARCH_SAME, not_used));
02875 }
02876
02877
02878 int sort_key_write(MI_SORT_PARAM *sort_param, const void *a)
02879 {
02880 uint32_t diff_pos[2];
02881 char llbuff[22],llbuff2[22];
02882 SORT_INFO *sort_info=sort_param->sort_info;
02883 MI_CHECK *param= sort_info->param;
02884 int cmp;
02885
02886 if (sort_info->key_block->inited)
02887 {
02888 cmp=ha_key_cmp(sort_param->seg,sort_info->key_block->lastkey,
02889 (unsigned char*) a, USE_WHOLE_KEY,SEARCH_FIND | SEARCH_UPDATE,
02890 diff_pos);
02891 if (param->stats_method == MI_STATS_METHOD_NULLS_NOT_EQUAL)
02892 ha_key_cmp(sort_param->seg,sort_info->key_block->lastkey,
02893 (unsigned char*) a, USE_WHOLE_KEY,
02894 SEARCH_FIND | SEARCH_NULL_ARE_NOT_EQUAL, diff_pos);
02895 else if (param->stats_method == MI_STATS_METHOD_IGNORE_NULLS)
02896 {
02897 diff_pos[0]= mi_collect_stats_nonulls_next(sort_param->seg,
02898 sort_param->notnull,
02899 sort_info->key_block->lastkey,
02900 (unsigned char*)a);
02901 }
02902 sort_param->unique[diff_pos[0]-1]++;
02903 }
02904 else
02905 {
02906 cmp= -1;
02907 if (param->stats_method == MI_STATS_METHOD_IGNORE_NULLS)
02908 mi_collect_stats_nonulls_first(sort_param->seg, sort_param->notnull,
02909 (unsigned char*)a);
02910 }
02911 if ((sort_param->keyinfo->flag & HA_NOSAME) && cmp == 0)
02912 {
02913 sort_info->dupp++;
02914 sort_info->info->lastpos=get_record_for_key(sort_info->info,
02915 sort_param->keyinfo,
02916 (unsigned char*) a);
02917 mi_check_print_warning(param,
02918 "Duplicate key for record at %10s against record at %10s",
02919 llstr(sort_info->info->lastpos,llbuff),
02920 llstr(get_record_for_key(sort_info->info,
02921 sort_param->keyinfo,
02922 sort_info->key_block->
02923 lastkey),
02924 llbuff2));
02925 param->testflag|=T_RETRY_WITHOUT_QUICK;
02926 return (sort_delete_record(sort_param));
02927 }
02928 return (sort_insert_key(sort_param,sort_info->key_block,
02929 (unsigned char*) a, HA_OFFSET_ERROR));
02930 }
02931
02932
02933
02934
02935 my_off_t get_record_for_key(MI_INFO *info, MI_KEYDEF *keyinfo,
02936 unsigned char *key) {
02937 return _mi_dpos(info,0,key+_mi_keylength(keyinfo,key));
02938 }
02939
02940
02941
02942
02943 int sort_insert_key(MI_SORT_PARAM *sort_param,
02944 register SORT_KEY_BLOCKS *key_block, unsigned char *key,
02945 my_off_t prev_block)
02946 {
02947 uint32_t a_length,t_length,nod_flag;
02948 my_off_t filepos,key_file_length;
02949 unsigned char *anc_buff,*lastkey;
02950 MI_KEY_PARAM s_temp;
02951 MI_INFO *info;
02952 MI_KEYDEF *keyinfo=sort_param->keyinfo;
02953 SORT_INFO *sort_info= sort_param->sort_info;
02954 MI_CHECK *param=sort_info->param;
02955
02956 anc_buff=key_block->buff;
02957 info=sort_info->info;
02958 lastkey=key_block->lastkey;
02959 nod_flag= (key_block == sort_info->key_block ? 0 :
02960 info->s->base.key_reflength);
02961
02962 if (!key_block->inited)
02963 {
02964 key_block->inited=1;
02965 if (key_block == sort_info->key_block_end)
02966 {
02967 mi_check_print_error(param,"To many key-block-levels; Try increasing sort_key_blocks");
02968 return(1);
02969 }
02970 a_length=2+nod_flag;
02971 key_block->end_pos=anc_buff+2;
02972 lastkey=0;
02973 }
02974 else
02975 a_length=mi_getint(anc_buff);
02976
02977
02978 if (nod_flag)
02979 _mi_kpointer(info,key_block->end_pos,prev_block);
02980
02981 t_length=(*keyinfo->pack_key)(keyinfo,nod_flag,
02982 (unsigned char*) 0,lastkey,lastkey,key,
02983 &s_temp);
02984 (*keyinfo->store_key)(keyinfo, key_block->end_pos+nod_flag,&s_temp);
02985 a_length+=t_length;
02986 mi_putint(anc_buff,a_length,nod_flag);
02987 key_block->end_pos+=t_length;
02988 if (a_length <= keyinfo->block_length)
02989 {
02990 _mi_move_key(keyinfo,key_block->lastkey,key);
02991 key_block->last_length=a_length-t_length;
02992 return(0);
02993 }
02994
02995
02996 mi_putint(anc_buff,key_block->last_length,nod_flag);
02997 memset(anc_buff+key_block->last_length, 0,
02998 keyinfo->block_length - key_block->last_length);
02999 key_file_length=info->state->key_file_length;
03000 if ((filepos=_mi_new(info,keyinfo,DFLT_INIT_HITS)) == HA_OFFSET_ERROR)
03001 return(1);
03002
03003
03004 if (key_file_length == info->state->key_file_length)
03005 {
03006 if (_mi_write_keypage(info, keyinfo, filepos, DFLT_INIT_HITS, anc_buff))
03007 return(1);
03008 }
03009 else if (my_pwrite(info->s->kfile,(unsigned char*) anc_buff,
03010 (uint) keyinfo->block_length,filepos, param->myf_rw))
03011 return(1);
03012
03013
03014 if (sort_insert_key(sort_param,key_block+1,key_block->lastkey,filepos))
03015 return(1);
03016
03017
03018 key_block->inited=0;
03019 return(sort_insert_key(sort_param, key_block,key,prev_block));
03020 }
03021
03022
03023
03024
03025 int sort_delete_record(MI_SORT_PARAM *sort_param)
03026 {
03027 uint32_t i;
03028 int old_file,error;
03029 unsigned char *key;
03030 SORT_INFO *sort_info=sort_param->sort_info;
03031 MI_CHECK *param=sort_info->param;
03032 MI_INFO *info=sort_info->info;
03033
03034 if ((param->testflag & (T_FORCE_UNIQUENESS|T_QUICK)) == T_QUICK)
03035 {
03036 mi_check_print_error(param,
03037 "Quick-recover aborted; Run recovery without switch -q or with switch -qq");
03038 return(1);
03039 }
03040 if (info->s->options & HA_OPTION_COMPRESS_RECORD)
03041 {
03042 mi_check_print_error(param,
03043 "Recover aborted; Can't run standard recovery on compressed tables with errors in data-file. Use switch 'myisamchk --safe-recover' to fix it\n",stderr);;
03044 return(1);
03045 }
03046
03047 old_file=info->dfile;
03048 info->dfile=info->rec_cache.file;
03049 if (sort_info->current_key)
03050 {
03051 key=info->lastkey+info->s->base.max_key_length;
03052 if ((error=(*info->s->read_rnd)(info,sort_param->record,info->lastpos,0)) &&
03053 error != HA_ERR_RECORD_DELETED)
03054 {
03055 mi_check_print_error(param,"Can't read record to be removed");
03056 info->dfile=old_file;
03057 return(1);
03058 }
03059
03060 for (i=0 ; i < sort_info->current_key ; i++)
03061 {
03062 uint32_t key_length=_mi_make_key(info,i,key,sort_param->record,info->lastpos);
03063 if (_mi_ck_delete(info,i,key,key_length))
03064 {
03065 mi_check_print_error(param,"Can't delete key %d from record to be removed",i+1);
03066 info->dfile=old_file;
03067 return(1);
03068 }
03069 }
03070 if (sort_param->calc_checksum)
03071 param->glob_crc-=(*info->s->calc_checksum)(info, sort_param->record);
03072 }
03073 error=flush_io_cache(&info->rec_cache) || (*info->s->delete_record)(info);
03074 info->dfile=old_file;
03075 info->state->records--;
03076 return(error);
03077 }
03078
03079
03080
03081 int flush_pending_blocks(MI_SORT_PARAM *sort_param)
03082 {
03083 uint32_t nod_flag,length;
03084 my_off_t filepos,key_file_length;
03085 SORT_KEY_BLOCKS *key_block;
03086 SORT_INFO *sort_info= sort_param->sort_info;
03087 myf myf_rw=sort_info->param->myf_rw;
03088 MI_INFO *info=sort_info->info;
03089 MI_KEYDEF *keyinfo=sort_param->keyinfo;
03090
03091 filepos= HA_OFFSET_ERROR;
03092 nod_flag=0;
03093 for (key_block=sort_info->key_block ; key_block->inited ; key_block++)
03094 {
03095 key_block->inited=0;
03096 length=mi_getint(key_block->buff);
03097 if (nod_flag)
03098 _mi_kpointer(info,key_block->end_pos,filepos);
03099 key_file_length=info->state->key_file_length;
03100 memset(key_block->buff+length, 0, keyinfo->block_length-length);
03101 if ((filepos=_mi_new(info,keyinfo,DFLT_INIT_HITS)) == HA_OFFSET_ERROR)
03102 return(1);
03103
03104
03105 if (key_file_length == info->state->key_file_length)
03106 {
03107 if (_mi_write_keypage(info, keyinfo, filepos,
03108 DFLT_INIT_HITS, key_block->buff))
03109 return(1);
03110 }
03111 else if (my_pwrite(info->s->kfile,(unsigned char*) key_block->buff,
03112 (uint) keyinfo->block_length,filepos, myf_rw))
03113 return(1);
03114 nod_flag=1;
03115 }
03116 info->s->state.key_root[sort_param->key]=filepos;
03117 return(0);
03118 }
03119
03120
03121
03122 static SORT_KEY_BLOCKS *alloc_key_blocks(MI_CHECK *param, uint32_t blocks,
03123 uint32_t buffer_length)
03124 {
03125 register uint32_t i;
03126 SORT_KEY_BLOCKS *block;
03127
03128 if (!(block=(SORT_KEY_BLOCKS*) malloc((sizeof(SORT_KEY_BLOCKS)+
03129 buffer_length+IO_SIZE)*blocks)))
03130 {
03131 mi_check_print_error(param,"Not enough memory for sort-key-blocks");
03132 return(0);
03133 }
03134 for (i=0 ; i < blocks ; i++)
03135 {
03136 block[i].inited=0;
03137 block[i].buff=(unsigned char*) (block+blocks)+(buffer_length+IO_SIZE)*i;
03138 }
03139 return(block);
03140 }
03141
03142
03143
03144
03145 int test_if_almost_full(MI_INFO *info)
03146 {
03147 if (info->s->options & HA_OPTION_COMPRESS_RECORD)
03148 return 0;
03149 return (my_off_t)(lseek(info->s->kfile, 0L, SEEK_END) / 10 * 9) >
03150 (my_off_t) info->s->base.max_key_file_length ||
03151 (my_off_t)(lseek(info->dfile, 0L, SEEK_END) / 10 * 9) >
03152 (my_off_t) info->s->base.max_data_file_length;
03153 }
03154
03155
03156
03157
03158 int write_data_suffix(SORT_INFO *sort_info, bool fix_datafile)
03159 {
03160 MI_INFO *info=sort_info->info;
03161
03162 if (info->s->options & HA_OPTION_COMPRESS_RECORD && fix_datafile)
03163 {
03164 unsigned char buff[MEMMAP_EXTRA_MARGIN];
03165 memset(buff, 0, sizeof(buff));
03166 if (my_b_write(&info->rec_cache,buff,sizeof(buff)))
03167 {
03168 mi_check_print_error(sort_info->param,
03169 "%d when writing to datafile",errno);
03170 return 1;
03171 }
03172 sort_info->param->read_cache.end_of_file+=sizeof(buff);
03173 }
03174 return 0;
03175 }
03176
03177
03178
03179 int update_state_info(MI_CHECK *param, MI_INFO *info,uint32_t update)
03180 {
03181 MYISAM_SHARE *share=info->s;
03182
03183 if (update & UPDATE_OPEN_COUNT)
03184 {
03185 share->state.open_count=0;
03186 share->global_changed=0;
03187 }
03188 if (update & UPDATE_STAT)
03189 {
03190 uint32_t i, key_parts= mi_uint2korr(share->state.header.key_parts);
03191 share->state.rec_per_key_rows=info->state->records;
03192 share->state.changed&= ~STATE_NOT_ANALYZED;
03193 if (info->state->records)
03194 {
03195 for (i=0; i<key_parts; i++)
03196 {
03197 if (!(share->state.rec_per_key_part[i]=param->rec_per_key_part[i]))
03198 share->state.changed|= STATE_NOT_ANALYZED;
03199 }
03200 }
03201 }
03202 if (update & (UPDATE_STAT | UPDATE_SORT | UPDATE_TIME | UPDATE_AUTO_INC))
03203 {
03204 if (update & UPDATE_TIME)
03205 {
03206 share->state.check_time= (long) time((time_t*) 0);
03207 if (!share->state.create_time)
03208 share->state.create_time=share->state.check_time;
03209 }
03210
03211
03212
03213
03214
03215
03216 if (info->lock_type == F_WRLCK)
03217 share->state.state= *info->state;
03218 if (mi_state_info_write(share->kfile,&share->state,1+2))
03219 goto err;
03220 share->changed=0;
03221 }
03222 {
03223 int error;
03224 uint32_t r_locks=share->r_locks,w_locks=share->w_locks;
03225 share->r_locks= share->w_locks= share->tot_locks= 0;
03226 error=_mi_writeinfo(info,WRITEINFO_NO_UNLOCK);
03227 share->r_locks=r_locks;
03228 share->w_locks=w_locks;
03229 share->tot_locks=r_locks+w_locks;
03230 if (!error)
03231 return 0;
03232 }
03233 err:
03234 mi_check_print_error(param,"%d when updating keyfile",errno);
03235 return 1;
03236 }
03237
03238
03239
03240
03241
03242
03243
03244
03245
03246
03247
03248
03249
03250
03251 void update_auto_increment_key(MI_CHECK *param, MI_INFO *info,
03252 bool repair_only)
03253 {
03254 unsigned char *record= 0;
03255
03256 if (!info->s->base.auto_key ||
03257 ! mi_is_key_active(info->s->state.key_map, info->s->base.auto_key - 1))
03258 {
03259 if (!(param->testflag & T_VERY_SILENT))
03260 mi_check_print_info(param,
03261 "Table: %s doesn't have an auto increment key\n",
03262 param->isam_file_name);
03263 return;
03264 }
03265 if (!(param->testflag & T_SILENT) &&
03266 !(param->testflag & T_REP))
03267 printf("Updating MyISAM file: %s\n", param->isam_file_name);
03268
03269
03270
03271
03272 if (!mi_alloc_rec_buff(info, SIZE_MAX, &record))
03273 {
03274 mi_check_print_error(param,"Not enough memory for extra record");
03275 return;
03276 }
03277
03278 mi_extra(info,HA_EXTRA_KEYREAD,0);
03279 if (mi_rlast(info, record, info->s->base.auto_key-1))
03280 {
03281 if (errno != HA_ERR_END_OF_FILE)
03282 {
03283 mi_extra(info,HA_EXTRA_NO_KEYREAD,0);
03284 free(mi_get_rec_buff_ptr(info, record));
03285 mi_check_print_error(param,"%d when reading last record",errno);
03286 return;
03287 }
03288 if (!repair_only)
03289 info->s->state.auto_increment=param->auto_increment_value;
03290 }
03291 else
03292 {
03293 uint64_t auto_increment= retrieve_auto_increment(info, record);
03294 set_if_bigger(info->s->state.auto_increment,auto_increment);
03295 if (!repair_only)
03296 set_if_bigger(info->s->state.auto_increment, param->auto_increment_value);
03297 }
03298 mi_extra(info,HA_EXTRA_NO_KEYREAD,0);
03299 free(mi_get_rec_buff_ptr(info, record));
03300 update_state_info(param, info, UPDATE_AUTO_INC);
03301 return;
03302 }
03303
03304
03305
03306
03307
03308
03309
03310
03311
03312
03313
03314
03315
03316
03317
03318
03319
03320
03321
03322
03323
03324
03325
03326
03327
03328
03329
03330
03331
03332
03333
03334
03335
03336
03337
03338
03339
03340
03341
03342
03343
03344
03345
03346
03347
03348
03349
03350
03351
03352
03353
03354
03355
03356 void update_key_parts(MI_KEYDEF *keyinfo, ulong *rec_per_key_part,
03357 uint64_t *unique, uint64_t *notnull,
03358 uint64_t records)
03359 {
03360 uint64_t count=0,tmp, unique_tuples;
03361 uint64_t tuples= records;
03362 uint32_t parts;
03363 for (parts=0 ; parts < keyinfo->keysegs ; parts++)
03364 {
03365 count+=unique[parts];
03366 unique_tuples= count + 1;
03367 if (notnull)
03368 {
03369 tuples= notnull[parts];
03370
03371
03372
03373
03374
03375 unique_tuples -= (records - notnull[parts]);
03376 }
03377
03378 if (unique_tuples == 0)
03379 tmp= 1;
03380 else if (count == 0)
03381 tmp= tuples;
03382 else
03383 tmp= (tuples + unique_tuples/2) / unique_tuples;
03384
03385
03386
03387
03388
03389 if (tmp < 1)
03390 tmp= 1;
03391 if (tmp >= (uint64_t) ~(ulong) 0)
03392 tmp=(uint64_t) ~(ulong) 0;
03393
03394 *rec_per_key_part=(ulong) tmp;
03395 rec_per_key_part++;
03396 }
03397 }
03398
03399
03400 static ha_checksum mi_byte_checksum(const unsigned char *buf, uint32_t length)
03401 {
03402 ha_checksum crc;
03403 const unsigned char *end=buf+length;
03404 for (crc=0; buf != end; buf++)
03405 crc=((crc << 1) + *((unsigned char*) buf)) +
03406 test(crc & (((ha_checksum) 1) << (8*sizeof(ha_checksum)-1)));
03407 return crc;
03408 }
03409
03410 static bool mi_too_big_key_for_sort(MI_KEYDEF *key, ha_rows rows)
03411 {
03412 uint32_t key_maxlength=key->maxlength;
03413 return (key->flag & (HA_BINARY_PACK_KEY | HA_VAR_LENGTH_KEY) &&
03414 ((uint64_t) rows * key_maxlength >
03415 (uint64_t) MAX_FILE_SIZE));
03416 }
03417
03418
03419
03420
03421
03422
03423
03424
03425
03426
03427 void mi_disable_non_unique_index(MI_INFO *info, ha_rows rows)
03428 {
03429 MYISAM_SHARE *share=info->s;
03430 MI_KEYDEF *key=share->keyinfo;
03431 uint32_t i;
03432
03433 assert(info->state->records == 0 &&
03434 (!rows || rows >= MI_MIN_ROWS_TO_DISABLE_INDEXES));
03435 for (i=0 ; i < share->base.keys ; i++,key++)
03436 {
03437 if (!(key->flag & (HA_NOSAME | HA_AUTO_KEY)) &&
03438 ! mi_too_big_key_for_sort(key,rows) && info->s->base.auto_key != i+1)
03439 {
03440 mi_clear_key_active(share->state.key_map, i);
03441 info->update|= HA_STATE_CHANGED;
03442 }
03443 }
03444 }
03445
03446
03447
03448
03449
03450
03451
03452
03453 bool mi_test_if_sort_rep(MI_INFO *info, ha_rows rows,
03454 uint64_t key_map, bool force)
03455 {
03456 MYISAM_SHARE *share=info->s;
03457 MI_KEYDEF *key=share->keyinfo;
03458 uint32_t i;
03459
03460
03461
03462
03463
03464 if (! mi_is_any_key_active(key_map))
03465 return false;
03466 for (i=0 ; i < share->base.keys ; i++,key++)
03467 {
03468 if (!force && mi_too_big_key_for_sort(key,rows))
03469 return false;
03470 }
03471 return true;
03472 }
03473
03474
03475 static void
03476 set_data_file_type(SORT_INFO *sort_info, MYISAM_SHARE *share)
03477 {
03478 if ((sort_info->new_data_file_type=share->data_file_type) ==
03479 COMPRESSED_RECORD && sort_info->param->testflag & T_UNPACK)
03480 {
03481 MYISAM_SHARE tmp;
03482
03483 if (share->options & HA_OPTION_PACK_RECORD)
03484 sort_info->new_data_file_type = DYNAMIC_RECORD;
03485 else
03486 sort_info->new_data_file_type = STATIC_RECORD;
03487
03488
03489 memcpy(&tmp, share, sizeof(*share));
03490 tmp.options= ~HA_OPTION_COMPRESS_RECORD;
03491 mi_setup_functions(&tmp);
03492 share->delete_record=tmp.delete_record;
03493 }
03494 }