Drizzled Public API Documentation

systab_dump_ms.cc

00001 /* Copyright (C) 2009 PrimeBase Technologies GmbH, Germany
00002  *
00003  * PrimeBase Media Stream for MySQL
00004  *
00005  * This program is free software; you can redistribute it and/or modify
00006  * it under the terms of the GNU General Public License as published by
00007  * the Free Software Foundation; either version 2 of the License, or
00008  * (at your option) any later version.
00009  *
00010  * This program is distributed in the hope that it will be useful,
00011  * but WITHOUT ANY WARRANTY; without even the implied warranty of
00012  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
00013  * GNU General Public License for more details.
00014  *
00015  * You should have received a copy of the GNU General Public License
00016  * along with this program; if not, write to the Free Software
00017  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
00018  *
00019  * Barry Leslie
00020  *
00021  * System dump table.
00022  *
00023  */
00024 #ifdef DRIZZLED
00025 #include <config.h>
00026 #include <drizzled/common.h>
00027 #include <drizzled/session.h>
00028 #include <drizzled/field/blob.h>
00029 #endif
00030 
00031 #include "cslib/CSConfig.h"
00032 
00033 #include <sys/types.h>
00034 #include <sys/stat.h>
00035 #include <stdlib.h>
00036 #include <time.h>
00037 
00038 
00039 //#include "mysql_priv.h"
00040 #include "cslib/CSGlobal.h"
00041 #include "cslib/CSStrUtil.h"
00042 
00043 #include "ha_pbms.h"
00044 //#include <plugin.h>
00045 
00046 #include "mysql_ms.h"
00047 #include "repository_ms.h"
00048 #include "database_ms.h"
00049 #include "compactor_ms.h"
00050 #include "open_table_ms.h"
00051 #include "discover_ms.h"
00052 #include "transaction_ms.h"
00053 #include "systab_variable_ms.h"
00054 #include "backup_ms.h"
00055 
00056 
00057 #include "systab_dump_ms.h"
00058 
00059 
00060 DT_FIELD_INFO pbms_dump_info[]=
00061 {
00062   {"Data",      NOVAL, NULL, MYSQL_TYPE_LONG_BLOB,  &my_charset_bin,  NOT_NULL_FLAG,  "A BLOB repository record"},
00063   {NULL,NOVAL, NULL, MYSQL_TYPE_STRING,NULL, 0, NULL}
00064 };
00065 
00066 DT_KEY_INFO pbms_dump_keys[]=
00067 {
00068   {NULL, 0, {NULL}}
00069 };
00070 
00071 
00072 /*
00073  * -------------------------------------------------------------------------
00074  * DUMP TABLE
00075  */
00076 //-----------------------
00077 MSDumpTable::MSDumpTable(MSSystemTableShare *share, TABLE *table):
00078   MSRepositoryTable(share, table)
00079 {
00080 }
00081 
00082 //-----------------------
00083 MSDumpTable::~MSDumpTable()
00084 {
00085 }
00086 
00087 //-----------------------
00088 void MSDumpTable::use()
00089 { 
00090   dt_hasInfo = dt_hasCompleted = dt_haveCloudInfo = false;
00091   dt_headerSize = 0;
00092   
00093   // Suspend the transaction writer while the dump is running.
00094   MSTransactionManager::suspend(true);
00095 
00096   MSRepositoryTable::use();
00097 }
00098 
00099 //-----------------------
00100 void MSDumpTable::unuse()
00101 {
00102   MSBackupInfo *backupInfo;
00103   
00104   backupInfo = myShare->mySysDatabase->myBlobCloud->cl_getBackupInfo();
00105   if (backupInfo) {
00106     enter_();
00107     push_(backupInfo);
00108     myShare->mySysDatabase->myBlobCloud->cl_clearBackupInfo();
00109     if (backupInfo->isBackupRunning()) {
00110       if (dt_hasCompleted) 
00111         backupInfo->backupCompleted(RETAIN(myShare->mySysDatabase));
00112       else
00113         backupInfo->backupTerminated(RETAIN(myShare->mySysDatabase));
00114     }
00115     release_(backupInfo);
00116     outer_();
00117   }
00118   
00119   MSTransactionManager::resume();
00120   MSRepositoryTable::unuse();
00121 }
00122 
00123 //-----------------------
00124 void MSDumpTable::seqScanInit()
00125 {
00126   dt_hasInfo = dt_hasCompleted = false;
00127   return MSRepositoryTable::seqScanInit();
00128 }
00129 //-----------------------
00130 bool MSDumpTable::seqScanNext(char *buf)
00131 {
00132   if (!dt_hasInfo) {
00133     dt_hasInfo = true;
00134     return returnInfoRow(buf);
00135   }
00136   // Reset the position
00137   if (!MSRepositoryTable::seqScanNext(buf)) 
00138     dt_hasCompleted = true;
00139   
00140   return !dt_hasCompleted;
00141 }
00142 
00143 //-----------------------
00144 bool MSDumpTable::returnDumpRow(char *record, uint64_t record_size, char *buf)
00145 {
00146   TABLE   *table = mySQLTable;
00147   Field   *curr_field;
00148   byte    *save;
00149   MY_BITMAP *save_write_set;
00150 
00151 
00152 
00153 
00154   /* ASSERT_COLUMN_MARKED_FOR_WRITE is failing when
00155    * I use store()!??
00156    * But I want to use it! :(
00157    */
00158   save_write_set = table->write_set;
00159   table->write_set = NULL;
00160 #ifdef DRIZZLED
00161   memset(buf, 0xFF, table->getNullBytes());
00162 #else
00163   memset(buf, 0xFF, table->s->null_bytes);
00164 #endif
00165   
00166   for (Field **field=GET_TABLE_FIELDS(table) ; *field ; field++) {
00167     curr_field = *field;
00168 
00169     save = curr_field->ptr;
00170 #if MYSQL_VERSION_ID < 50114
00171     curr_field->ptr = (byte *) buf + curr_field->offset();
00172 #else
00173 #ifdef DRIZZLED
00174     curr_field->ptr = (byte *) buf + curr_field->offset(curr_field->getTable()->getInsertRecord());
00175 #else
00176     curr_field->ptr = (byte *) buf + curr_field->offset(curr_field->table->record[0]);
00177 #endif
00178 #endif
00179     switch (curr_field->field_name[0]) {
00180       case 'D':
00181       case 'd':
00182         // Data         LONGBLOB
00183         ASSERT(strcmp(curr_field->field_name, "Data") == 0);
00184         if (record_size <= 0xFFFFFFF) {
00185           ((Field_blob *) curr_field)->set_ptr(record_size, (byte *) record);
00186           setNotNullInRecord(curr_field, buf);
00187         }
00188         break;
00189     }
00190     curr_field->ptr = save;
00191   }
00192 
00193   table->write_set = save_write_set;
00194   return true;
00195 }
00196 
00197 //-----------------------
00198 bool MSDumpTable::returnRow(MSBlobHeadPtr blob, char *buf)
00199 {
00200   uint64_t    record_size, blob_repo_size;
00201   uint16_t    ref_size, ref_count, refs = 0, table_refs = 0, header_size;
00202   uint8_t   blob_storage_type;
00203   MSRepoPointersRec ptr;
00204   MSDatabase *myDB = myShare->mySysDatabase;
00205   enter_();
00206 
00207   // Reset the references for the BLOB and recreate
00208   // the temp log references.
00209   ref_count = CS_GET_DISK_2(blob->rb_ref_count_2);
00210   ref_size = CS_GET_DISK_1(blob->rb_ref_size_1);
00211 
00212   blob_storage_type = CS_GET_DISK_1(blob->rb_storage_type_1);
00213 
00214   header_size = CS_GET_DISK_2(blob->rb_head_size_2);
00215   blob_repo_size = CS_GET_DISK_6(blob->rb_blob_repo_size_6);
00216   
00217   iBlobBuffer->setLength(header_size);
00218   iRepoFile->read(iBlobBuffer->getBuffer(0), iRepoOffset, (size_t) header_size, header_size);
00219 
00220   // First check to see if the BLOB is referenced
00221   ptr.rp_chars = iBlobBuffer->getBuffer(0) + dt_headerSize;
00222   for (int count = 0; count < ref_count; count++) {
00223     int ref_type = CS_GET_DISK_2(ptr.rp_ref->rr_type_2);
00224     
00225     switch (ref_type) {
00226       case MS_BLOB_TABLE_REF:
00227         table_refs++;
00228         break;
00229         
00230       case MS_BLOB_FREE_REF:
00231       case MS_BLOB_DELETE_REF:
00232         break;
00233     
00234       default: // Assumed to be a MSRepoBlobRefRec.
00235         // Only committed references are backed up.
00236         if (IS_COMMITTED(CS_GET_DISK_8(ptr.rp_blob_ref->er_blob_ref_id_8))) {
00237           refs++;
00238         } else {
00239           CS_SET_DISK_2(ptr.rp_ref->rr_type_2, MS_BLOB_FREE_REF);
00240         }
00241         
00242         break;
00243     }
00244     
00245     ptr.rp_chars += ref_size;   
00246   }
00247   
00248   
00249   if (refs && table_refs) { // Unreferenced BLOBs are ignored.
00250     if (blob_storage_type == MS_CLOUD_STORAGE) {
00251       CloudKeyRec cloud_key;
00252       MSRepoFile::getBlobKey(blob, &cloud_key);
00253       myDB->myBlobCloud->cl_backupBLOB(&cloud_key);
00254       record_size = header_size;
00255     } else {
00256       record_size = header_size + blob_repo_size;
00257       iBlobBuffer->setLength(record_size);
00258       iRepoFile->read(iBlobBuffer->getBuffer(header_size), iRepoOffset + header_size, (size_t) blob_repo_size, blob_repo_size);
00259     }
00260   } else {
00261     record_size = 0; // An empty record is returned for unreferenced BLOBs.
00262   }
00263 
00264 
00265   return_(returnDumpRow(iBlobBuffer->getBuffer(0), record_size, buf));
00266 }
00267 
00268 //-----------------------
00269 #define INC_INFO_SPACE(i) record_size+=i; space-=i;ptr+=i;
00270 #define MS_DUMP_MAGIC 0x5A74C1EB
00271 typedef struct {
00272   CSDiskValue4 ti_table_id_4;
00273   char     ti_name[1]; // variable length buffer
00274 } TabInfoRec, *TabInfoPtr;
00275 
00276 typedef struct {
00277   CSDiskValue4 di_magic_4;
00278   CSDiskValue2 di_header_size_2;
00279 } RepInfoRec, *RepInfoPtr;
00280 
00281 // Repository DUMP info record format: 
00282 // <Dump magic><BLOB header size><database ID><backup number><sysTables size><sysTables dump>[<table ID><table name>]...
00283 bool MSDumpTable::returnInfoRow(char *buf)
00284 {
00285   uint64_t    record_size = 0, space = 1024;
00286   char    *ptr;
00287   MSTable   *tab;
00288   uint32_t    space_needed, next_tab = 0, cloudRef, cloudbackupNo, backupRef;
00289   RepInfoPtr  rep_info;
00290   TabInfoPtr  tab_info;
00291   CSStringBuffer *sysTablesDump;
00292   MSBackupInfo *backupInfo;
00293   CSDiskData  d;
00294   enter_();
00295 
00296   // Setup the sysvar table with the cloud backup number then dump it.
00297   if (myShare->mySysDatabase->myBlobType == MS_CLOUD_STORAGE) {
00298     cloudbackupNo = myShare->mySysDatabase->myBlobCloud->cl_getNextBackupNumber();
00299     cloudRef = myShare->mySysDatabase->myBlobCloud->cl_getDefaultCloudRef();
00300   } else {
00301     // It is still possible that the database contains BLOBs in cloud storage
00302     // even if it isn't currently flaged to use cloud storage.
00303     cloudbackupNo = cloudRef = 0;
00304   }
00305 
00306   backupInfo = MSBackupInfo::startDump(RETAIN(myShare->mySysDatabase), cloudRef, cloudbackupNo);
00307   backupRef = backupInfo->getBackupRefId();
00308   myShare->mySysDatabase->myBlobCloud->cl_setBackupInfo(backupInfo);  
00309   
00310   dt_cloudbackupDBID = myShare->mySysDatabase->myDatabaseID;
00311   
00312   sysTablesDump = PBMSSystemTables::dumpSystemTables(RETAIN(myShare->mySysDatabase));
00313   push_(sysTablesDump);
00314   
00315   iBlobBuffer->setLength(space + sysTablesDump->length() + 4 + 4);
00316   ptr = iBlobBuffer->getBuffer(0);
00317   rep_info = (RepInfoPtr) iBlobBuffer->getBuffer(0);
00318   dt_headerSize = sizeof(MSBlobHeadRec);
00319   
00320   
00321   CS_SET_DISK_4(rep_info->di_magic_4, MS_DUMP_MAGIC);
00322   CS_SET_DISK_2(rep_info->di_header_size_2, dt_headerSize);
00323   
00324   INC_INFO_SPACE(sizeof(RepInfoRec));
00325   
00326   d.rec_chars = ptr;
00327   CS_SET_DISK_4(d.int_val->val_4, dt_cloudbackupDBID);
00328   INC_INFO_SPACE(4);
00329   
00330   d.rec_chars = ptr;
00331   CS_SET_DISK_4(d.int_val->val_4, backupRef);
00332   INC_INFO_SPACE(4);
00333   
00334   // Add the system tables to the dump
00335   d.rec_chars = ptr;
00336   CS_SET_DISK_4(d.int_val->val_4, sysTablesDump->length());
00337   INC_INFO_SPACE(4);
00338   memcpy(ptr, sysTablesDump->getBuffer(0), sysTablesDump->length());
00339   INC_INFO_SPACE(sysTablesDump->length());
00340   release_(sysTablesDump);
00341   sysTablesDump = NULL;
00342       
00343   tab_info = (TabInfoPtr)ptr;
00344   
00345   // Get a list of the tables containing BLOB references. 
00346   while ((tab = myShare->mySysDatabase->getNextTable(&next_tab))) {
00347     push_(tab);
00348     space_needed = tab->myTableName->length() + 5;
00349     if (space < space_needed) {
00350       space += 1024;
00351       iBlobBuffer->setLength(space);
00352       ptr = iBlobBuffer->getBuffer(0) + record_size;
00353     }
00354     
00355     tab_info = (TabInfoPtr)ptr;
00356     CS_SET_DISK_4(tab_info->ti_table_id_4, tab->myTableID);
00357     strcpy(tab_info->ti_name, tab->myTableName->getCString());
00358     INC_INFO_SPACE(space_needed);
00359     
00360     release_(tab);
00361   }
00362   
00363   return_(returnDumpRow(iBlobBuffer->getBuffer(0), record_size, buf));
00364 }
00365 
00366 #define INC_INFO_REC(i) info_buffer+=i; length-=i; tab_info = (TabInfoPtr) info_buffer;
00367 //-----------------------
00368 void MSDumpTable::setUpRepository(const char *info_buffer, uint32_t length)
00369 {
00370   uint32_t tab_id, magic;
00371   MSDatabase *myDB = myShare->mySysDatabase;
00372   RepInfoPtr  rep_info = (RepInfoPtr) info_buffer;
00373   TabInfoPtr  tab_info;
00374   uint32_t    sys_size, backupRefID;
00375   MSBackupInfo *backupInfo; 
00376   CSDiskData  d;
00377   
00378   if (length < sizeof(RepInfoRec)) {
00379     CSException::throwException(CS_CONTEXT, CS_ERR_INVALID_RECORD, "Invalid repository info record.");
00380   }
00381 
00382   magic = CS_GET_DISK_4(rep_info->di_magic_4);
00383   if (CS_GET_DISK_4(rep_info->di_magic_4) != MS_DUMP_MAGIC) {
00384     CSException::throwException(CS_CONTEXT, CS_ERR_BAD_HEADER_MAGIC, "Invalid repository info record.");
00385   }
00386   
00387   dt_headerSize = CS_GET_DISK_2(rep_info->di_header_size_2);
00388   INC_INFO_REC(sizeof(RepInfoRec));
00389   
00390   d.rec_cchars = info_buffer;
00391   dt_cloudbackupDBID = CS_GET_DISK_4(d.int_val->val_4);
00392   INC_INFO_REC(4);
00393   
00394   // Get the backup information
00395   d.rec_cchars = info_buffer;
00396   backupRefID = CS_GET_DISK_4(d.int_val->val_4);
00397   INC_INFO_REC(4);
00398   
00399   // If the backup information is missing then the restore may still
00400   // be able to complete so long as cloud storage was not used.
00401   backupInfo = MSBackupInfo::findBackupInfo(backupRefID);
00402   if (backupInfo)  {
00403     myShare->mySysDatabase->myBlobCloud->cl_setBackupInfo(backupInfo);
00404     dt_haveCloudInfo = true;
00405   }
00406   
00407   // Restore the System table.
00408   d.rec_cchars = info_buffer;
00409   sys_size = CS_GET_DISK_4(d.int_val->val_4);
00410   INC_INFO_REC(4);
00411   
00412   PBMSSystemTables::restoreSystemTables(RETAIN(myDB), info_buffer, sys_size);
00413   INC_INFO_REC(sys_size);
00414 
00415   while (length > 5) {
00416     tab_id = CS_GET_DISK_4(tab_info->ti_table_id_4);
00417     myDB->addTable(tab_id, tab_info->ti_name, 0, false);
00418     INC_INFO_REC(strlen(tab_info->ti_name) +5);
00419   }
00420   
00421   if (length)
00422     CSException::throwException(CS_CONTEXT, CS_ERR_INVALID_RECORD, "Invalid repository info record.");    
00423 }
00424 
00425 
00426 //-----------------------
00427 void MSDumpTable::insertRow(char *buf)
00428 { 
00429   TABLE *table = mySQLTable;
00430   Field_blob *field;
00431   uint32_t packlength, length;
00432   const char *blob_rec, *blob_ptr;
00433   
00434   field = (Field_blob *)GET_FIELD(table, 0);
00435   
00436     /* Get the blob record: */
00437 #ifdef DRIZZLED
00438     blob_rec= buf + field->offset(table->getInsertRecord());
00439     packlength= field->pack_length() - table->getShare()->sizeBlobPtr();
00440 #else
00441     blob_rec= buf + field->offset(table->record[0]);
00442     packlength= field->pack_length() - table->s->sizeBlobPtr();
00443 #endif
00444 
00445     memcpy(&blob_ptr, blob_rec +packlength, sizeof(char*));
00446     length= field->get_length();
00447   
00448   if (!dt_hasInfo) {
00449     setUpRepository(blob_ptr, length);
00450     dt_hasInfo = true;
00451   } else
00452     insertRepoRow((MSBlobHeadPtr)blob_ptr, length);
00453   
00454 }
00455 
00456 //-----------------------
00457 void MSDumpTable::insertRepoRow(MSBlobHeadPtr blob, uint32_t length)
00458 { 
00459   MSRepository *repo;
00460   MSRepoFile *repo_file;
00461   uint64_t    repo_offset;
00462   uint64_t    blob_data_size;
00463   uint32_t    auth_code;
00464   uint16_t ref_size, ref_count, refs = 0, table_refs = 0;
00465   uint8_t   blob_storage_type;
00466   MSRepoPointersRec ptr;
00467   MSDatabase *myDB = myShare->mySysDatabase;
00468   CloudKeyRec cloud_key;
00469   enter_();
00470 
00471   if (!length)
00472     exit_();
00473   
00474   if (length != (CS_GET_DISK_2(blob->rb_head_size_2) + CS_GET_DISK_6(blob->rb_blob_repo_size_6))) {
00475     CSException::throwException(CS_CONTEXT, MS_ERR_INVALID_RECORD, "Damaged Repository record");
00476   }
00477   
00478   // Get a repository file.
00479   repo = myDB->lockRepo(length);
00480   frompool_(repo);
00481   
00482   repo_file = myDB->getRepoFileFromPool(repo->myRepoID, false);
00483   frompool_(repo_file);
00484 
00485   repo_offset = repo->myRepoFileSize;
00486   
00487   // Reset the references for the BLOB and recreate
00488   // the temp log references.
00489   auth_code = CS_GET_DISK_4(blob->rb_auth_code_4);
00490   ref_count = CS_GET_DISK_2(blob->rb_ref_count_2);
00491   ref_size = CS_GET_DISK_1(blob->rb_ref_size_1);
00492   blob_data_size = CS_GET_DISK_6(blob->rb_blob_data_size_6);
00493 
00494   blob_storage_type = CS_GET_DISK_1(blob->rb_storage_type_1);
00495   if (blob_storage_type == MS_CLOUD_STORAGE) {
00496     MSRepoFile::getBlobKey(blob, &cloud_key);
00497   }
00498 
00499   // First check to see if the BLOB is referenced
00500   ptr.rp_chars = ((char*) blob) + dt_headerSize;
00501   for (int count = 0; count < ref_count; count++) {
00502     int ref_type = CS_GET_DISK_2(ptr.rp_ref->rr_type_2);
00503     
00504     switch (ref_type) {
00505       case MS_BLOB_TABLE_REF:
00506         table_refs++;
00507         break;
00508         
00509       case MS_BLOB_FREE_REF:
00510       case MS_BLOB_DELETE_REF:
00511         break;
00512     
00513       default: // Assumed to be a MSRepoBlobRefRec.
00514         // Only committed references are backed up.
00515         if (IS_COMMITTED(CS_GET_DISK_8(ptr.rp_blob_ref->er_blob_ref_id_8))) {
00516           refs++;
00517         } else {
00518           CS_SET_DISK_2(ptr.rp_ref->rr_type_2, MS_BLOB_FREE_REF);
00519         }
00520         
00521         break;
00522     }
00523     
00524     ptr.rp_chars += ref_size;   
00525   }
00526   
00527   
00528   if (refs && table_refs) { // Unreferenced BLOBs are ignored.
00529   
00530   
00531     // Set table references.
00532     ptr.rp_chars = ((char*) blob) + dt_headerSize;
00533     for (int count = 0; count < ref_count; count++) {
00534       int ref_type = CS_GET_DISK_2(ptr.rp_ref->rr_type_2);
00535       MSOpenTable *otab;
00536       uint32_t    tab_id;
00537       uint64_t    blob_id;
00538       
00539       switch (ref_type) {
00540         case MS_BLOB_TABLE_REF:
00541           tab_id = CS_GET_DISK_4(ptr.rp_tab_ref->tr_table_id_4);
00542           blob_id = CS_GET_DISK_6(ptr.rp_tab_ref->tr_blob_id_6);
00543           otab = MSTableList::getOpenTableByID(myDB->myDatabaseID, tab_id);
00544       
00545           frompool_(otab);
00546           otab->getDBTable()->setBlobHandle(otab, blob_id, repo->myRepoID, repo_offset, blob_data_size, dt_headerSize, auth_code);
00547           backtopool_(otab);
00548           break;
00549           
00550         case MS_BLOB_DELETE_REF:
00551           break;
00552                 
00553         case MS_BLOB_FREE_REF:
00554         default: 
00555           break;
00556       }
00557         
00558       ptr.rp_chars += ref_size;   
00559     } 
00560   
00561     // Write the repository record.
00562     repo_file->write(blob, repo_offset, length);
00563     repo->myRepoFileSize += length;
00564     
00565 #ifdef HAVE_ALIAS_SUPPORT
00566     uint16_t alias_offset;
00567     if (alias_offset = CS_GET_DISK_2(blob->rb_alias_offset_2)) { 
00568       myDB->registerBlobAlias(repo->myRepoID, repo_offset, ((char*)blob) + alias_offset);
00569     }
00570 #endif    
00571     if (blob_storage_type == MS_CLOUD_STORAGE) {
00572       if (!dt_haveCloudInfo) {
00573         CSException::throwException(CS_CONTEXT, MS_ERR_MISSING_CLOUD_REFFERENCE, "Missing cloud backup information.");
00574       }
00575       myDB->myBlobCloud->cl_restoreBLOB(&cloud_key, dt_cloudbackupDBID);
00576     }
00577   }
00578 
00579   backtopool_(repo_file);
00580   backtopool_(repo);
00581   exit_();
00582 }
00583 
00584 
00585