Drizzled Public API Documentation

trans_log_ms.h

00001 /* Copyright (C) 2009 PrimeBase Technologies GmbH, Germany
00002  *
00003  * PrimeBase Media Stream for MySQL
00004  *
00005  * This program is free software; you can redistribute it and/or modify
00006  * it under the terms of the GNU General Public License as published by
00007  * the Free Software Foundation; either version 2 of the License, or
00008  * (at your option) any later version.
00009  *
00010  * This program is distributed in the hope that it will be useful,
00011  * but WITHOUT ANY WARRANTY; without even the implied warranty of
00012  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
00013  * GNU General Public License for more details.
00014  *
00015  * You should have received a copy of the GNU General Public License
00016  * along with this program; if not, write to the Free Software
00017  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
00018  *
00019  * Barry Leslie
00020  *
00021  * 2009-06-09
00022  *
00023  * H&G2JCtL
00024  *
00025  * PBMS transaction handling.
00026  *
00027  * PBMS uses 1 circular transaction log. All BLOB reference operations are written to this log
00028  * and are applied to the repository when committed. There is 1 thread dedicated to reading the 
00029  * transaction log and applying the changes. During an engine level backup this thread is suspended 
00030  * so that no transactions will be applied to the repository files as they are backed up.
00031  *
00032  */
00033  
00034 #pragma once
00035 #ifndef __TRANSLOG_MS_H__
00036 #define __TRANSLOG_MS_H__
00037 #include <stddef.h>
00038 
00039 #include "cslib/CSDefs.h"
00040 #include "cslib/CSFile.h"
00041 #define CHECK_TIDS
00042 #define CRASH_TEST
00043 
00044 #ifdef CRASH_TEST
00045 extern uint32_t trans_test_crash_point;
00046 #define MAX_CRASH_POINT 10
00047 #else
00048 #define MAX_CRASH_POINT 0
00049 #endif
00050 
00051 typedef uint32_t TRef;
00052 
00053 /*
00054   Transaction log info:
00055   
00056   The transaction log is a circular log of fixed length records. There is assumed to be one 
00057   reader thread and multiple writer threads. As records are written the 'eol' (End Of Log)
00058   marker is advanced and as they are read the 'start' marker is advanved. When iether marker
00059   reaches the end of the log a wrap around is done the marker is position back to the top of
00060   the list.
00061   
00062   When both markers are at the same location then the log is empty. The log is full if the 
00063   eol marker is just behind the start marker.
00064   
00065   If an overflow occurs then the overflow flag in the log header is set and records are written 
00066   to the end of the log. New records will continue to be written to the end of log until the 
00067   reader thread has read ALL of the records in the non overflow portion of the list. When all 
00068   of these records have been read then the list size will be adjusted to include the overflow
00069   record and the start and eol markers are repositioned and the overflow flag in the 
00070   header is switched off.
00071   
00072   
00073 */
00074 typedef struct MSDiskTransHead {
00075   CSDiskValue4      th_magic_4;         /* Table magic number. */
00076   CSDiskValue2      th_version_2;       /* The header version. */
00077 
00078   CSDiskValue4      th_next_txn_id_4;     /* The next valid transaction ID. */
00079 
00080   CSDiskValue2      th_check_point_2;     /* The frequency whith which the start/end positions are updated in the header. */
00081   
00082   CSDiskValue4      th_requested_cache_size_4;  /* The transaction cache list size in transaction. */
00083 
00084   CSDiskValue8      th_list_size_8;       /* The transaction log list size in records. */
00085   CSDiskValue8      th_requested_list_size_8; /* The desired list size. The log will be adjusted to this size as soon as it is convenient.*/
00086 
00087   CSDiskValue1      th_recovered_1;       /* A flag to indicate if the log was closed properly. */
00088 
00089   CSDiskValue1      th_overflow_1;        /* A flag to indicate if overflow has occurred. */
00090 
00091   // th_start_8 and th_eol_8 are always written at the same time.
00092   CSDiskValue8      th_start_8;         /* The index of the first valid record. */
00093   CSDiskValue8      th_eol_8;         /* The index of the first unused record or End Of Log (eol). */
00094   CSDiskValue1      th_checksum_1;        /* The current record checksum seed. */
00095 } MSDiskTransHeadRec, *MSDiskTransHeadPtr;
00096 
00097 
00098 typedef struct MSTrans_tag {
00099   uint32_t  tr_id;      // The transaction ID
00100   uint8_t tr_type;    // The transaction type. If the first bit is set then the transaction is an autocommit.
00101   uint32_t  tr_db_id;   // The database ID for the operation.
00102   uint32_t  tr_tab_id;    // The table ID for the operation.
00103   uint64_t  tr_blob_id;   // The blob ID for the operation.
00104   uint64_t  tr_blob_ref_id; // The blob reference id.
00105   uint8_t tr_check;   // The transaction record checksum.
00106 } MSTransRec, *MSTransPtr;
00107 
00108 
00109 typedef struct MSTransStats {
00110   uint64_t  ts_LogSize;     // The number of records in the transaction log.
00111   uint32_t  ts_PercentFull;   // The % of the desired log size in use. This can be > 100%.
00112   uint64_t  ts_MaxSize;     // The log size high water mark.
00113   uint32_t  ts_OverflowCount; // The number of times the log has overflowen.
00114   bool  ts_IsOverflowing;
00115   
00116   uint32_t ts_TransCacheSize; // The number of transactions currently in the cache.
00117   uint32_t ts_PercentTransCacheUsed;  // The number of transactions currently in the cache.
00118   uint32_t  ts_PercentCacheHit; // The % of the transactions that were cached on writing.
00119 } MSTransStatsRec, *MSTransStatsPtr;
00120 
00121 typedef enum {  MS_RollBackTxn = 0, 
00122         MS_PartialRollBackTxn,
00123         MS_CommitTxn, 
00124         MS_ReferenceTxn, 
00125         MS_DereferenceTxn, 
00126         MS_RecoveredTxn     
00127 } MS_Txn;
00128 
00129 typedef enum {  MS_Running = 0,
00130         MS_RolledBack, 
00131         MS_Committed, 
00132         MS_Recovered      
00133 } MS_TxnState;
00134 
00135 
00136 #define TRANS_SET_AUTOCOMMIT(t) (t |= 0X80) 
00137 #define TRANS_IS_AUTOCOMMIT(t) (t & 0X80) 
00138 
00139 #define TRANS_SET_START(t) (t |= 0X40)  
00140 #define TRANS_IS_START(t) (t & 0X40)  
00141 
00142 #define TRANS_TYPE_IS_TERMINATED(t) (((t) == MS_RollBackTxn) || ((t) == MS_CommitTxn) || ((t) == MS_RecoveredTxn))  
00143 #define TRANS_IS_TERMINATED(t) (TRANS_TYPE_IS_TERMINATED(TRANS_TYPE(t))  || TRANS_IS_AUTOCOMMIT(t)) 
00144 #define TRANS_TYPE(t) (t & 0X0F)  
00145 
00146 typedef bool (*CanContinueFunc)();
00147 typedef void (*LoadFunc)(uint64_t log_position, MSTransPtr rec);
00148 
00149 class MSTransCache;
00150 class MSTrans : public CSSharedRefObject {
00151 
00152 public:
00153   
00154   MSTrans();
00155   ~MSTrans();
00156   
00157   void txn_LogTransaction(MS_Txn type, bool autocommit = false, uint32_t db_id = 0, uint32_t tab_id = 0, uint64_t blob_id = 0, uint64_t blob_ref_id = 0);
00158 
00159   void txn_LogPartialRollBack(uint32_t rollBackCount)
00160   {
00161     /* Partial rollbacks store the rollback count in the place of the database id. */
00162     txn_LogTransaction(MS_PartialRollBackTxn, false, rollBackCount);
00163   }
00164   
00165   void txn_SetCheckPoint(uint16_t checkpoint)
00166   {
00167     enter_();
00168     
00169     // Important lock order. Writer threads never lock the reader but the reader
00170     // may lock this object so always lock the reader first.
00171     lock_(txn_reader);
00172     lock_(this);
00173     
00174     txn_MaxCheckPoint = checkpoint;
00175     
00176     if (txn_MaxCheckPoint < 10)
00177       txn_MaxCheckPoint = 10;
00178       
00179     if (txn_MaxCheckPoint > txn_MaxRecords)
00180       txn_MaxCheckPoint = txn_MaxRecords/2;
00181     
00182     if (txn_MaxCheckPoint > txn_ReqestedMaxRecords)
00183       txn_MaxCheckPoint = txn_ReqestedMaxRecords/2;
00184     
00185     CS_SET_DISK_2(txn_DiskHeader.th_check_point_2, txn_MaxCheckPoint);
00186     
00187     txn_File->write(&(txn_DiskHeader.th_check_point_2), offsetof(MSDiskTransHeadRec, th_check_point_2), 2);
00188     txn_File->flush();
00189     txn_File->sync();
00190     
00191     unlock_(this);
00192     unlock_(txn_reader);
00193     
00194     exit_();
00195   }
00196   
00197   void txn_SetCacheSize(uint32_t new_size);
00198   
00199   // txn_SetLogSize() may not take effect immediately but will be done
00200   // when there is free space at the end of the log.
00201   void txn_SetLogSize(uint64_t new_size);
00202   
00203   void txn_Close(); 
00204   
00205   uint64_t  txn_GetSize();    // Returns the size of the log in transaction records.
00206   
00207   uint64_t  txn_GetNumRecords() // Returns the number of transactions records waiting to be processed.
00208   {             // This doesn't include overflow.
00209     uint64_t size;
00210     if (txn_Start == txn_EOL)
00211       size = 0;
00212     else if (txn_Start < txn_EOL) 
00213       size = txn_EOL - txn_Start;
00214     else 
00215       size = txn_MaxRecords - (txn_Start - txn_EOL);
00216       
00217     return size;
00218   }
00219 
00220   // While a backup is in progress the transaction thread will not be signaled 
00221   // about completed transactions.
00222   void txn_BackupStarting() 
00223   {
00224     txn_Doingbackup = true;
00225     txn_reader->suspend();
00226   }
00227   
00228   bool txn_haveNextTransaction();
00229   
00230   void txn_BackupCompleted()
00231   {
00232     txn_Doingbackup = false;
00233     txn_reader->resume();
00234   }
00235   
00236   // The following should only be called by the transaction processing thread.
00237   
00238   // txn_GetNextTransaction() gets the next completed transaction.
00239   // If there is none ready it waits for one. 
00240   void txn_GetNextTransaction(MSTransPtr tran, MS_TxnState *state); 
00241     
00242   void txn_SetReader(CSDaemon *reader) {txn_reader = reader;}
00243   
00244   // Search the transaction log for a MS_ReferenceTxn record for the given BLOB.
00245   bool txn_FindBlobRef(MS_TxnState *state, uint32_t db_id, uint32_t tab_id, uint64_t blob_id);
00246   
00247   // Mark all transactions for a given database as dropped. Including commited transactions.
00248   void txn_dropDatabase(uint32_t db_id);
00249   
00250 
00251   uint64_t txn_GetStartPosition() { return txn_Start;}
00252   
00253   const char  *txn_GetTXNLogPath() {return txn_File->myFilePath->getCString();}
00254 private:
00255   friend class ReadTXNLog;
00256   
00257   uint16_t    txn_MaxCheckPoint;  // The maximum records to be written ore read before the positions in the header are updated.
00258 
00259   // These fields are only used by the reader thread:
00260   bool    txn_Doingbackup;// Is the database being backed up.
00261   CSDaemon  *txn_reader;  // THe transaction log reader daemon. (unreferenced)
00262   bool    txn_IsTxnValid; // Is the current transaction valid.
00263   TRef    txn_CurrentTxn; // The current transaction.
00264   uint32_t    txn_TxnIndex; // The record index into the current transaction.
00265   int32_t   txn_StartCheckPoint; // Counter to determin when the read position should be flushed.
00266   
00267   void txn_PerformIdleTasks();
00268   
00269   MSTransCache  *txn_TransCache;  // Transaction cache
00270   
00271   void txn_ResizeLog();
00272   
00273   void txn_NewTransaction(); // Clears the old transaction ID
00274   
00275   bool txn_IsFull()
00276   {
00277     return (txn_HaveOverflow || ((txn_GetNumRecords() +1) == txn_MaxRecords));
00278   }
00279   
00280   
00281   uint32_t        txn_BlockingTransaction; // The transaction ID the transaction thread is waiting on.
00282 
00283   MSDiskTransHeadRec  txn_DiskHeader;
00284   CSFile        *txn_File;
00285   
00286   int32_t       txn_EOLCheckPoint; // Counter to determin when the EOL position should be flushed.
00287   
00288   // The size of the transaction log can be adjusted by setting txn_ReqestedMaxRecords.
00289   // The log size will be adjusted as soon as there are free slots at the bottom of the list.
00290   uint64_t        txn_MaxRecords;     // The number of record slots in the current list.
00291   uint64_t        txn_ReqestedMaxRecords; // The number of record slots requested.    
00292 
00293   uint64_t        txn_HighWaterMark; // Keeps track of the log size high water mark.
00294   uint64_t        txn_OverflowCount; // A count of the number of times the transaction log has over flown.
00295 #ifdef DEBUG  
00296 public:
00297   void        txn_DumpLog(const char *file);
00298 #endif
00299   uint32_t        txn_MaxTID;
00300   bool        txn_Recovered;        // Has the log been recovered.
00301   bool        txn_HaveOverflow;     // A flag to indicate the list has overfown.
00302   uint64_t        txn_Overflow;       // The index of the next overflow record. 
00303   uint64_t        txn_EOL;          // The index of the first unused record or End Of Log (eol). 
00304   uint64_t        txn_Start;          // The index of the first valid record. 
00305 
00306 public: 
00307   void txn_GetStats(MSTransStatsPtr stats);   // Get the current performance statistics.
00308   
00309 private:
00310   uint8_t       txn_Checksum;       // The current record checksum seed. 
00311   
00312   void txn_SetFile(CSFile *tr_file);    // Set the file to use for the transaction log.
00313   bool txn_ValidRecord(MSTransPtr rec); // Check to see if a record is valid.
00314   void txn_GetRecordAt(uint64_t index, MSTransPtr rec); // Reads 1 record from the log.
00315   void txn_ResetReadPosition(uint64_t pos); // Reset txn_Start
00316   void txn_ResetEOL();
00317     
00318   void txn_Recover();             // Recover the transaction log.
00319   
00320   void txn_ReadLog(uint64_t read_start, bool log_locked, CanContinueFunc canContinue, LoadFunc load); // A generic method for reading the log
00321   void txn_LoadTransactionCache(uint64_t read_start); // Load the transactions in the log into cache.
00322   
00323   void txn_AddTransaction(uint8_t tran_type, bool autocommit = false, uint32_t db_id = 0, uint32_t tab_id = 0, uint64_t blob_id = 0, uint64_t blob_ref_id = 0);
00324 
00325   
00326 public:
00327   static MSTrans* txn_NewMSTrans(const char *log_path, bool dump_log = false);
00328 };
00329 
00330 class ReadTXNLog {
00331   public:
00332   ReadTXNLog(MSTrans *txn_log): rl_log(txn_log){}
00333   virtual ~ReadTXNLog(){}
00334     
00335   MSTrans *rl_log;
00336   void rl_ReadLog(uint64_t read_start, bool log_locked);
00337   virtual bool rl_CanContinue();
00338   virtual void rl_Load(uint64_t log_position, MSTransPtr rec);
00339   void rl_Store(uint64_t log_position, MSTransPtr rec);
00340   void rl_Flush();
00341 };
00342 
00343 #endif //__TRANSLOG_MS_H__