00001
00002
00003
00004
00005
00006
00007
00008
00009
00010
00011
00012
00013
00014
00015
00016
00017
00018
00019
00020
00021
00022
00023
00024
00025
00026
00027
00028
00029
00030 #pragma once
00031 #ifndef __REPOSITORY_MS_H__
00032 #define __REPOSITORY_MS_H__
00033 #include <stddef.h>
00034
00035 #include "cslib/CSDefs.h"
00036 #include "cslib/CSFile.h"
00037 #include "cslib/CSMd5.h"
00038 #include "engine_ms.h"
00039 #include "cloud_ms.h"
00040 #include "pbmslib.h"
00041
00042 #define MS_BLOB_HEADER_MAGIC 0x9213BA24
00043 #define MS_REPO_FILE_MAGIC 0x5678CDEF
00044 #define MS_REPO_FILE_VERSION 3
00045 #define MS_REPO_FILE_HEAD_SIZE 128
00046
00047 #ifdef DEBUG
00048 #define MS_REPO_MIN_REF_COUNT 3 // Initial number of references to allow space for:(Table, Delete, Cloud)
00049 #define MS_REPO_MIN_MATADATA 0
00050 #else
00051 #define MS_REPO_MIN_REF_COUNT 6 // Initial number of references to allow space for.
00052 #define MS_REPO_MIN_MATADATA 128
00053 #endif
00054
00055 #define BLOB_IN_REPOSITORY(t) ( t < MS_CLOUD_STORAGE)
00056 #define BLOB_IN_CLOUD(t) ( t == MS_CLOUD_STORAGE)
00057
00058
00059
00060
00061 #define COMMIT_MASK(id) ((id) & 0X7FFFFFFFFFFFFFFFll) // The high bit is used internally to flag uncommitted references.
00062 #define IS_COMMITTED(id) (((id) & 0X8000000000000000ll) == 0)
00063 #define UNCOMMITTED(id) ((id) | 0X8000000000000000ll)
00064
00065 class MSOpenTable;
00066 class MSDatabase;
00067 class MSRepository;
00068 class CSHTTPOutputStream;
00069
00070
00071
00072
00073
00074
00075
00076
00077
00078
00079
00080
00081
00082
00083
00084
00085 typedef struct MSRepoHead {
00086 CSDiskValue4 rh_magic_4;
00087 CSDiskValue2 rh_version_2;
00088 CSDiskValue2 rh_repo_head_size_2;
00089 CSDiskValue2 rh_blob_head_size_2;
00090 CSDiskValue2 rh_def_ref_size_2;
00091 CSDiskValue8 rh_garbage_count_8;
00092
00093
00094
00095
00096 CSDiskValue8 rh_recovery_offset_8;
00097 CSDiskValue4 rh_last_temp_time_4;
00098 CSDiskValue4 rh_last_access_4;
00099 CSDiskValue4 rh_create_time_4;
00100 CSDiskValue4 rh_last_ref_4;
00101
00102 CSDiskValue4 rh_reserved_4;
00103 } MSRepoHeadRec, *MSRepoHeadPtr;
00104
00105 #define MS_BLOB_ALLOCATED 1
00106 #define MS_BLOB_REFERENCED 2
00107 #define MS_BLOB_DELETED 3
00108 #define MS_BLOB_MOVED 4
00109
00110
00111 #define VALID_BLOB_STATUS(s) (s >= MS_BLOB_ALLOCATED && s <= MS_BLOB_MOVED)
00112 #define IN_USE_BLOB_STATUS(s) (s >= MS_BLOB_ALLOCATED && s <= MS_BLOB_REFERENCED)
00113
00114 #define MS_SHORT_AUTH_CODE(ac) ((uint16_t) (((ac) & 0x0000FFFF) ^ (ac) >> 16))
00115
00116
00117
00118
00119
00120
00121
00122
00123
00124
00125
00126
00127 typedef struct MSBlobHead {
00128
00129
00130
00131
00132 CSDiskValue4 rb_last_access_4;
00133 CSDiskValue4 rb_access_count_4;
00134 CSDiskValue4 rb_create_time_4;
00135 CSDiskValue4 rd_magic_4;
00136 CSDiskValue1 rb_storage_type_1;
00137
00138 CSDiskValue2 rb_ref_count_2;
00139 CSDiskValue1 rb_ref_size_1;
00140 CSDiskValue4 rb_mod_time_4;
00141
00142
00143
00144
00145
00146
00147 CSDiskValue2 rb_head_size_2;
00148 CSDiskValue6 rb_blob_repo_size_6;
00149 CSDiskValue6 rb_blob_data_size_6;
00150 Md5Digest rb_blob_checksum_md5d;
00151
00152 CSDiskValue4 rb_alias_hash_4;
00153 CSDiskValue2 rb_alias_offset_2;
00154 CSDiskValue2 rb_mdata_offset_2;
00155 CSDiskValue2 rb_mdata_size_2;
00156
00157
00158
00159
00160
00161
00162
00163
00164
00165 CSDiskValue4 rb_s3_key_id_4;
00166
00167
00168
00169
00170
00171 CSDiskValue4 rb_s3_cloud_ref_4;
00172
00173
00174
00175
00176 CSDiskValue4 rb_unused[2];
00177
00178
00179 CSDiskValue1 rb_status_1;
00180 CSDiskValue4 rb_backup_id_4;
00181 CSDiskValue4 rb_last_ref_4;
00182 CSDiskValue4 rb_auth_code_4;
00183
00184
00185 } MSBlobHeadRec, *MSBlobHeadPtr;
00186 #define MS_METADAT_OFFSET(header_size, current_metadata_size, metadata_size) (header_size - current_metadata_size - metadata_size)
00187 #define MS_MIN_BLOB_HEAD_SIZE ((uint16_t)(offsetof(MSBlobHeadRec, rb_auth_code_4) + 4))
00188
00189 #define MS_VAR_SPACE(bh) ((int32_t)((CS_GET_DISK_2(bh->rb_head_size_2) - MS_MIN_BLOB_HEAD_SIZE) -(CS_GET_DISK_2(bh->rb_ref_count_2) * CS_GET_DISK_1(bh->rb_ref_size_1)) - CS_GET_DISK_2(bh->rb_mdata_size_2)))
00190 #define MS_CAN_ADD_REFS(bh, n) (MS_VAR_SPACE(bh) >= (int32_t)(n * CS_GET_DISK_1(bh->rb_ref_size_1)))
00191 #define MS_CAN_ADD_MDATA(bh, l) (MS_VAR_SPACE(bh) >= (int32_t)l)
00192
00193
00194 #define MS_BLOB_STAT_OFFS offsetof(MSBlobHeadRec, rb_status_1)
00195 #define MS_BLOB_META_OFFS offsetof(MSBlobHeadRec, rb_alias_offset_2)
00196
00197 #define MS_BLOB_FREE_REF 0x0000
00198 #define MS_BLOB_TABLE_REF 0xFFFF
00199 #define MS_BLOB_DELETE_REF 0xFFFE
00200
00201 #define INVALID_INDEX 0xFFFF
00202
00203
00204
00205 typedef struct MSRepoGenericRef {
00206 CSDiskValue2 rr_type_2;
00207 CSDiskValue2 rr_reserved_2;
00208 uint8_t er_unused[8];
00209 } MSRepoGenericRefRec, *MSRepoGenericRefPtr;
00210
00211
00212
00213
00214
00215
00216
00217
00218
00219
00220
00221
00222
00223
00224
00225
00226
00227
00228
00229
00230
00231 typedef struct MSRepoTableRef {
00232 CSDiskValue2 rr_type_2;
00233 CSDiskValue4 tr_table_id_4;
00234 CSDiskValue6 tr_blob_id_6;
00235 } MSRepoTableRefRec, *MSRepoTableRefPtr;
00236
00237
00238 typedef struct MSRepoTempRef {
00239 CSDiskValue2 rr_type_2;
00240 CSDiskValue2 tp_del_ref_2;
00241
00242 CSDiskValue4 tp_log_id_4;
00243 CSDiskValue4 tp_offset_4;
00244 } MSRepoTempRefRec, *MSRepoTempRefPtr;
00245
00246
00247
00248
00249
00250
00251
00252
00253
00254
00255
00256
00257
00258
00259
00260 typedef struct MSRepoBlobRef {
00261 CSDiskValue2 er_table_2;
00262 CSDiskValue2 er_col_index_2;
00263 CSDiskValue8 er_blob_ref_id_8;
00264 } MSRepoBlobRefRec, *MSRepoBlobRefPtr;
00265
00266 typedef union MSRepoPointers {
00267 char *rp_chars;
00268 uint8_t *rp_bytes;
00269 MSBlobHeadPtr rp_head;
00270 MSRepoGenericRefPtr rp_ref;
00271 MSRepoTableRefPtr rp_tab_ref;
00272 MSRepoTempRefPtr rp_temp_ref;
00273 MSRepoBlobRefPtr rp_blob_ref;
00274 } MSRepoPointersRec, *MSRepoPointersPtr;
00275
00276 #define MS_BLOB_KEY_SIZE 17
00277
00278 class MSRepoFile : public CSFile, public CSPooled {
00279 public:
00280 MSRepository *myRepo;
00281 bool isFileInUse;
00282 MSRepoFile *nextFile;
00283
00284 MSRepoFile();
00285 virtual ~MSRepoFile();
00286
00287 uint64_t readBlobChunk(PBMSBlobIDPtr blob_id, uint64_t rep_offset, uint64_t blob_offset, uint64_t buffer_size, char *buffer);
00288 void writeBlobChunk(PBMSBlobIDPtr blob_id, uint64_t rep_offset, uint64_t blob_offset, uint64_t data_size, char *data);
00289
00290 void sendBlob(MSOpenTable *otab, uint64_t offset, uint64_t req_offset, uint64_t req_size, uint32_t auth_code, bool with_auth_code, bool info_only, CSHTTPOutputStream *stream);
00291 void referenceBlob(MSOpenTable *otab, uint64_t offset, uint16_t head_size, uint32_t tab_id, uint64_t blob_id, uint64_t blob_ref_id, uint32_t auth_code, uint16_t col_index);
00292 void setBlobMetaData(MSOpenTable *otab, uint64_t offset, const char *meta_data, uint16_t meta_data_len, bool reset_alias, const char *alias);
00293 void releaseBlob(MSOpenTable *otab, uint64_t offset, uint16_t head_size, uint32_t tab_id, uint64_t blob_id, uint64_t blob_ref_id, uint32_t auth_code);
00294 void commitBlob(MSOpenTable *otab, uint64_t offset, uint16_t head_size, uint32_t tab_id, uint64_t blob_id, uint64_t blob_ref_id, uint32_t auth_code);
00295 private:
00296 bool getBlobRefSpace(MSOpenTable *otab, uint64_t offset, uint16_t head_size, uint32_t tab_id, uint64_t blob_id, uint64_t blob_ref_id,
00297 uint32_t auth_code, MSRepoTableRefPtr *tab_ref, MSRepoGenericRefPtr *free_ref, uint16_t *tab_ref_cnt, uint64_t *blob_size);
00298 void realFreeBlob(MSOpenTable *otab, char *buffer, uint32_t auth_code, uint64_t offset, uint16_t head_size, uint64_t blob_size, size_t ref_size);
00299 public:
00300 void freeTableReference(MSOpenTable *otab, uint64_t offset, uint16_t head_size, uint32_t tab_id, uint64_t blob_id, uint32_t auth_code);
00301 void checkBlob(CSStringBuffer *buffer, uint64_t offset, uint32_t auth_code, uint32_t temp_log_id, uint32_t temp_log_offset);
00302
00303 void updateAccess(MSBlobHeadPtr blob, uint64_t rep_offset);
00304 virtual void returnToPool();
00305
00306 virtual CSObject *getNextLink() { return iNextLink; }
00307 virtual CSObject *getPrevLink() { return iPrevLink; }
00308 virtual void setNextLink(CSObject *link) { iNextLink = link; }
00309 virtual void setPrevLink(CSObject *link) { iPrevLink = link; }
00310
00311 friend class MSRepository;
00312
00313 private:
00314
00315 CSObject *iNextLink;
00316 CSObject *iPrevLink;
00317
00318 void update_blob_header(MSOpenTable *otab, uint64_t offset, uint64_t blob_size, uint16_t head_size, uint16_t new_head_size);
00319 void removeBlob(MSOpenTable *otab, uint32_t tab_id, uint64_t blob_id, uint64_t offset, uint32_t auth_code);
00320 static MSRepoFile *newRepoFile(MSRepository *repo, CSPath *path);
00321
00322 void updateGarbage(uint64_t size);
00323
00324 public:
00325 static void getBlobKey(MSBlobHeadPtr blob, CloudKeyPtr key)
00326 {
00327 key->creation_time = CS_GET_DISK_4(blob->rb_create_time_4);
00328 key->ref_index = CS_GET_DISK_4(blob->rb_s3_key_id_4);
00329 key->cloud_ref = CS_GET_DISK_4(blob->rb_s3_cloud_ref_4);
00330 }
00331
00332 };
00333
00334 #define CS_REPO_REC_LOCK_COUNT 31
00335
00336 typedef enum RepoLockStates {
00337 REPO_UNLOCKED = 0,
00338 REPO_COMPACTING = 1,
00339 REPO_WRITE = 2,
00340 REPO_BACKUP = 4
00341 } RepoLockState;
00342
00343
00344
00345
00346
00347
00348
00349
00350
00351
00352 class MSRepository : public CSSharedRefObject, public CSPooled {
00353 public:
00354 uint32_t myRepoID;
00355 off64_t myRepoFileSize;
00356 uint32_t myRepoLockState;
00357 bool isRemovingFP;
00358 CSMutex myRepoLock[CS_REPO_REC_LOCK_COUNT];
00359 CSMutex myRepoWriteLock;
00360 MSDatabase *myRepoDatabase;
00361 off64_t myGarbageCount;
00362 size_t myRepoHeadSize;
00363 int myRepoDefRefSize;
00364 size_t myRepoBlobHeadSize;
00365
00366 off64_t myRecoveryOffset;
00367 time_t myLastTempTime;
00368 time_t myLastAccessTime;
00369 time_t myLastCreateTime;
00370 time_t myLastRefTime;
00371
00372 bool mustBeDeleted;
00373
00374 MSRepository(uint32_t id, MSDatabase *db, off64_t file_size);
00375 ~MSRepository();
00376
00377
00378 void openRepoFileForWriting(MSOpenTable *otab);
00379 uint64_t receiveBlob(MSOpenTable *otab, uint16_t head_size, uint64_t blob_size, Md5Digest *checksum = NULL, CSInputStream *stream = NULL);
00380 uint64_t copyBlob(MSOpenTable *otab, uint64_t size, CSInputStream *stream);
00381 void writeBlobHead(MSOpenTable *otab, uint64_t offset, uint8_t ref_size, uint16_t head_size, uint64_t size, Md5Digest *checksum, char *metadata, uint16_t metadata_size, uint64_t blob_id, uint32_t auth_code, uint32_t log_id, uint32_t log_offset, uint8_t blob_type, CloudKeyPtr cloud_key);
00382
00383 void setRepoFileSize(MSOpenTable *otab, off64_t offset);
00384 void syncHead(MSRepoFile *fh);
00385 MSRepoFile *openRepoFile();
00386
00387 virtual void returnToPool();
00388
00389 MSRepoFile *getRepoFile();
00390 void addRepoFile(MSRepoFile *file);
00391 void removeRepoFile(MSRepoFile *file);
00392 void returnRepoFile(MSRepoFile *file);
00393
00394 bool removeRepoFilesNotInUse();
00395
00396 uint16_t getDefaultHeaderSize(uint16_t metadata_size) { return myRepoBlobHeadSize + ((metadata_size)?metadata_size:MS_REPO_MIN_MATADATA) + myRepoDefRefSize * MS_REPO_MIN_REF_COUNT;}
00397 off64_t getRepoFileSize();
00398 size_t getRepoHeadSize();
00399 size_t getRepoBlobHeadSize();
00400 CSMutex *getRepoLock(off64_t offset);
00401 uint32_t getRepoID();
00402 uint32_t getGarbageLevel();
00403
00404 uint32_t initBackup();
00405 bool lockedForBackup();
00406 void backupCompleted();
00407 bool isRepoLocked() { return myRepoXLock;}
00408 void lockRepo(RepoLockState state);
00409 void unlockRepo(RepoLockState state);
00410
00411 friend class MSRepoFile;
00412
00413 private:
00414 bool myRepoXLock;
00415
00416 MSRepoFile *iFilePool;
00417 CSLinkedList iPoolFiles;
00418
00419 CSPath *getRepoFilePath();
00420 void signalCompactor();
00421
00422 };
00423
00424 #endif