Drizzled Public API Documentation

CSS3Protocol.cc

00001 /* Copyright (C) 2009 PrimeBase Technologies GmbH, Germany
00002  *
00003  * PrimeBase Media Stream for MySQL
00004  *
00005  * This program is free software; you can redistribute it and/or modify
00006  * it under the terms of the GNU General Public License as published by
00007  * the Free Software Foundation; either version 2 of the License, or
00008  * (at your option) any later version.
00009  *
00010  * This program is distributed in the hope that it will be useful,
00011  * but WITHOUT ANY WARRANTY; without even the implied warranty of
00012  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
00013  * GNU General Public License for more details.
00014  *
00015  * You should have received a copy of the GNU General Public License
00016  * along with this program; if not, write to the Free Software
00017  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
00018  *
00019  *  Created by Barry Leslie on 10/02/09.
00020  *
00021  */
00022 #include "CSConfig.h"
00023 #include <inttypes.h>
00024 #include <stdlib.h>
00025 
00026 #include <curl/curl.h>
00027 
00028 #include "CSGlobal.h"
00029 #include "CSString.h"
00030 #include "CSStrUtil.h"
00031 #include "CSEncode.h"
00032 #include "CSS3Protocol.h"
00033 #include "CSXML.h"
00034 
00035 #ifdef S3_UNIT_TEST
00036 //#define SHOW_SIGNING
00037 // Uncomment this line to trace network action during request. Very Usefull!!
00038 #define DEBUG_CURL
00039 #define DUMP_ERRORS
00040 #endif
00041 
00042 //#define DUMP_ERRORS
00043 //#define SHOW_SIGNING
00044 
00045 #define HEX_CHECKSUM_VALUE_SIZE (2 *CHECKSUM_VALUE_SIZE)
00046 
00047 #define THROW_CURL_IF(v) { if (v) CSException::throwException(CS_CONTEXT, CS_ERR_GENERIC_ERROR, ms_curl_error);}
00048 
00049 //-------------------------------
00050 static const char *retryCodes[] = {
00051   "ExpiredToken",
00052   "InternalError",
00053   "OperationAborted",
00054   "RequestTimeout",
00055   "SlowDown",
00056   NULL
00057 };
00058 
00059 //======================================
00060 static size_t receive_data(void *ptr, size_t size, size_t nmemb, void *stream);
00061 static size_t receive_header(void *ptr, size_t size, size_t nmemb, void *stream);
00062 static size_t send_callback(void *ptr, size_t size, size_t nmemb, void *stream);
00063 
00064 class S3ProtocolCon : CSXMLBuffer, public CSObject {
00065 
00066   private:
00067   
00068   virtual bool openNode(char *path, char *value) {
00069     if (value && *value && (strcmp(path,"/error/code/") == 0)) {
00070       printf("S3 ERROR Code: %s\n", value);
00071       for (int i = 0; retryCodes[i] && !ms_retry; i++)
00072         ms_retry = (strcmp(value, retryCodes[i]) == 0);
00073         
00074       if (ms_retry && !strcmp("slowdown", value)) 
00075         ms_slowDown = true;
00076     } else if (value && *value && (strcmp(path,"/error/message/") == 0)) {
00077       printf("S3 ERROR MESSAGE: %s\n", value);
00078     }
00079     return true;
00080   }
00081 
00082   virtual bool closeNode(char *path) {
00083     (void)path;
00084     return true;
00085   }
00086 
00087   virtual bool addAttribute(char *path, char *name, char *value) {
00088     (void)path;
00089     (void)name;
00090     (void)value;
00091     return true;
00092   }
00093   
00094   //-------------------------------
00095   void parse_s3_error()
00096   {
00097     enter_();
00098 
00099     if (!ms_errorReply)
00100       CSException::throwException(CS_CONTEXT, CS_ERR_GENERIC_ERROR, "Missing HTTP reply: possible S3 connection failure.");
00101 
00102   #ifdef DUMP_ERRORS
00103     printf("ms_errorReply:\n===========\n%s\n===========\n", ms_errorReply->getCString());
00104   #endif
00105     
00106     if (!parseData(ms_errorReply->getCString(), ms_errorReply->length(), 0)){
00107       int   err;
00108       char  *msg;
00109 
00110       getError(&err, &msg);
00111       CSException::throwException(CS_CONTEXT, CS_ERR_GENERIC_ERROR, msg);
00112     }
00113     
00114     exit_();
00115   }
00116   
00117   public:
00118   
00119   CSHTTPHeaders ms_reply_headers;
00120   CSStringBuffer  ms_buffer; // A scratch buffer
00121 
00122   CURL      *ms_curl; 
00123   struct curl_slist *ms_header_list;  // A curl list of headers to be sent with the next request.
00124 
00125   CSInputStream *ms_inputStream;
00126   CSOutputStream  *ms_outputStream;
00127   
00128   CSMd5     ms_md5;
00129   char      ms_s3Checksum[HEX_CHECKSUM_VALUE_SIZE +1];
00130   bool      ms_calculate_md5;
00131   
00132   bool      ms_notFound; // True if the object could not be found
00133   bool      ms_retry; // True if the request failed with a retry error.
00134   bool      ms_slowDown;
00135   
00136   CSStringBuffer  *ms_errorReply;
00137   char      ms_curl_error[CURL_ERROR_SIZE];
00138   
00139   off64_t     ms_data_size;
00140   
00141   unsigned int  ms_replyStatus;
00142   bool      ms_throw_error; // Gets set if an exception occurs in a callback.
00143   bool      ms_old_libcurl;
00144   char      *ms_safe_url;
00145   time_t      ms_last_modified;
00146   
00147   S3ProtocolCon():
00148     ms_curl(NULL),
00149     ms_header_list(NULL),
00150     ms_inputStream(NULL),
00151     ms_outputStream(NULL),
00152     ms_calculate_md5(false),
00153     ms_notFound(false),
00154     ms_retry(false),
00155     ms_slowDown(false),
00156     ms_errorReply(NULL),
00157     ms_data_size(0),
00158     ms_replyStatus(0),
00159     ms_throw_error(false),
00160     ms_old_libcurl(false),
00161     ms_safe_url(NULL),
00162     ms_last_modified(0)
00163   {
00164   
00165     ms_curl = curl_easy_init();
00166     if (!ms_curl)
00167       CSException::throwException(CS_CONTEXT, CS_ERR_GENERIC_ERROR, "curl_easy_init() failed.");
00168 
00169     curl_version_info_data *curl_ver = curl_version_info(CURLVERSION_NOW); 
00170     
00171     // libCurl versions prior to 7.17.0 did not make copies of strings passed into curl_easy_setopt()
00172     // If this version requirement is a problem I can do this myself, if I have to, I guess. :(
00173     if (curl_ver->version_num < 0X071700 ) {
00174       ms_old_libcurl = true;
00175       
00176       //char msg[200];
00177       //snprintf(msg, 200, "libcurl version %s is too old, require version 7.17.0 or newer.", curl_ver->version);
00178       //CSException::throwException(CS_CONTEXT, CS_ERR_GENERIC_ERROR, msg);
00179     }
00180     
00181     if (curl_easy_setopt(ms_curl, CURLOPT_ERRORBUFFER, ms_curl_error))
00182       CSException::throwException(CS_CONTEXT, CS_ERR_GENERIC_ERROR, "curl_easy_setopt(CURLOPT_ERRORBUFFER) failed.");
00183     
00184 #ifdef DEBUG_CURL
00185     curl_easy_setopt(ms_curl, CURLOPT_VERBOSE, 1L);
00186 #endif    
00187     //THROW_CURL_IF(curl_easy_setopt(ms_curl, CURLOPT_TCP_NODELAY, 1L));
00188   
00189 
00190     THROW_CURL_IF(curl_easy_setopt(ms_curl, CURLOPT_NOPROGRESS, 1L));
00191     THROW_CURL_IF(curl_easy_setopt(ms_curl, CURLOPT_WRITEFUNCTION, receive_data));
00192     THROW_CURL_IF(curl_easy_setopt(ms_curl, CURLOPT_READFUNCTION, send_callback));  
00193     THROW_CURL_IF(curl_easy_setopt(ms_curl, CURLOPT_HEADERFUNCTION, receive_header));
00194     THROW_CURL_IF(curl_easy_setopt(ms_curl, CURLOPT_WRITEDATA, this));
00195     THROW_CURL_IF(curl_easy_setopt(ms_curl, CURLOPT_READDATA, this));
00196     THROW_CURL_IF(curl_easy_setopt(ms_curl, CURLOPT_WRITEHEADER, this));
00197     
00198 
00199     THROW_CURL_IF(curl_easy_setopt(ms_curl, CURLOPT_FOLLOWLOCATION, 1L)); // Follow redirects.
00200   
00201   }
00202   
00203   ~S3ProtocolCon()
00204   {
00205     if (ms_curl) 
00206       curl_easy_cleanup(ms_curl);     
00207     if (ms_header_list) 
00208       curl_slist_free_all(ms_header_list);      
00209     if (ms_inputStream)
00210       ms_inputStream->release();
00211     if (ms_outputStream)
00212       ms_outputStream->release();
00213     if (ms_errorReply)
00214       ms_errorReply->release();
00215       
00216     ms_reply_headers.clearHeaders();
00217     
00218     if (ms_safe_url)
00219       cs_free(ms_safe_url);
00220   }
00221 
00222   inline void check_reply_status() 
00223   {
00224     if (ms_replyStatus > 199 && ms_replyStatus < 300)
00225       return;
00226     
00227     
00228     
00229     switch (ms_replyStatus) {
00230       case 200:
00231       case 204: // No Content
00232       //case 301: // Moved Permanently
00233       //case 307: // Temporary Redirect
00234         break;
00235       case 404: // Not Found
00236       case 403: // Forbidden (S3 object not found)
00237         ms_notFound = true;
00238         break;
00239       case 500: 
00240         ms_retry = true;
00241         break;
00242       default: {
00243         parse_s3_error();
00244         
00245         
00246         
00247         if (!ms_retry) {
00248           enter_();
00249           CSException::throwException(CS_CONTEXT, CS_ERR_GENERIC_ERROR, ms_errorReply->getCString());
00250           outer_();
00251         } else if (ms_slowDown) {
00252           enter_();
00253           CSException::logException(CS_CONTEXT, CS_ERR_GENERIC_ERROR, "S3 slow down request.");
00254           self->sleep(10); // sleep for 1/100 second.
00255           outer_();
00256         }
00257       }
00258     }
00259     
00260   }
00261 
00262   
00263   inline void ms_reset()
00264   {
00265     // Remove any old headers
00266     if (ms_header_list) {
00267       curl_slist_free_all(ms_header_list);
00268       ms_header_list = NULL;
00269     }
00270 
00271     ms_reply_headers.clearHeaders();
00272     ms_replyStatus = 0;
00273     ms_throw_error = false;
00274     if (ms_errorReply)
00275       ms_errorReply->setLength(0);
00276       
00277     ms_s3Checksum[0] = 0;
00278     ms_notFound = false;
00279     ms_retry = false;
00280     
00281     if (ms_outputStream) {
00282       ms_outputStream->release();
00283       ms_outputStream = NULL;
00284     }
00285     if (ms_inputStream) {
00286       ms_inputStream->release();
00287       ms_inputStream = NULL;
00288     }
00289     
00290     if (ms_safe_url) {
00291       cs_free(ms_safe_url);
00292       ms_safe_url = NULL;
00293     }
00294   }
00295   
00296   inline void ms_setHeader(const char *header)
00297   {
00298     ms_header_list = curl_slist_append(ms_header_list, header);
00299     if (!ms_header_list) 
00300       CSException::throwException(CS_CONTEXT, CS_ERR_GENERIC_ERROR, "curl_slist_append() failed.");
00301   }
00302 
00303 
00304 private:  
00305   inline const char *safe_url(const char *url)
00306   {
00307     if (ms_old_libcurl == false)
00308       return url;
00309       
00310     if (ms_safe_url) {
00311       cs_free(ms_safe_url);
00312       ms_safe_url = NULL;
00313     }
00314     ms_safe_url = cs_strdup(url);
00315     return ms_safe_url;
00316   }
00317   
00318 public: 
00319   inline void ms_setURL(const char *url)
00320   {
00321     //printf("URL: \"%s\n", url);
00322     THROW_CURL_IF(curl_easy_setopt(ms_curl, CURLOPT_URL, safe_url(url)));
00323   }
00324   
00325   inline void ms_execute_delete_request()
00326   {
00327     CURLcode rtc;
00328     
00329     THROW_CURL_IF(curl_easy_setopt(ms_curl, CURLOPT_HTTPHEADER, ms_header_list));
00330     THROW_CURL_IF(curl_easy_setopt(ms_curl, CURLOPT_CUSTOMREQUEST, "DELETE"));
00331 
00332     rtc = curl_easy_perform(ms_curl);
00333     
00334     THROW_CURL_IF(curl_easy_setopt(ms_curl, CURLOPT_CUSTOMREQUEST, NULL)); // IMPORTANT: Reset this to it's default value
00335 
00336     if (rtc && !ms_throw_error)
00337       CSException::throwException(CS_CONTEXT, CS_ERR_GENERIC_ERROR, ms_curl_error);
00338       
00339     if (ms_throw_error) {
00340       enter_();
00341       throw_();
00342       outer_();
00343     }
00344     
00345     check_reply_status();
00346   }
00347   
00348   inline void ms_execute_copy_request()
00349   {
00350     CURLcode rtc;
00351     
00352     THROW_CURL_IF(curl_easy_setopt(ms_curl, CURLOPT_HTTPHEADER, ms_header_list));
00353     THROW_CURL_IF(curl_easy_setopt(ms_curl, CURLOPT_INFILESIZE_LARGE, 0));
00354     THROW_CURL_IF(curl_easy_setopt(ms_curl, CURLOPT_UPLOAD, 1L));
00355     
00356     rtc = curl_easy_perform(ms_curl);
00357     
00358     if (rtc && !ms_throw_error)
00359       CSException::throwException(CS_CONTEXT, CS_ERR_GENERIC_ERROR, ms_curl_error);
00360       
00361     if (ms_throw_error) {
00362       enter_();
00363       throw_();
00364       outer_();
00365     }
00366     
00367     check_reply_status();
00368   }
00369   
00370   inline void ms_execute_get_request(CSOutputStream *output)
00371   {
00372     enter_();
00373     
00374     if (output) {
00375       push_(output);
00376       THROW_CURL_IF(curl_easy_setopt(ms_curl, CURLOPT_HTTPGET, 1L));
00377     } else {
00378       THROW_CURL_IF(curl_easy_setopt(ms_curl, CURLOPT_NOBODY, 1L));
00379     }
00380     
00381     // 
00382     THROW_CURL_IF(curl_easy_setopt(ms_curl, CURLOPT_FILETIME, 1L));
00383     THROW_CURL_IF(curl_easy_setopt(ms_curl, CURLOPT_HTTPHEADER, ms_header_list));
00384     // Ask curl to parse the Last-Modified header.  This is easier than
00385     // parsing it ourselves.
00386 
00387     ms_outputStream = output; 
00388     if (curl_easy_perform(ms_curl) && !ms_throw_error) {
00389       ms_outputStream = NULL; 
00390       CSException::throwException(CS_CONTEXT, CS_ERR_GENERIC_ERROR, ms_curl_error);
00391     }
00392     ms_outputStream = NULL; 
00393     if (output){
00394       release_(output);
00395     }
00396     
00397     if (ms_throw_error) 
00398       throw_();
00399     
00400     check_reply_status();
00401     curl_easy_getinfo(ms_curl, CURLINFO_FILETIME, &ms_last_modified);
00402     exit_();    
00403   }
00404   inline void ms_execute_put_request(CSInputStream *input, off64_t size)
00405   {
00406     enter_();
00407     
00408     push_(input); 
00409     THROW_CURL_IF(curl_easy_setopt(ms_curl, CURLOPT_HTTPHEADER, ms_header_list));
00410     THROW_CURL_IF(curl_easy_setopt(ms_curl, CURLOPT_INFILESIZE_LARGE, size));
00411     THROW_CURL_IF(curl_easy_setopt(ms_curl, CURLOPT_UPLOAD, 1L));
00412     //THROW_CURL_IF(curl_easy_setopt(ms_curl, CURLOPT_POSTFIELDSIZE_LARGE, size));
00413     //THROW_CURL_IF(curl_easy_setopt(ms_curl, CURLOPT_POST, 1L));
00414 
00415     ms_md5.md5_init();
00416     
00417     ms_data_size = size;
00418     ms_inputStream = input; 
00419     if (curl_easy_perform(ms_curl) && !ms_throw_error) {
00420       ms_inputStream = NULL;  
00421       CSException::throwException(CS_CONTEXT, CS_ERR_GENERIC_ERROR, ms_curl_error);
00422     }
00423     ms_inputStream = NULL;
00424     release_(input);  
00425 
00426       
00427     if (ms_throw_error)
00428       throw_();
00429     
00430     check_reply_status();
00431     
00432     if (ms_calculate_md5) {
00433       // If the data was not sent with an md5 checksum then verify
00434       // the server's md5 value with the one calculated during the send.
00435       char checksum[HEX_CHECKSUM_VALUE_SIZE +1];
00436       Md5Digest digest;
00437       
00438       ms_md5.md5_get_digest(&digest);
00439       cs_bin_to_hex(HEX_CHECKSUM_VALUE_SIZE +1, checksum, CHECKSUM_VALUE_SIZE, digest.val);
00440       
00441       cs_strToUpper(ms_s3Checksum);
00442       if (strcmp(checksum, ms_s3Checksum)) {
00443         // The request should be restarted in this case.
00444         ms_retry = true;
00445         CSException::logException(CS_CONTEXT, CS_ERR_CHECKSUM_ERROR, "Calculated checksum did not match S3 checksum");
00446       }
00447     }
00448 
00449     exit_();    
00450   }
00451   
00452 };
00453 
00454 //======================================
00455 
00456 
00457 
00458 
00459 //======================================
00460 CSS3Protocol::CSS3Protocol():
00461   s3_server(NULL),
00462   s3_public_key(NULL),
00463   s3_private_key(NULL),
00464   s3_maxRetries(5),
00465   s3_sleepTime(0)
00466 {
00467   new_(s3_server, CSStringBuffer());
00468   s3_server->append("s3.amazonaws.com/");
00469 
00470   s3_public_key = CSString::newString("");
00471   s3_private_key = CSString::newString("");
00472   
00473 }
00474 
00475 //------------------
00476 CSS3Protocol::~CSS3Protocol()
00477 {
00478   if (s3_server)
00479     s3_server->release();
00480   
00481   if (s3_public_key)
00482     s3_public_key->release();
00483   
00484   if (s3_private_key)
00485     s3_private_key->release();
00486 }
00487   
00488 //------------------
00489 CSString *CSS3Protocol::s3_getSignature(const char *verb, 
00490                     const char *md5, 
00491                     const char *content_type, 
00492                     const char *date, 
00493                     const char *bucket, 
00494                     const char *key,
00495                     CSString *headers 
00496                   )
00497 {
00498   CSStringBuffer *s3_buffer;
00499   enter_();
00500   if (headers)
00501     push_(headers);
00502   
00503   new_(s3_buffer, CSStringBuffer());
00504   push_(s3_buffer);
00505   
00506   s3_buffer->setLength(0);
00507   s3_buffer->append(verb);  
00508   s3_buffer->append("\n");  
00509   if (md5) s3_buffer->append(md5);  
00510   s3_buffer->append("\n");  
00511   if (content_type) s3_buffer->append(content_type);  
00512   s3_buffer->append("\n");  
00513   s3_buffer->append(date);
00514   if (headers) { 
00515     // Note: headers are assumed to be in lower case, sorted, and containing no white space.
00516     s3_buffer->append("\n");  
00517     s3_buffer->append(headers->getCString());
00518   }
00519   s3_buffer->append("\n/");
00520   s3_buffer->append(bucket);
00521   s3_buffer->append("/");
00522   s3_buffer->append(key);
00523 
00524 #ifdef SHOW_SIGNING
00525 printf("signing:\n=================\n%s\n=================\n",  s3_buffer->getCString());
00526 printf("Public Key:\"%s\"\n",   s3_public_key->getCString());
00527 printf("Private Key:\"%s\"\n",  s3_private_key->getCString());
00528 if(0){
00529   const char *ptr = s3_buffer->getCString();
00530   while (*ptr) {
00531     printf("%x ", *ptr); ptr++;
00532   }
00533   printf("\n");
00534 }
00535 #endif
00536 
00537   CSString *sig = signature(s3_buffer->getCString(), s3_private_key->getCString());
00538   release_(s3_buffer);
00539   if (headers) 
00540     release_(headers);
00541 
00542   return_(sig);
00543 }
00544 //----------------------
00545 // CURL callback functions:
00547 //----------------------
00548 //-----------------
00549 static bool try_ReadStream(CSThread *self, S3ProtocolCon *con, unsigned char *ptr, size_t buffer_size, size_t *data_sent)
00550 {
00551   volatile bool rtc = true;
00552   try_(a) {
00553     *data_sent = con->ms_inputStream->read((char*)ptr, buffer_size);
00554     if (*data_sent <= con->ms_data_size) {
00555       con->ms_data_size -= *data_sent;
00556       if (*data_sent)
00557         con->ms_md5.md5_append(ptr, *data_sent); // Calculating the checksum for the data sent.
00558     } else if (*data_sent > con->ms_data_size) 
00559       CSException::RecordException(CS_CONTEXT, CS_ERR_GENERIC_ERROR, "Blob larger than expected.");
00560     else if (con->ms_data_size && !*data_sent)
00561       CSException::RecordException(CS_CONTEXT, CS_ERR_GENERIC_ERROR, "Blob smaller than expected.");
00562     rtc = false;
00563   }
00564   
00565   catch_(a)
00566   cont_(a);
00567   return rtc;
00568 }
00569 
00570 //----------------------
00571 static size_t send_callback(void *ptr, size_t objs, size_t obj_size, void *v_con)
00572 {
00573   S3ProtocolCon *con = (S3ProtocolCon*) v_con;
00574   size_t data_sent, buffer_size = objs * obj_size;
00575 
00576   if (!con->ms_data_size)
00577     return 0;
00578     
00579   enter_();
00580   if (try_ReadStream(self, con, (unsigned char*)ptr, buffer_size, &data_sent)) {
00581     con->ms_throw_error = true;
00582     data_sent = (size_t)-1;
00583   }
00584   
00585   return_(data_sent);
00586 }
00587 
00588 //-----------------
00589 static bool try_WriteStream(CSThread *self, S3ProtocolCon *con, char *ptr, size_t data_len)
00590 {
00591   volatile bool rtc = true;
00592   try_(a) {
00593     if (con->ms_replyStatus >= 400) { // Collect the error reply.
00594       if (!con->ms_errorReply)
00595         con->ms_errorReply = new CSStringBuffer(50);    
00596       con->ms_errorReply->append(ptr, data_len);
00597     } else if ( con->ms_outputStream)
00598       con->ms_outputStream->write(ptr, data_len);
00599     rtc = false;
00600   }
00601   
00602   catch_(a)
00603   cont_(a);
00604   return rtc;
00605 }
00606 
00607 //----------------------
00608 static size_t receive_data(void *vptr, size_t objs, size_t obj_size, void *v_con)
00609 {
00610   S3ProtocolCon *con = (S3ProtocolCon*) v_con;
00611   size_t data_len = objs * obj_size;
00612 
00613   enter_();
00614   if (try_WriteStream(self, con, (char*)vptr, data_len)) {
00615     con->ms_throw_error = true;
00616     data_len = (size_t)-1;
00617   }
00618 
00619   return_(data_len);  
00620 }
00621 
00622 #define IS_REDIRECT(s) ((s >= 300) && (s < 400))
00623 //----------------------
00624 static bool try_addHeader(CSThread *self, S3ProtocolCon *con, char *name, uint32_t name_len, char *value, uint32_t value_len)
00625 {
00626   volatile bool rtc = true;
00627   
00628   try_(a) {
00629     con->ms_reply_headers.addHeader(name, name_len, value, value_len);
00630     rtc = false;
00631   }
00632   
00633   catch_(a);
00634   cont_(a);
00635   return rtc;
00636 }
00637 
00638 //----------------------
00639 static size_t receive_header(void *header, size_t objs, size_t obj_size, void *v_con)
00640 {
00641   S3ProtocolCon *con = (S3ProtocolCon*) v_con;
00642   size_t size = objs * obj_size;
00643   char *end, *ptr = (char*) header, *name, *value = NULL;
00644   uint32_t name_len =0, value_len = 0;
00645   
00646 //printf( "receive_header: %s\n", ptr);
00647   end = ptr + size;
00648   if (*(end -2) == '\r' && *(end -1) == '\n')
00649     end -=2;
00650     
00651   while ((end != ptr) && (*ptr == ' ')) ptr++;
00652   if (end == ptr)
00653     return size;
00654   
00655   // Get the reply status.
00656   // Status 100 = Continue
00657   if (((!con->ms_replyStatus) || (con->ms_replyStatus == 100) || IS_REDIRECT(con->ms_replyStatus) ) 
00658       && !strncasecmp(ptr, "HTTP", 4)
00659     ) {
00660     char status[4];
00661     while ((end != ptr) && (*ptr != ' ')) ptr++; // skip HTTP stuff
00662     while ((end != ptr) && (*ptr == ' ')) ptr++; // find the start of eh status code.
00663     if (end == ptr)
00664       return size;
00665       
00666     if (end < (ptr +3)) // expecting a 3 digit status code.
00667       return size;
00668       
00669     memcpy(status, ptr, 3);
00670     status[3] = 0;
00671     
00672     con->ms_replyStatus = atoi(status);
00673   }
00674   
00675   name = ptr;
00676   while ((end != ptr) && (*ptr != ':')) ptr++;
00677   if (end == ptr)
00678     return size;
00679   name_len = ptr - name;
00680   
00681   ptr++; 
00682   while ((end != ptr) && (*ptr == ' ')) ptr++;
00683   if (end == ptr)
00684     return size;
00685   
00686   value = ptr;
00687   value_len = end - value;
00688   
00689   while (name[name_len-1] == ' ') name_len--;
00690   while (value[value_len-1] == ' ') value_len--;
00691   
00692   if (!strncasecmp(name, "ETag", 4)) {
00693     if (*value == '"') {
00694       value++; value_len -=2; // Strip quotation marks from checksum string.
00695     }
00696     if (value_len == HEX_CHECKSUM_VALUE_SIZE) {
00697       memcpy(con->ms_s3Checksum, value, value_len);
00698       con->ms_s3Checksum[value_len] = 0;
00699     }
00700   }
00701   
00702   enter_();
00703   if (try_addHeader(self, con, name, name_len, value, value_len)) {
00704     con->ms_throw_error = true;
00705     size = (size_t)-1;
00706   }
00707   return_(size);
00708 }
00709 
00710 //----------------------
00711 
00712 #define SET_DATE_FROM_TIME(t, d) {strftime(d, sizeof(d), "%a, %d %b %Y %H:%M:%S GMT", gmtime(&t));}
00713 #define SET_DATE(d) {time_t t = time(NULL); SET_DATE_FROM_TIME(t, d);}
00714 
00715 bool CSS3Protocol::s3_delete(const char *bucket, const char *key)
00716 {
00717   CSStringBuffer *s3_buffer;
00718   char date[64];
00719   CSString *signed_str;
00720   uint32_t retry_count = 0;
00721   S3ProtocolCon *con_data;
00722 
00723   enter_();
00724 
00725   new_(s3_buffer, CSStringBuffer());
00726   push_(s3_buffer);
00727 
00728   new_(con_data, S3ProtocolCon());
00729   push_(con_data);
00730 
00731 retry:
00732   // Clear old settings. 
00733   con_data->ms_reset(); 
00734   
00735   SET_DATE(date);
00736  
00737   // Build the URL
00738   s3_buffer->setLength(0);
00739   s3_buffer->append("http://");
00740   s3_buffer->append(bucket);
00741   s3_buffer->append("."); 
00742   s3_buffer->append(s3_server->getCString());
00743   s3_buffer->append(key);
00744 
00745   con_data->ms_setURL(s3_buffer->getCString());
00746   
00747   // Add the 'DATE' header
00748   s3_buffer->setLength(0);
00749   s3_buffer->append("Date: ");  
00750   s3_buffer->append(date);
00751   con_data->ms_setHeader(s3_buffer->getCString());
00752 
00753   // Create the authentication signature and add the 'Authorization' header
00754   signed_str = s3_getSignature("DELETE", NULL, NULL, date, bucket, key);
00755   push_(signed_str);
00756   s3_buffer->setLength(0);
00757   s3_buffer->append("Authorization: AWS "); 
00758   s3_buffer->append(s3_public_key->getCString());
00759   s3_buffer->append(":"); 
00760   s3_buffer->append(signed_str->getCString());  
00761   release_(signed_str); signed_str = NULL;
00762   
00763   con_data->ms_setHeader(s3_buffer->getCString());
00764   
00765   con_data->ms_execute_delete_request();
00766   
00767   if (con_data->ms_retry) {
00768     if (retry_count == s3_maxRetries) {
00769       CSException::throwException(CS_CONTEXT, CS_ERR_GENERIC_ERROR, "S3 operation aborted after max retries.");
00770     }
00771   //printf("RETRY: s3_delete()\n");
00772     retry_count++;
00773     self->sleep(s3_sleepTime);
00774     goto retry;
00775   }
00776   
00777   bool notFound = con_data->ms_notFound;
00778   release_(con_data);
00779   release_(s3_buffer);
00780   
00781   return_(!notFound);
00782 }
00783 
00784 //-------------------------------
00785 void CSS3Protocol::s3_copy(const char *dest_server, const char *dest_bucket, const char *dest_key, const char *src_bucket, const char *src_key)
00786 {
00787     CSStringBuffer *s3_buffer;
00788   char date[64];
00789   CSString *signed_str;
00790   uint32_t retry_count = 0;
00791   S3ProtocolCon *con_data;
00792 
00793   enter_();
00794 
00795   new_(s3_buffer, CSStringBuffer());
00796   push_(s3_buffer);
00797 
00798   new_(con_data, S3ProtocolCon());
00799   push_(con_data);
00800   
00801   if (!dest_server)
00802     dest_server = s3_server->getCString();
00803 
00804 retry:
00805   // Clear old settings. 
00806   con_data->ms_reset(); 
00807   
00808   SET_DATE(date);
00809  
00810   // Build the URL
00811   s3_buffer->setLength(0);
00812   s3_buffer->append("http://");
00813   s3_buffer->append(dest_bucket);
00814   s3_buffer->append("."); 
00815   s3_buffer->append(s3_server->getCString());
00816   s3_buffer->append(dest_key);
00817 
00818   con_data->ms_setURL(s3_buffer->getCString());
00819   
00820   // Add the destination location
00821   s3_buffer->setLength(0);
00822   s3_buffer->append("Host: ");  
00823   s3_buffer->append(dest_bucket);
00824   s3_buffer->append("."); 
00825   s3_buffer->append(dest_server);
00826   s3_buffer->setLength(s3_buffer->length() -1); // trim the '/'
00827   con_data->ms_setHeader(s3_buffer->getCString());
00828   
00829   // Add the source location
00830   s3_buffer->setLength(0);
00831   s3_buffer->append("x-amz-copy-source:");  
00832   s3_buffer->append(src_bucket);
00833   s3_buffer->append("/");
00834   s3_buffer->append(src_key);
00835   con_data->ms_setHeader(s3_buffer->getCString());
00836   
00837   // Create the authentication signature and add the 'Authorization' header
00838   signed_str = s3_getSignature("PUT", NULL, NULL, date, dest_bucket, dest_key, CSString::newString(s3_buffer->getCString()));
00839   push_(signed_str);
00840 
00841   // Add the 'DATE' header
00842   s3_buffer->setLength(0);
00843   s3_buffer->append("Date: ");  
00844   s3_buffer->append(date);
00845   con_data->ms_setHeader(s3_buffer->getCString());
00846 
00847   // Add the signature
00848   s3_buffer->setLength(0);
00849   s3_buffer->append("Authorization: AWS "); 
00850   s3_buffer->append(s3_public_key->getCString());
00851   s3_buffer->append(":"); 
00852   s3_buffer->append(signed_str->getCString());  
00853   release_(signed_str); signed_str = NULL;
00854   con_data->ms_setHeader(s3_buffer->getCString());
00855   
00856   con_data->ms_execute_copy_request();
00857   
00858   if (con_data->ms_notFound) {
00859     s3_buffer->setLength(0);
00860     s3_buffer->append("Cloud copy failed, object not found: ");
00861     s3_buffer->append(src_bucket);
00862     s3_buffer->append(" ");
00863     s3_buffer->append(src_key);
00864     CSException::throwException(CS_CONTEXT, CS_ERR_GENERIC_ERROR, s3_buffer->getCString());
00865   }
00866   
00867   if (con_data->ms_retry) {
00868     if (retry_count == s3_maxRetries) {
00869       CSException::throwException(CS_CONTEXT, CS_ERR_GENERIC_ERROR, "S3 operation aborted after max retries.");
00870     }
00871   //printf("RETRY: s3_copy()\n");
00872     retry_count++;
00873     self->sleep(s3_sleepTime);
00874     goto retry;
00875   }
00876   
00877   release_(con_data);
00878   release_(s3_buffer);
00879   
00880   exit_();
00881 }
00882 
00883 
00884 //-------------------------------
00885 CSVector *CSS3Protocol::s3_receive(CSOutputStream *output, const char *bucket, const char *key, bool *found, S3RangePtr range, time_t *last_modified)
00886 {
00887   CSStringBuffer *s3_buffer;
00888     char date[64];
00889   CSString *signed_str;
00890   uint32_t retry_count = 0;
00891   S3ProtocolCon *con_data;
00892   CSVector *replyHeaders;
00893   CSString *range_header = NULL;
00894   const char *http_op;
00895 
00896   enter_();
00897 
00898   if (output) {
00899     push_(output);
00900     http_op = "GET";
00901   } else
00902     http_op = "HEAD";
00903 
00904   new_(s3_buffer, CSStringBuffer());
00905   push_(s3_buffer);
00906 
00907   new_(con_data, S3ProtocolCon());
00908   push_(con_data);
00909 
00910 retry:
00911   // Clear old settings. 
00912   con_data->ms_reset(); 
00913   
00914   SET_DATE(date);
00915  
00916   // Build the URL
00917   s3_buffer->setLength(0);
00918   s3_buffer->append("http://");
00919   s3_buffer->append(bucket);
00920   s3_buffer->append("."); 
00921   s3_buffer->append(s3_server->getCString());
00922   s3_buffer->append(key);
00923 
00924   con_data->ms_setURL(s3_buffer->getCString());
00925   
00926   // Add the 'DATE' header
00927   s3_buffer->setLength(0);
00928   s3_buffer->append("Date: ");  
00929   s3_buffer->append(date);
00930   con_data->ms_setHeader(s3_buffer->getCString());
00931 
00932   if (range) {
00933     char buffer[80];
00934     snprintf(buffer, 80,"Range: bytes=%"PRIu64"-%"PRIu64, range->startByte, range->endByte);
00935 
00936     range_header = CSString::newString(buffer);
00937   }
00938   // Create the authentication signature and add the 'Authorization' header
00939   if (range_header)
00940     con_data->ms_setHeader(range_header->getCString());
00941   signed_str = s3_getSignature(http_op, NULL, NULL, date, bucket, key, NULL);
00942   push_(signed_str);
00943   s3_buffer->setLength(0);
00944   s3_buffer->append("Authorization: AWS "); 
00945   s3_buffer->append(s3_public_key->getCString());
00946   s3_buffer->append(":"); 
00947   s3_buffer->append(signed_str->getCString());  
00948   release_(signed_str); signed_str = NULL;
00949   con_data->ms_setHeader(s3_buffer->getCString());
00950   
00951   if (output) output->retain();
00952   con_data->ms_execute_get_request(output);
00953   
00954   if (con_data->ms_retry) {
00955     if (retry_count == s3_maxRetries) {
00956       CSException::throwException(CS_CONTEXT, CS_ERR_GENERIC_ERROR, "S3 operation aborted after max retries.");
00957     }
00958   //printf("RETRY: s3_receive()\n");
00959     retry_count++;
00960     output->reset();
00961     self->sleep(s3_sleepTime);
00962     goto retry;
00963   }
00964   
00965   if (last_modified)
00966     *last_modified = con_data->ms_last_modified;
00967   *found = !con_data->ms_notFound;
00968   replyHeaders = con_data->ms_reply_headers.takeHeaders();
00969   release_(con_data);
00970   release_(s3_buffer);
00971   if (output)
00972     release_(output);
00973   
00974   return_(replyHeaders);
00975 }
00976 
00977 class S3ListParser : public CSXMLBuffer {
00978 
00979   CSVector *list;
00980   public:
00981 
00982 
00983   bool parseListData(const char *data, size_t len, CSVector *keys)
00984   {
00985     list = keys;
00986     return parseData(data, len, 0);
00987   }
00988 
00989   private:
00990   virtual bool openNode(char *path, char *value) {
00991     if (value && *value && (strcmp(path,"/listbucketresult/contents/key/") == 0))
00992       list->add(CSString::newString(value));
00993     return true;
00994   }
00995 
00996   virtual bool closeNode(char *path) {
00997     (void)path;
00998     return true;
00999   }
01000 
01001   virtual bool addAttribute(char *path, char *name, char *value) {
01002     (void)path;
01003     (void)name;
01004     (void)value;
01005     return true;
01006   }
01007 
01008 };
01009 
01010 //-------------------------------
01011 static CSVector *parse_s3_list(CSMemoryOutputStream *output)
01012 {
01013   S3ListParser s3ListParser;
01014   const char *data;
01015   CSVector *vector;
01016   size_t len;
01017   
01018   enter_();
01019 
01020   push_(output);
01021   
01022   new_(vector, CSVector(10)); 
01023   push_(vector);  
01024 
01025   data = (const char *) output->getMemory(&len);
01026   if (!s3ListParser.parseListData(data, len, vector)) {
01027     int   err;
01028     char  *msg;
01029 
01030     s3ListParser.getError(&err, &msg);
01031     CSException::throwException(CS_CONTEXT, CS_ERR_GENERIC_ERROR, msg);
01032   }
01033 
01034   pop_(vector);
01035   release_(output);
01036   return_(vector);
01037 }
01038 
01039 
01040 //-------------------------------
01041 CSVector *CSS3Protocol::s3_list(const char *bucket, const char *key_prefix, uint32_t max)
01042 {
01043   CSStringBuffer *s3_buffer;
01044     char date[64];
01045   CSString *signed_str;
01046   CSMemoryOutputStream *output;
01047   uint32_t retry_count = 0;
01048   S3ProtocolCon *con_data;
01049   enter_();
01050 
01051   new_(s3_buffer, CSStringBuffer());
01052   push_(s3_buffer);
01053 
01054   output = CSMemoryOutputStream::newStream(1024, 1024);
01055   push_(output);
01056   
01057   new_(con_data, S3ProtocolCon());
01058   push_(con_data);
01059 
01060 retry:
01061 
01062   // Clear old settings. 
01063   con_data->ms_reset(); 
01064   
01065   SET_DATE(date);
01066  
01067   // Build the URL
01068   s3_buffer->setLength(0);
01069   s3_buffer->append("http://");
01070   s3_buffer->append(bucket);
01071   s3_buffer->append("."); 
01072   s3_buffer->append(s3_server->getCString());
01073 //s3_buffer->append("/"); 
01074 //s3_buffer->append(bucket);
01075   if (key_prefix) {
01076     s3_buffer->append("?prefix=");
01077     s3_buffer->append(key_prefix);
01078   }
01079   
01080   if (max) {
01081     if (key_prefix)
01082       s3_buffer->append("&max-keys=");
01083     else
01084       s3_buffer->append("?max-keys=");
01085     s3_buffer->append(max);
01086   }
01087 
01088   con_data->ms_setURL(s3_buffer->getCString());
01089   
01090   // Add the 'DATE' header
01091   s3_buffer->setLength(0);
01092   s3_buffer->append("Date: ");  
01093   s3_buffer->append(date);
01094   con_data->ms_setHeader(s3_buffer->getCString());
01095 
01096   // Create the authentication signature and add the 'Authorization' header
01097   signed_str = s3_getSignature("GET", NULL, NULL, date, bucket, "");
01098   push_(signed_str);
01099   s3_buffer->setLength(0);
01100   s3_buffer->append("Authorization: AWS "); 
01101   s3_buffer->append(s3_public_key->getCString());
01102   s3_buffer->append(":"); 
01103   s3_buffer->append(signed_str->getCString());  
01104   release_(signed_str); signed_str = NULL;
01105   con_data->ms_setHeader(s3_buffer->getCString());
01106   
01107   con_data->ms_execute_get_request(RETAIN(output));
01108   
01109   if (con_data->ms_retry) {
01110     if (retry_count == s3_maxRetries) {
01111       CSException::throwException(CS_CONTEXT, CS_ERR_GENERIC_ERROR, "S3 operation aborted after max retries.");
01112     }
01113   //printf("RETRY: s3_list()\n");
01114     retry_count++;
01115     output->reset();
01116     self->sleep(s3_sleepTime);
01117     goto retry;
01118   }
01119   
01120   release_(con_data);
01121   pop_(output);
01122   release_(s3_buffer);
01123   return_(parse_s3_list(output));
01124 }
01125 
01126 //-------------------------------
01127 CSString *CSS3Protocol::s3_getAuthorization(const char *bucket, const char *key, const char *content_type, uint32_t *s3AuthorizationTime)
01128 {
01129     char date[64];
01130   CSString *signed_str;
01131   time_t sys_time;
01132 
01133   enter_();
01134 
01135   if (!content_type)
01136     content_type = "binary/octet-stream";
01137     
01138   sys_time = time(NULL);
01139   
01140   *s3AuthorizationTime = (uint32_t)sys_time;
01141   
01142   SET_DATE_FROM_TIME(sys_time, date);
01143   signed_str = s3_getSignature("PUT", NULL, content_type, date, bucket, key);
01144   return_(signed_str);
01145 }
01146 
01147 //-------------------------------
01148 CSVector *CSS3Protocol::s3_send(CSInputStream *input, const char *bucket, const char *key, off64_t size, const char *content_type, Md5Digest *digest, const char *s3Authorization, time_t s3AuthorizationTime)
01149 {
01150   CSStringBuffer *s3_buffer;
01151     char date[64];
01152   CSString *signed_str;
01153   uint32_t retry_count = 0;
01154   S3ProtocolCon *con_data;
01155   CSVector *replyHeaders;
01156   char checksum[32], *md5 = NULL;
01157 
01158   enter_();
01159   push_(input);
01160 
01161   new_(s3_buffer, CSStringBuffer());
01162   push_(s3_buffer);
01163 
01164   new_(con_data, S3ProtocolCon());
01165   push_(con_data);
01166     
01167   if (!content_type)
01168     content_type = "binary/octet-stream";
01169     
01170 retry:
01171 
01172   // Clear old settings. 
01173   con_data->ms_reset(); 
01174   
01175   if (s3Authorization) {
01176     SET_DATE_FROM_TIME(s3AuthorizationTime, date);
01177   } else {
01178     SET_DATE(date);
01179   }
01180   
01181   // Build the URL
01182   s3_buffer->setLength(0);
01183   s3_buffer->append("http://");
01184   s3_buffer->append(bucket);
01185   s3_buffer->append("."); 
01186   s3_buffer->append(s3_server->getCString());
01187   s3_buffer->append(key);
01188 
01189   con_data->ms_setURL(s3_buffer->getCString());
01190   
01191   // Add the 'DATE' header
01192   s3_buffer->setLength(0);
01193   s3_buffer->append("Date: ");  
01194   s3_buffer->append(date);
01195   con_data->ms_setHeader(s3_buffer->getCString());
01196   
01197   // Add the 'Content-Type' header
01198   s3_buffer->setLength(0);
01199   s3_buffer->append("Content-Type: ");  
01200   s3_buffer->append(content_type);
01201   con_data->ms_setHeader(s3_buffer->getCString());
01202     
01203   if (digest) {
01204     // Add the Md5 checksum header
01205     md5 = checksum;
01206     memset(checksum, 0, 32);
01207     base64Encode(digest->val, 16, checksum, 32);
01208     
01209     s3_buffer->setLength(0);
01210     s3_buffer->append("Content-MD5: "); 
01211     s3_buffer->append(checksum);
01212     con_data->ms_setHeader(s3_buffer->getCString());    
01213     con_data->ms_calculate_md5 = false;
01214   } else 
01215     con_data->ms_calculate_md5 = true;
01216   
01217 
01218   // Create the authentication signature and add the 'Authorization' header
01219   if (!s3Authorization)
01220     signed_str = s3_getSignature("PUT", md5, content_type, date, bucket, key);
01221   else
01222     signed_str = CSString::newString(s3Authorization);
01223   push_(signed_str);
01224   s3_buffer->setLength(0);
01225   s3_buffer->append("Authorization: AWS "); 
01226   s3_buffer->append(s3_public_key->getCString());
01227   s3_buffer->append(":"); 
01228   s3_buffer->append(signed_str->getCString());  
01229   release_(signed_str); signed_str = NULL;
01230   con_data->ms_setHeader(s3_buffer->getCString());
01231   
01232   con_data->ms_execute_put_request(RETAIN(input), size);
01233   
01234   if (con_data->ms_retry) {
01235     if (retry_count == s3_maxRetries) {
01236       CSException::throwException(CS_CONTEXT, CS_ERR_GENERIC_ERROR, "S3 operation aborted after max retries.");
01237     }
01238   //printf("RETRY: s3_send()\n");
01239     retry_count++;
01240     input->reset();
01241     self->sleep(s3_sleepTime);
01242     goto retry;
01243   }
01244   
01245   replyHeaders = con_data->ms_reply_headers.takeHeaders();
01246 
01247   release_(con_data);
01248   release_(s3_buffer);
01249   release_(input);
01250   return_(replyHeaders);
01251 }
01252 
01253 //-------------------------------
01254 CSString *CSS3Protocol::s3_getDataURL(const char *bucket, const char *key, uint32_t keep_alive)
01255 {
01256   CSStringBuffer *s3_buffer;
01257   char timeout[32];
01258   CSString *signed_str;
01259   enter_();
01260   
01261   new_(s3_buffer, CSStringBuffer());
01262   push_(s3_buffer);
01263 
01264   snprintf(timeout, 32, "%"PRId32"", ((uint32_t)time(NULL)) + keep_alive);
01265   
01266   signed_str = s3_getSignature("GET", NULL, NULL, timeout, bucket, key);
01267 //printf("Unsafe: \"%s\"\n", signed_str->getCString());
01268   signed_str = urlEncode(signed_str); // Because the signature is in the URL it must be URL encoded.
01269 //printf("  Safe: \"%s\"\n", signed_str->getCString());
01270   push_(signed_str);
01271   
01272   s3_buffer->setLength(0);  
01273   s3_buffer->append("http://");
01274   s3_buffer->append(bucket);
01275   s3_buffer->append("."); 
01276   s3_buffer->append(s3_server->getCString());
01277   s3_buffer->append(key);
01278 
01279   s3_buffer->append("?AWSAccessKeyId=");
01280   s3_buffer->append(s3_public_key->getCString());
01281   s3_buffer->append("&Expires=");
01282   s3_buffer->append(timeout);
01283   s3_buffer->append("&Signature=");
01284   s3_buffer->append(signed_str->getCString());
01285   
01286   release_(signed_str);
01287   
01288   pop_(s3_buffer);
01289   CSString *str = CSString::newString(s3_buffer);
01290   return_(str); 
01291 }
01292 
01293 //#define S3_UNIT_TEST
01294 #ifdef S3_UNIT_TEST
01295 static void show_help_info(const char *cmd)
01296 {
01297   printf("Get authenticated query string:\n\t%s q <bucket> <object_key> <timeout>\n", cmd);
01298   printf("Delete object:\n\t%s d <bucket> <object_key>\n", cmd);
01299   printf("Delete all object with a given prefix:\n\t%s D <bucket> <object_prefix>\n", cmd);
01300   printf("Get object, data will be written to 'prottest.out':\n\t%s g <bucket> <object_key> <timeout>\n", cmd);
01301   printf("Get object header only:\n\t%s h <bucket> <object_key> <timeout>\n", cmd);
01302   printf("Put (Upload) an object:\n\t%s p <bucket> <object_key> <file>\n", cmd);
01303   printf("List objects in the bucket:\n\t%s l <bucket> [<object_prefix> [max_list_size]]\n", cmd);
01304   printf("Copy object:\n\t%s c <src_bucket> <src_object_key> <dst_bucket> <dst_object_key> \n", cmd);
01305   printf("Copy all object with a given prefix:\n\t%s C <src_bucket> <object_key_prefix> <dst_bucket> \n", cmd);
01306 }
01307 
01308 void dump_headers(CSVector *header_array)
01309 {
01310   CSHTTPHeaders headers;
01311   
01312   headers.setHeaders(header_array);
01313   printf("Reply Headers:\n");
01314   printf("--------------\n");
01315   
01316   for (uint32_t i = 0; i < headers.numHeaders(); i++) {
01317     CSHeader *h = headers.getHeader(i);
01318     
01319     printf("%s : %s\n", h->getNameCString(), h->getValueCString());
01320     h->release();
01321   }
01322   printf("--------------\n");
01323   headers.clearHeaders();
01324 }
01325 
01326 int main(int argc, char **argv)
01327 {
01328   CSThread *main_thread;
01329   const char *pub_key;
01330   const char *priv_key;
01331   const char *server;
01332   CSS3Protocol *prot = NULL;
01333   
01334   if (argc < 3) {
01335     show_help_info(argv[0]);
01336     return 0;
01337   }
01338   
01339   if (! CSThread::startUp()) {
01340     CSException::throwException(CS_CONTEXT, ENOMEM, "CSThread::startUp() failed.");
01341     return 1;
01342   }
01343   
01344   cs_init_memory();
01345   
01346   main_thread = new CSThread( NULL);
01347   CSThread::setSelf(main_thread);
01348   
01349   enter_();
01350   try_(a) {
01351   
01352     pub_key = getenv("S3_ACCESS_KEY_ID");
01353     priv_key = getenv("S3_SECRET_ACCESS_KEY");
01354     new_(prot, CSS3Protocol());
01355     push_(prot);
01356     
01357     server = getenv("S3_SERVER");
01358     if ((server == NULL) || (*server == 0))
01359       server = "s3.amazonaws.com/";
01360     prot->s3_setServer(server);
01361     prot->s3_setPublicKey(pub_key);
01362     prot->s3_setPrivateKey(priv_key);
01363     prot->s3_setMaxRetries(0);
01364     
01365     switch (argv[1][0]) {
01366       case 'q': // Get the query string
01367         if (argc == 5) {
01368           CSString *qstr = prot->s3_getDataURL(argv[2], argv[3], atoi(argv[4]));
01369           printf("To test call:\ncurl -L -D - \"%s\"\n", qstr->getCString());
01370           qstr->release();
01371         } else
01372           printf("Bad command: q <bucket> <object_key> <timeout>\n");
01373         
01374         break;
01375       case 'd': // Delete the object
01376         if (argc == 4) {
01377           printf("delete %s %s\n", argv[2], argv[3]);
01378           if (!prot->s3_delete(argv[2], argv[3]))
01379             printf("%s/%s could not be found.\n", argv[2], argv[3]);
01380 
01381         } else
01382           printf("Bad command: d <bucket> <object_key>\n");
01383         
01384         break;
01385       case 'D': // Delete  objects like
01386         if (argc == 4) {
01387           CSVector *list;
01388           CSString *key;
01389           
01390           list = prot->s3_list(argv[2], argv[3]);
01391           push_(list);
01392           while (key = (CSString*) list->take(0)) {
01393             printf("Deleting %s\n", key->getCString());
01394             prot->s3_delete(argv[2], key->getCString());
01395             key->release();
01396           }
01397           release_(list);
01398           
01399         } else
01400           printf("Bad command: D <bucket> <object_key_prefix>\n");
01401         
01402         break;
01403       case 'g':  // Get the object
01404         if ((argc == 4) || (argc == 6)) {
01405           CSFile *output; 
01406           CSVector *headers;
01407           bool found;       
01408           S3RangeRec *range_ptr = NULL, range =   {0,0};    
01409           
01410           if (argc == 6) {
01411             range.startByte = atoi(argv[4]);
01412             range.endByte = atoi(argv[5]);
01413             range_ptr = &range;
01414           }
01415           
01416           output = CSFile::newFile("prottest.out");
01417           push_(output);
01418           output->open(CSFile::CREATE | CSFile::TRUNCATE);
01419           headers = prot->s3_receive(output->getOutputStream(), argv[2], argv[3], &found, range_ptr);
01420           if (!found)
01421             printf("%s/%s could not be found.\n", argv[2], argv[3]);
01422             
01423           dump_headers(headers);
01424             
01425           release_(output);
01426         } else
01427           printf("Bad command: g <bucket> <object_key>\n");
01428         
01429         break;
01430         
01431       case 'h':  // Get the object header
01432         if (argc == 4) {
01433           CSVector *headers;
01434           bool found; 
01435           S3RangeRec range =  {0,0};    
01436           
01437           headers = prot->s3_receive(NULL, argv[2], argv[3], &found);
01438           if (!found)
01439             printf("%s/%s could not be found.\n", argv[2], argv[3]);
01440             
01441           dump_headers(headers);
01442             
01443         } else
01444           printf("Bad command: h <bucket> <object_key>\n");
01445         
01446         break;
01447         
01448       case 'p':  // Put (Upload) the object
01449         if (argc == 5) {
01450           CSFile *input;
01451           Md5Digest digest;
01452           CSVector *headers;
01453           
01454           input = CSFile::newFile(argv[4]);
01455           push_(input);
01456           input->open(CSFile::READONLY);
01457           input->md5Digest(&digest);
01458           headers = prot->s3_send(input->getInputStream(), argv[2], argv[3], input->myFilePath->getSize(), NULL, &digest);
01459           dump_headers(headers);
01460           release_(input);
01461         } else
01462           printf("Bad command: p <bucket> <object_key> <file> \n");
01463         
01464         break;
01465         
01466       case 'c':  // Copy the object
01467         if (argc == 6) {
01468           prot->s3_copy(NULL, argv[4], argv[5], argv[2], argv[3]);
01469         } else
01470           printf("Bad command: c <src_bucket> <src_object_key> <dst_bucket> <dst_object_key>\n");
01471         
01472         break;
01473         
01474       case 'C':  // Copy  objects like
01475         if (argc == 5) {
01476           CSVector *list;
01477           CSString *key;
01478           
01479           list = prot->s3_list(argv[2], argv[3]);
01480           push_(list);
01481           while (key = (CSString*) list->take(0)) {
01482             printf("Copying %s\n", key->getCString());
01483             prot->s3_copy(NULL, argv[4], key->getCString(), argv[2], key->getCString());
01484             key->release();
01485           }
01486           release_(list);
01487           
01488         } else
01489           printf("Bad command: C <src_bucket> <object_key_prefix> <dst_bucket>\n");
01490         
01491         break;
01492       case 'l':  // List the object
01493         if ((argc == 3) || (argc == 4) || (argc == 5)) {
01494           uint32_t max = 0;
01495           char *prefix = NULL;
01496           CSVector *list;
01497           CSString *key;
01498           
01499           if (argc > 3) {
01500             prefix = argv[3];
01501             if (!strlen(prefix))
01502               prefix = NULL;
01503           }
01504           
01505           if (argc == 5) 
01506             max = atol(argv[4]);
01507             
01508           list = prot->s3_list(argv[2], prefix, max);
01509           push_(list);
01510           while (key = (CSString*) list->take(0)) {
01511             printf("%s\n", key->getCString());
01512             key->release();
01513           }
01514           release_(list);
01515           
01516         } else
01517           printf("Bad command: l <bucket> [<object_prefix> [max_list_size]] \n");
01518         
01519         break;
01520       default:
01521         printf("Unknown command.\n");
01522         show_help_info(argv[0]);
01523     }
01524     
01525     release_(prot);
01526   }
01527   
01528   catch_(a);    
01529   self->logException();
01530   
01531   cont_(a);
01532     
01533   outer_()
01534   main_thread->release();
01535   cs_exit_memory();
01536   CSThread::shutDown();
01537   return 0;
01538 }
01539 
01540 #endif
01541 
01542