wvhttppool.h

00001 /* -*- Mode: C++ -*-
00002  * Worldvisions Weaver Software:
00003  *   Copyright (C) 1997-2002 Net Integration Technologies, Inc.
00004  *
00005  * A fast, easy-to-use, parallelizing, pipelining HTTP/1.1 file retriever.
00006  * 
00007  * Just create a WvHttpPool object, add it to your list, and use pool.addurl()
00008  * to get a WvStream* that gives you the file you requested.
00009  */ 
00010 #ifndef __WVHTTPPOOL_H
00011 #define __WVHTTPPOOL_H
00012 
00013 #include "ftpparse.h"
00014 #include "wvurl.h"
00015 #include "wvistreamlist.h"
00016 #include "wvstreamclone.h"
00017 #include "wvlog.h"
00018 #include "wvhashtable.h"
00019 #include "wvhttp.h"
00020 #include "wvbufstream.h"
00021 #include "wvbuf.h"
00022 #include "wvcont.h"
00023 
00024 class WvBufUrlStream;
00025 class WvUrlStream;
00026 class WvHttpStream;
00027 
00028 static const WvString DEFAULT_ANON_PW("weasels@");
00029 
00030 class WvUrlRequest
00031 {
00032 public:
00033     WvUrl url;
00034     WvString headers;
00035     WvUrlStream *instream;
00036     WvBufUrlStream *outstream;
00037     WvStream *putstream;
00038 
00039     bool pipeline_test;
00040     bool inuse;
00041     bool is_dir;
00042     bool create_dirs;
00043     WvString method;
00044     
00045     WvUrlRequest(WvStringParm _url, WvStringParm _method, WvStringParm _headers,
00046                  WvStream *content_source, bool _create_dirs, bool _pipeline_test);
00047     ~WvUrlRequest();
00048     
00049     void done();
00050 };
00051 
00052 DeclareWvList(WvUrlRequest);
00053 
00054 
00055 struct WvUrlLink
00056 {
00057     WvString linkname;
00058     WvUrl url;
00059 
00060     WvUrlLink(WvStringParm _linkname, WvStringParm _url)
00061         : linkname(_linkname), url(_url)
00062     {}
00063 };
00064 DeclareWvList(WvUrlLink);
00065 
00066 
00067 class WvBufUrlStream : public WvBufStream
00068 {
00069 public:
00070     WvString url;
00071     WvString proto;
00072     WvUrlLinkList links;  // HTML links or FTP directory listing
00073 
00074     // HTTP stuff...
00075     WvString version;
00076     int status;
00077     WvHTTPHeaderDict headers; 
00078 
00079     WvBufUrlStream() : status(0), headers(10)
00080         {}
00081     virtual ~WvBufUrlStream()
00082         {}
00083 
00084 public:
00085     const char *wstype() const { return "WvBufUrlStream"; }
00086 };
00087 
00088 DeclareWvTable(WvIPPortAddr);
00089 
00090 
00091 class WvUrlStream : public WvStreamClone
00092 {
00093 public:
00094     class Target
00095     {
00096     public:
00097         WvIPPortAddr remaddr;
00098         WvString username;
00099 
00100         Target(const WvIPPortAddr &_remaddr, WvStringParm _username)
00101             : remaddr(_remaddr), username(_username) {}
00102 
00103         ~Target() {}
00104 
00105         bool operator== (const Target &n2) const
00106         { return (username == n2.username && remaddr == n2.remaddr); }
00107     };
00108     Target target;
00109     static int max_requests;
00110 
00111 protected:
00112     WvLog log;
00113     WvUrlRequestList urls, waiting_urls;
00114     int request_count;
00115     WvUrlRequest *curl; // current url
00116     virtual void doneurl() = 0;
00117     virtual void request_next() = 0;
00118 
00119 public:
00120     WvUrlStream(const WvIPPortAddr &_remaddr, WvStringParm _username, 
00121         WvStringParm logname)
00122         : WvStreamClone(new WvTCPConn(_remaddr)), target(_remaddr, _username),
00123           log(logname, WvLog::Debug)
00124     {
00125         request_count = 0;
00126         curl = NULL;
00127     }
00128 
00129     virtual ~WvUrlStream() {};
00130 
00131     virtual void close() = 0;
00132     void addurl(WvUrlRequest *url);
00133     void delurl(WvUrlRequest *url);
00134     // only implemented in WvHttpStream
00135     virtual size_t remaining()
00136     { return 0; }
00137     
00138     virtual void execute() = 0;
00139     
00140 public:
00141     const char *wstype() const { return "WvUrlStream"; }
00142 };
00143 
00144 unsigned WvHash(const WvUrlStream::Target &n);
00145 
00146 DeclareWvDict(WvUrlStream, WvUrlStream::Target, target);
00147 
00148 
00149 class WvHttpStream : public WvUrlStream
00150 {
00151 public:
00152     static bool global_enable_pipelining;
00153     bool enable_pipelining;
00154     
00155 private:
00156     int pipeline_test_count;
00157     bool ssl;
00158     bool sent_url_request;      // Have we sent a request to the server yet?
00159     WvIPPortAddrTable &pipeline_incompatible;
00160     WvString http_response, pipeline_test_response;
00161     WvDynBuf putstream_data;
00162     
00163     enum { Unknown, Chunked, ContentLength, Infinity,
00164            PostHeadInfinity, PostHeadChunked, PostHeadStream,
00165            ChuckInfinity, ChuckChunked, ChuckStream } encoding;
00166     size_t bytes_remaining;
00167     bool in_chunk_trailer, last_was_pipeline_test, in_doneurl;
00168 
00169     virtual void doneurl();
00170     virtual void request_next();
00171     void start_pipeline_test(WvUrl *url);
00172     WvString request_str(WvUrlRequest *url, bool keep_alive);
00173     void send_request(WvUrlRequest *url);
00174     void pipelining_is_broken(int why);
00175     
00176 public:
00177     WvHttpStream(const WvIPPortAddr &_remaddr, WvStringParm _username,
00178          bool ssl, WvIPPortAddrTable &_pipeline_incompatible);
00179     virtual ~WvHttpStream();
00180 
00181     virtual void close();
00182     virtual bool pre_select(SelectInfo &si);
00183     virtual bool post_select(SelectInfo &si);
00184     virtual void execute();
00185     virtual size_t remaining()
00186     { return bytes_remaining; }
00187     
00188 public:
00189     const char *wstype() const { return "WvHttpStream"; }
00190 };
00191 
00192 
00193 class WvFtpStream : public WvUrlStream
00194 {
00195     bool logged_in, pasv_acked;
00196     WvString password;
00197     WvTCPConn *data;
00198     time_t last_request_time;
00199 
00200     virtual void doneurl();
00201     virtual void request_next();
00202 
00203     // Disregard all lines that are of the form "xxx-", meaning that another
00204     // line follows.  Only the last line is important for us.
00205     char *get_important_line();
00206 
00207     // Parse response to "PASV" command and returns a pointer to the address
00208     // of the data port (or NULL if it can't parse the response)..
00209     // This mucks about with line.
00210     WvIPPortAddr *parse_pasv_response(char *line);
00211 
00212     WvString parse_for_links(char *line);
00213 
00214     WvCont cont;
00215     void* real_execute(void*);
00216 
00217 public:
00218     WvFtpStream(const WvIPPortAddr &_remaddr, WvStringParm _username,
00219                 WvStringParm _password);
00220 
00221     virtual bool pre_select(SelectInfo &si);
00222     virtual bool post_select(SelectInfo &si);
00223     virtual void close();
00224     virtual void execute();
00225     
00226 public:
00227     const char *wstype() const { return "WvFtpStream"; }
00228 };
00229 
00230 
00231 // FIXME: Rename this to WvUrlPool someday.
00232 class WvHttpPool : public WvIStreamList
00233 {
00234     WvLog log;
00235     WvResolver dns;
00236     WvUrlStreamDict conns;
00237     WvUrlRequestList urls;
00238     int num_streams_created;
00239     
00240     WvIPPortAddrTable pipeline_incompatible;
00241     
00242 public:
00243     WvHttpPool();
00244     virtual ~WvHttpPool();
00245     
00246     virtual bool pre_select(SelectInfo &si);
00247     virtual void execute();
00248     
00249     WvBufUrlStream *addurl(WvStringParm _url, WvStringParm _method = "GET",
00250                             WvStringParm _headers = "",
00251                             WvStream *content_source = NULL,
00252                             bool create_dirs = false);
00253 
00254     // For URL uploads.  create_dirs should be true if you want all
00255     // non-existent directories in _url to be created.
00256 //    WvBufUrlStream *addputurl(WvStringParm _url, WvStringParm _headers,
00257 //                            WvStream *s, bool create_dirs = false);
00258 private:
00259     void unconnect(WvUrlStream *s);
00260     
00261 public:
00262     bool idle() const 
00263         { return !urls.count(); }
00264     
00265 public:
00266     const char *wstype() const { return "WvHttpPool"; }
00267 };
00268 
00269 
00270 #endif // __WVHTTPPOOL_H

Generated on Fri Oct 5 18:20:28 2007 for WvStreams by  doxygen 1.5.3