00001
00002
00003
00004
00005
00006
00007
00008
00009 #include <ctype.h>
00010 #include <time.h>
00011 #include "wvhttppool.h"
00012 #include "wvbufstream.h"
00013 #include "wvtcp.h"
00014 #include "strutils.h"
00015
00016 bool WvHttpStream::global_enable_pipelining = true;
00017 int WvUrlStream::max_requests = 100;
00018
00019 unsigned WvHash(const WvUrlStream::Target &n)
00020 {
00021 WvString key("%s%s", n.remaddr, n.username);
00022 return (WvHash(key));
00023 }
00024
00025
00026 WvUrlRequest::WvUrlRequest(WvStringParm _url, WvStringParm _method,
00027 WvStringParm _headers, WvStream *content_source,
00028 bool _create_dirs, bool _pipeline_test)
00029 : url(_url), headers(_headers)
00030 {
00031 instream = NULL;
00032 create_dirs = _create_dirs;
00033 pipeline_test = _pipeline_test;
00034 method = _method;
00035 is_dir = false;
00036
00037 if (pipeline_test)
00038 {
00039 outstream = NULL;
00040 putstream = NULL;
00041 }
00042 else
00043 {
00044 WvBufUrlStream *x = new WvBufUrlStream;
00045 outstream = x;
00046 x->death_notify = (WvStream **)&outstream;
00047 x->url = url;
00048
00049 putstream = content_source;
00050 }
00051 inuse = false;
00052 }
00053
00054
00055 WvUrlRequest::~WvUrlRequest()
00056 {
00057 done();
00058 }
00059
00060
00061 void WvUrlRequest::done()
00062 {
00063 if (outstream)
00064 {
00065 outstream->death_notify = NULL;
00066 outstream->seteof();
00067 outstream = NULL;
00068 }
00069 if (putstream)
00070 putstream = NULL;
00071 inuse = false;
00072 }
00073
00074
00075 void WvUrlStream::addurl(WvUrlRequest *url)
00076 {
00077 log(WvLog::Debug4, "Adding a new url: '%s'\n", url->url);
00078
00079 assert(url->outstream);
00080
00081 if (!url->url.isok())
00082 return;
00083
00084 waiting_urls.append(url, false, "waiting_url");
00085 request_next();
00086 }
00087
00088
00089 void WvUrlStream::delurl(WvUrlRequest *url)
00090 {
00091 log(WvLog::Debug4, "Removing an url: '%s'\n", url->url);
00092
00093 if (url == curl)
00094 doneurl();
00095 waiting_urls.unlink(url);
00096 urls.unlink(url);
00097 }
00098
00099
00100 WvHttpPool::WvHttpPool()
00101 : log("HTTP Pool", WvLog::Debug), conns(10),
00102 pipeline_incompatible(50)
00103 {
00104 log("Pool initializing.\n");
00105 num_streams_created = 0;
00106 }
00107
00108
00109 WvHttpPool::~WvHttpPool()
00110 {
00111 log("Created %s individual session%s during this run.\n",
00112 num_streams_created, num_streams_created == 1 ? "" : "s");
00113 if (geterr())
00114 log("Error was: %s\n", errstr());
00115
00116
00117
00118 zap();
00119 conns.zap();
00120 }
00121
00122
00123 void WvHttpPool::pre_select(SelectInfo &si)
00124 {
00125
00126
00127
00128 WvIStreamList::pre_select(si);
00129
00130 WvUrlStreamDict::Iter ci(conns);
00131 for (ci.rewind(); ci.next(); )
00132 {
00133 if (!ci->isok())
00134 si.msec_timeout = 0;
00135 }
00136
00137 WvUrlRequestList::Iter i(urls);
00138 for (i.rewind(); i.next(); )
00139 {
00140 if (!i->instream)
00141 {
00142 log(WvLog::Debug4, "Checking dns for '%s'\n", i->url.gethost());
00143 if (i->url.resolve())
00144 si.msec_timeout = 0;
00145 else
00146 dns.pre_select(i->url.gethost(), si);
00147 }
00148 }
00149 }
00150
00151
00152 bool WvHttpPool::post_select(SelectInfo &si)
00153 {
00154 bool sure = false;
00155
00156 WvUrlStreamDict::Iter ci(conns);
00157 for (ci.rewind(); ci.next(); )
00158 {
00159 if (!ci->isok())
00160 {
00161 log(WvLog::Debug4, "Selecting true because of a dead stream.\n");
00162 unconnect(ci.ptr());
00163 ci.rewind();
00164 sure = true;
00165 }
00166 }
00167
00168 WvUrlRequestList::Iter i(urls);
00169 for (i.rewind(); i.next(); )
00170 {
00171 if ((!i->outstream && !i->inuse) || !i->url.isok())
00172 {
00173
00174
00175 if (!i->url.isok())
00176 {
00177 log("URL not okay: '%s'\n", i->url);
00178 i->done();
00179 }
00180
00181 WvUrlStream::Target target(i->url.getaddr(), i->url.getuser());
00182 WvUrlStream *s = conns[target];
00183 if (s)
00184 s->delurl(i.ptr());
00185 i.xunlink();
00186 continue;
00187 }
00188
00189 if (!i->instream)
00190 {
00191 log(WvLog::Debug4, "Checking dns for '%s'\n", i->url.gethost());
00192 if (i->url.resolve() || dns.post_select(i->url.gethost(), si))
00193 {
00194 log(WvLog::Debug4, "Selecting true because of '%s'\n", i->url);
00195 sure = true;
00196 }
00197 }
00198 }
00199
00200 return WvIStreamList::post_select(si) || sure;
00201 }
00202
00203
00204 void WvHttpPool::execute()
00205 {
00206 WvIStreamList::execute();
00207
00208 WvUrlRequestList::Iter i(urls);
00209 for (i.rewind(); i.next(); )
00210 {
00211 WvUrlStream *s;
00212
00213 if (!i->outstream || !i->url.isok() || !i->url.resolve())
00214 continue;
00215
00216 WvUrlStream::Target target(i->url.getaddr(), i->url.getuser());
00217
00218
00219
00220 s = conns[target];
00221
00222
00223 if (s && !s->isok())
00224 {
00225 unconnect(s);
00226 s = NULL;
00227 }
00228
00229 if (!i->outstream)
00230 continue;
00231
00232 if (!s)
00233 {
00234 num_streams_created++;
00235 if (!strncasecmp(i->url.getproto(), "http", 4))
00236 s = new WvHttpStream(target.remaddr, target.username,
00237 i->url.getproto() == "https",
00238 pipeline_incompatible);
00239 else if (!strcasecmp(i->url.getproto(), "ftp"))
00240 s = new WvFtpStream(target.remaddr, target.username,
00241 i->url.getpassword());
00242 conns.add(s, true);
00243
00244
00245 append(s, false, "http/ftp stream");
00246 }
00247
00248 if (!i->instream)
00249 {
00250 s->addurl(i.ptr());
00251 i->instream = s;
00252 }
00253 }
00254 }
00255
00256
00257 WvBufUrlStream *WvHttpPool::addurl(WvStringParm _url, WvStringParm _method,
00258 WvStringParm _headers, WvStream *content_source, bool create_dirs)
00259 {
00260 log(WvLog::Debug4, "Adding a new url to pool: '%s'\n", _url);
00261 WvUrlRequest *url = new WvUrlRequest(_url, _method, _headers, content_source,
00262 create_dirs, false);
00263 urls.append(url, true, "addurl");
00264
00265 return url->outstream;
00266 }
00267
00268
00269 void WvHttpPool::unconnect(WvUrlStream *s)
00270 {
00271 if (!s->target.username)
00272 log("Unconnecting stream to %s.\n", s->target.remaddr);
00273 else
00274 log("Unconnecting stream to %s@%s.\n", s->target.username,
00275 s->target.remaddr);
00276
00277 WvUrlRequestList::Iter i(urls);
00278 for (i.rewind(); i.next(); )
00279 {
00280 if (i->instream == s)
00281 i->instream = NULL;
00282 }
00283
00284 unlink(s);
00285 conns.remove(s);
00286 }