1 /******************************************************************************* 2 3 copyright: Copyright (c) 2004 Kris Bell. All rights reserved 4 5 license: BSD style: $(LICENSE) 6 7 version: Initial release: April 2004 8 Outback release: December 2006 9 10 author: Kris - original module 11 author: h3r3tic - fixed a number of Post issues and 12 bugs in the 'params' construction 13 14 Redirection handling guided via 15 http://ppewww.ph.gla.ac.uk/~flavell/www/post-redirect.html 16 17 *******************************************************************************/ 18 19 module tango.net.http.HttpClient; 20 21 private import tango.time.Time; 22 23 private import tango.net.Uri, 24 tango.net.device.Socket, 25 tango.net.InternetAddress; 26 27 private import tango.io.device.Array; 28 29 private import tango.io.stream.Lines; 30 private import tango.io.stream.Buffered; 31 32 private import tango.net.http.HttpConst, 33 tango.net.http.HttpParams, 34 tango.net.http.HttpHeaders, 35 tango.net.http.HttpTriplet, 36 tango.net.http.HttpCookies; 37 38 private import tango.core.Exception : IOException; 39 40 private import Integer = tango.text.convert.Integer; 41 42 /******************************************************************************* 43 44 Supports the basic needs of a client making requests of an HTTP 45 server. The following is an example of how this might be used: 46 47 --- 48 // callback for client reader 49 void sink (void[] content) 50 { 51 Stdout (cast(char[]) content); 52 } 53 54 // create client for a GET request 55 auto client = new HttpClient (HttpClient.Get, "http://www.yahoo.com"); 56 57 // make request 58 client.open; 59 60 // check return status for validity 61 if (client.isResponseOK) 62 { 63 // extract content length 64 auto length = client.getResponseHeaders.getInt (HttpHeader.ContentLength); 65 66 // display all returned headers 67 Stdout (client.getResponseHeaders); 68 69 // display remaining content 70 client.read (&sink, length); 71 } 72 else 73 Stderr (client.getResponse); 74 75 client.close(); 76 --- 77 78 See modules HttpGet and HttpPost for simple wrappers instead. 79 80 *******************************************************************************/ 81 82 class HttpClient 83 { 84 /// callback for sending PUT content 85 alias scope void delegate (OutputBuffer) Pump; 86 87 // this is struct rather than typedef to avoid compiler bugs 88 private struct RequestMethod 89 { 90 const(char)[] name; 91 } 92 93 // class members; there's a surprising amount of stuff here! 94 private Uri uri; 95 private BufferedInput input; 96 private BufferedOutput output; 97 private Array tokens; 98 private Lines!(char) line; 99 private Socket socket; 100 private RequestMethod method; 101 private InternetAddress address; 102 private HttpParams paramsOut; 103 private HttpHeadersView headersIn; 104 private HttpHeaders headersOut; 105 private HttpCookies cookiesOut; 106 private ResponseLine responseLine; 107 108 // default to three second timeout on read operations ... 109 private float timeout = 3.0; 110 111 // enable uri encoding? 112 private bool encode = true; 113 114 // should we perform internal redirection? 115 private bool doRedirect = true; 116 117 // attempt keepalive? 118 private bool keepalive = false; 119 120 // limit the number of redirects, or catch circular redirects 121 private uint redirections, 122 redirectionLimit = 5; 123 124 // the http version being sent with requests 125 private const(char)[] httpVersion; 126 127 // http version id 128 public enum Version {OnePointZero, OnePointOne}; 129 130 // standard set of request methods ... 131 static immutable RequestMethod Get = {"GET"}, 132 Put = {"PUT"}, 133 Head = {"HEAD"}, 134 Post = {"POST"}, 135 Trace = {"TRACE"}, 136 Delete = {"DELETE"}, 137 Options = {"OPTIONS"}, 138 Connect = {"CONNECT"}; 139 140 /*********************************************************************** 141 142 Create a client for the given URL. The argument should be 143 fully qualified with an "http:" or "https:" scheme, or an 144 explicit port should be provided. 145 146 ***********************************************************************/ 147 148 this (RequestMethod method, const(char)[] url) 149 { 150 this (method, new Uri(url)); 151 } 152 153 /*********************************************************************** 154 155 Create a client with the provided Uri instance. The Uri should 156 be fully qualified with an "http:" or "https:" scheme, or an 157 explicit port should be provided. 158 159 ***********************************************************************/ 160 161 this (RequestMethod method, Uri uri) 162 { 163 this.uri = uri; 164 this.method = method; 165 166 responseLine = new ResponseLine; 167 headersIn = new HttpHeadersView; 168 tokens = new Array (1024 * 4); 169 170 input = new BufferedInput (null, 1024 * 16); 171 output = new BufferedOutput (null, 1024 * 16); 172 173 paramsOut = new HttpParams; 174 headersOut = new HttpHeaders; 175 cookiesOut = new HttpCookies (headersOut, HttpHeader.Cookie); 176 177 // decode the host name (may take a second or two) 178 auto host = uri.getHost; 179 if (host) 180 address = new InternetAddress (host, uri.getValidPort()); 181 else 182 error ("invalid url provided to HttpClient ctor"); 183 184 paramsOut.parse(new Array(cast(void[]) uri.query)); 185 186 // default the http version to 1.0 187 setVersion (Version.OnePointZero); 188 } 189 190 /*********************************************************************** 191 192 Get the current input headers, as returned by the host request. 193 194 ***********************************************************************/ 195 196 HttpHeadersView getResponseHeaders() 197 { 198 return headersIn; 199 } 200 201 /*********************************************************************** 202 203 Gain access to the request headers. Use this to add whatever 204 headers are required for a request. 205 206 ***********************************************************************/ 207 208 HttpHeaders getRequestHeaders() 209 { 210 return headersOut; 211 } 212 213 /*********************************************************************** 214 215 Gain access to the request parameters. Use this to add x=y 216 style parameters to the request. These will be appended to 217 the request assuming the original Uri does not contain any 218 of its own. 219 220 ***********************************************************************/ 221 222 HttpParams getRequestParams() 223 { 224 return paramsOut; 225 } 226 227 /*********************************************************************** 228 229 Return the Uri associated with this client 230 231 ***********************************************************************/ 232 233 UriView getUri() 234 { 235 return uri; 236 } 237 238 /*********************************************************************** 239 240 Return the response-line for the latest request. This takes 241 the form of "version status reason" as defined in the HTTP 242 RFC. 243 244 ***********************************************************************/ 245 246 ResponseLine getResponse() 247 { 248 return responseLine; 249 } 250 251 /*********************************************************************** 252 253 Return the HTTP status code set by the remote server 254 255 ***********************************************************************/ 256 257 int getStatus() 258 { 259 return responseLine.getStatus(); 260 } 261 262 /*********************************************************************** 263 264 Return whether the response was OK or not 265 266 ***********************************************************************/ 267 268 bool isResponseOK() 269 { 270 return getStatus() is HttpResponseCode.OK; 271 } 272 273 /*********************************************************************** 274 275 Add a cookie to the outgoing headers 276 277 ***********************************************************************/ 278 279 HttpClient addCookie (Cookie cookie) 280 { 281 cookiesOut.add (cookie); 282 return this; 283 } 284 285 /*********************************************************************** 286 287 Close all resources used by a request. You must invoke this 288 between successive open() calls. 289 290 ***********************************************************************/ 291 292 void close () 293 { 294 if (socket) 295 { 296 socket.shutdown(); 297 socket.detach(); 298 socket = null; 299 } 300 } 301 302 /*********************************************************************** 303 304 Reset the client such that it is ready for a new request. 305 306 ***********************************************************************/ 307 308 HttpClient reset () 309 { 310 headersIn.reset(); 311 headersOut.reset(); 312 paramsOut.reset(); 313 redirections = 0; 314 return this; 315 } 316 317 /*********************************************************************** 318 319 Set the request method 320 321 ***********************************************************************/ 322 323 HttpClient setRequest (RequestMethod method) 324 { 325 this.method = method; 326 return this; 327 } 328 329 /*********************************************************************** 330 331 Set the request version 332 333 ***********************************************************************/ 334 335 HttpClient setVersion (Version v) 336 { 337 __gshared immutable versions = ["HTTP/1.0", "HTTP/1.1"]; 338 339 httpVersion = versions[v]; 340 return this; 341 } 342 343 /*********************************************************************** 344 345 enable/disable the internal redirection suppport 346 347 ***********************************************************************/ 348 349 HttpClient enableRedirect (bool yes = true) 350 { 351 doRedirect = yes; 352 return this; 353 } 354 355 /*********************************************************************** 356 357 set timeout period for read operation 358 359 ***********************************************************************/ 360 361 HttpClient setTimeout (float interval) 362 { 363 timeout = interval; 364 return this; 365 } 366 367 /*********************************************************************** 368 369 Control keepalive option 370 371 ***********************************************************************/ 372 373 HttpClient keepAlive (bool yes = true) 374 { 375 keepalive = yes; 376 return this; 377 } 378 379 /*********************************************************************** 380 381 Control Uri output encoding 382 383 ***********************************************************************/ 384 385 HttpClient encodeUri (bool yes = true) 386 { 387 encode = yes; 388 return this; 389 } 390 391 /*********************************************************************** 392 393 Make a request for the resource specified via the constructor, 394 using the specified timeout period (in milli-seconds).The 395 return value represents the input buffer, from which all 396 returned headers and content may be accessed. 397 398 ***********************************************************************/ 399 400 InputBuffer open () 401 { 402 return open (method, null); 403 } 404 405 /*********************************************************************** 406 407 Make a request for the resource specified via the constructor, 408 using a callback for pumping additional data to the host. This 409 defaults to a three-second timeout period. The return value 410 represents the input buffer, from which all returned headers 411 and content may be accessed. 412 413 ***********************************************************************/ 414 415 InputBuffer open (Pump pump) 416 { 417 return open (method, pump); 418 } 419 420 /*********************************************************************** 421 422 Make a request for the resource specified via the constructor 423 using the specified timeout period (in micro-seconds), and a 424 user-defined callback for pumping additional data to the host. 425 The callback would be used when uploading data during a 'put' 426 operation (or equivalent). The return value represents the 427 input buffer, from which all returned headers and content may 428 be accessed. 429 430 Note that certain request-headers may generated automatically 431 if they are not present. These include a Host header and, in 432 the case of Post, both ContentType & ContentLength for a query 433 type of request. The latter two are *not* produced for Post 434 requests with 'pump' specified ~ when using 'pump' to output 435 additional content, you must explicitly set your own headers. 436 437 Note also that IOException instances may be thrown. These 438 should be caught by the client to ensure a close() operation 439 is always performed 440 441 ***********************************************************************/ 442 443 InputBuffer open (RequestMethod method, Pump pump) 444 { 445 try { 446 this.method = method; 447 if (++redirections > redirectionLimit) 448 error ("too many redirections, or a circular redirection"); 449 450 // new socket for each request? 451 if (keepalive is false) 452 close(); 453 454 // create socket and connect it. Retain prior socket if 455 // not closed between calls 456 if (socket is null) 457 { 458 socket = createSocket(); 459 socket.timeout = cast(int)(timeout * 1000); 460 socket.connect (address); 461 } 462 463 // setup buffers for input and output 464 output.output (socket); 465 input.input (socket); 466 input.clear(); 467 468 // setup a Host header 469 if (headersOut.get (HttpHeader.Host, null) is null) 470 headersOut.add (HttpHeader.Host, uri.getHost); 471 472 // http/1.0 needs connection:close 473 if (keepalive is false) 474 headersOut.add (HttpHeader.Connection, "close"); 475 476 // format encoded request 477 output.append (method.name) 478 .append (" "); 479 480 // patch request path? 481 auto path = uri.getPath; 482 if (path.length is 0) 483 path = "/"; 484 485 // emit path 486 if (encode) 487 uri.encode (&output.write, path, uri.IncPath); 488 else 489 output.append (path); 490 491 // attach/extend query parameters if user has added some 492 tokens.clear(); 493 paramsOut.produce ((const(void)[] p){if (tokens.readable) tokens.write("&"); 494 return uri.encode(&tokens.write, cast(char[]) p, uri.IncQuery);}); 495 auto query = cast(char[]) tokens.slice(); 496 497 // emit query? 498 if (query.length) 499 { 500 output.append ("?").append(query); 501 502 if (method is Post && pump.funcptr is null) 503 { 504 // we're POSTing query text - add default info 505 if (headersOut.get (HttpHeader.ContentType, null) is null) 506 headersOut.add (HttpHeader.ContentType, "application/x-www-form-urlencoded"); 507 508 if (headersOut.get (HttpHeader.ContentLength, null) is null) 509 { 510 headersOut.addInt (HttpHeader.ContentLength, query.length); 511 pump = (OutputBuffer o){o.append(query);}; 512 } 513 } 514 } 515 516 // complete the request line, and emit headers too 517 output.append (" ") 518 .append (httpVersion) 519 .append (HttpConst.Eol); 520 521 headersOut.produce (&output.write, HttpConst.Eol); 522 output.append (HttpConst.Eol); 523 524 if (pump.funcptr) 525 pump (output); 526 527 // send entire request 528 output.flush(); 529 530 // Token for initial parsing of input header lines 531 if (line is null) 532 line = new Lines!(char) (input); 533 else 534 line.set(input); 535 536 // skip any blank lines 537 while (line.next && line.get().length is 0) 538 {} 539 540 // is this a bogus request? 541 if (line.get().length is 0) 542 error ("truncated response"); 543 544 // read response line 545 if (! responseLine.parse (line.get())) 546 error (responseLine.error()); 547 548 // parse incoming headers 549 headersIn.reset().parse (this.input); 550 551 // check for redirection 552 if (doRedirect) 553 switch (responseLine.getStatus()) 554 { 555 case HttpResponseCode.Found: 556 case HttpResponseCode.SeeOther: 557 case HttpResponseCode.MovedPermanently: 558 case HttpResponseCode.TemporaryRedirect: 559 // drop this connection 560 close(); 561 562 // remove any existing Host header 563 headersOut.remove (HttpHeader.Host); 564 565 // parse redirected uri 566 auto redirect = headersIn.get (HttpHeader.Location, "[missing Location header]"); 567 uri.relParse (redirect.dup); 568 569 // decode the host name (may take a second or two) 570 auto host = uri.getHost(); 571 if (host) 572 address = new InternetAddress (uri.getHost(), uri.getValidPort()); 573 else 574 error ("redirect has invalid url: "~redirect); 575 576 // figure out what to do 577 if (method is Get || method is Head) 578 return open (method, pump); 579 else 580 if (method is Post) 581 return redirectPost (pump, responseLine.getStatus()); 582 else 583 error ("unexpected redirect for method "~method.name); 584 goto default; 585 default: 586 break; 587 } 588 589 // return the input buffer 590 return input; 591 } finally {redirections = 0;} 592 } 593 594 /*********************************************************************** 595 596 Read the content from the returning input stream, up to a 597 maximum length, and pass content to the given sink delegate 598 as it arrives. 599 600 Exits when length bytes have been processed, or an Eof is 601 seen on the stream. 602 603 ***********************************************************************/ 604 605 void read (scope void delegate(const(void)[]) sink, size_t len = size_t.max) 606 { 607 while (true) 608 { 609 auto content = input.slice(); 610 if (content.length > len) 611 { 612 sink (content [0 .. len]); 613 input.skip (len); 614 break; 615 } 616 else 617 { 618 len -= content.length; 619 sink (content); 620 input.clear(); 621 if (input.populate() is input.Eof) 622 break; 623 } 624 } 625 } 626 627 /*********************************************************************** 628 629 Handle redirection of Post 630 631 Guidance for the default behaviour came from this page: 632 http://ppewww.ph.gla.ac.uk/~flavell/www/post-redirect.html 633 634 ***********************************************************************/ 635 636 InputBuffer redirectPost (Pump pump, int status) 637 { 638 switch (status) 639 { 640 // use Get method to complete the Post 641 case HttpResponseCode.Found: 642 case HttpResponseCode.SeeOther: 643 644 // remove POST headers first! 645 headersOut.remove (HttpHeader.ContentLength); 646 headersOut.remove (HttpHeader.ContentType); 647 paramsOut.reset(); 648 return open (Get, null); 649 650 // try entire Post again, if user say OK 651 case HttpResponseCode.MovedPermanently: 652 case HttpResponseCode.TemporaryRedirect: 653 if (canRepost (status)) 654 return open (this.method, pump); 655 // fall through! 656 goto default; 657 default: 658 error ("Illegal redirection of Post"); 659 } 660 return null; 661 } 662 663 /*********************************************************************** 664 665 Handle user-notification of Post redirection. This should 666 be overridden appropriately. 667 668 Guidance for the default behaviour came from this page: 669 http://ppewww.ph.gla.ac.uk/~flavell/www/post-redirect.html 670 671 ***********************************************************************/ 672 673 bool canRepost (uint status) 674 { 675 return false; 676 } 677 678 /*********************************************************************** 679 680 Overridable socket factory, for use with HTTPS and so on 681 682 ***********************************************************************/ 683 684 protected Socket createSocket () 685 { 686 return new Socket; 687 } 688 689 /********************************************************************** 690 691 throw an exception, after closing the socket 692 693 **********************************************************************/ 694 695 private void error (const(char)[] msg) 696 { 697 close(); 698 throw new IOException (msg.idup); 699 } 700 } 701 702 703 /****************************************************************************** 704 705 Class to represent an HTTP response-line 706 707 ******************************************************************************/ 708 709 private class ResponseLine : HttpTriplet 710 { 711 private const(char)[] vers, 712 reason; 713 private int status; 714 715 /********************************************************************** 716 717 test the validity of these tokens 718 719 **********************************************************************/ 720 721 override bool test () 722 { 723 vers = tokens[0]; 724 reason = tokens[2]; 725 status = cast(int) Integer.convert (tokens[1]); 726 if (status is 0) 727 { 728 status = cast(int) Integer.convert (tokens[2]); 729 if (status is 0) 730 { 731 failed = "Invalid HTTP response: '"~tokens[0]~"' '"~tokens[1]~"' '" ~tokens[2] ~"'"; 732 return false; 733 } 734 } 735 return true; 736 } 737 738 /********************************************************************** 739 740 Return HTTP version 741 742 **********************************************************************/ 743 744 const(char)[] getVersion () 745 { 746 return vers; 747 } 748 749 /********************************************************************** 750 751 Return reason text 752 753 **********************************************************************/ 754 755 const(char)[] getReason () 756 { 757 return reason; 758 } 759 760 /********************************************************************** 761 762 Return status integer 763 764 **********************************************************************/ 765 766 int getStatus () 767 { 768 return status; 769 } 770 } 771 772 773 /****************************************************************************** 774 775 ******************************************************************************/ 776 777 debug (HttpClient) 778 { 779 import tango.io.Stdout; 780 781 void main() 782 { 783 // callback for client reader 784 void sink (const(void)[] content) 785 { 786 Stdout (cast(const(char)[]) content); 787 } 788 789 // create client for a GET request 790 auto client = new HttpClient (HttpClient.Get, "http://www.microsoft.com"); 791 792 // make request 793 client.open; 794 795 // check return status for validity 796 if (client.isResponseOK) 797 { 798 // display all returned headers 799 foreach (header; client.getResponseHeaders) 800 Stdout.formatln ("{} {}", header.name.value, header.value); 801 802 // extract content length 803 auto length = client.getResponseHeaders.getInt (HttpHeader.ContentLength); 804 805 // display remaining content 806 client.read (&sink, length); 807 } 808 else 809 Stderr (client.getResponse); 810 811 client.close(); 812 } 813 }