1 // Written in the D programming language. 2 3 /** 4 Networking client functionality as provided by $(HTTP curl.haxx.se/libcurl, 5 libcurl). The libcurl library must be installed on the system in order to use 6 this module. 7 8 $(SCRIPT inhibitQuickIndex = 1;) 9 10 $(DIVC quickindex, 11 $(BOOKTABLE , 12 $(TR $(TH Category) $(TH Functions) 13 ) 14 $(TR $(TDNW High level) $(TD $(MYREF download) $(MYREF upload) $(MYREF get) 15 $(MYREF post) $(MYREF put) $(MYREF del) $(MYREF options) $(MYREF trace) 16 $(MYREF connect) $(MYREF byLine) $(MYREF byChunk) 17 $(MYREF byLineAsync) $(MYREF byChunkAsync) ) 18 ) 19 $(TR $(TDNW Low level) $(TD $(MYREF HTTP) $(MYREF FTP) $(MYREF 20 SMTP) ) 21 ) 22 ) 23 ) 24 25 Note: 26 You may need to link with the $(B curl) library, e.g. by adding $(D "libs": ["curl"]) 27 to your $(B dub.json) file if you are using $(LINK2 http://code.dlang.org, DUB). 28 29 Windows x86 note: 30 A DMD compatible libcurl static library can be downloaded from the dlang.org 31 $(LINK2 https://downloads.dlang.org/other/index.html, download archive page). 32 33 This module is not available for iOS, tvOS or watchOS. 34 35 Compared to using libcurl directly, this module allows simpler client code for 36 common uses, requires no unsafe operations, and integrates better with the rest 37 of the language. Furthermore it provides $(MREF_ALTTEXT range, std,range) 38 access to protocols supported by libcurl both synchronously and asynchronously. 39 40 A high level and a low level API are available. The high level API is built 41 entirely on top of the low level one. 42 43 The high level API is for commonly used functionality such as HTTP/FTP get. The 44 $(LREF byLineAsync) and $(LREF byChunkAsync) functions asynchronously 45 perform the request given, outputting the fetched content into a $(MREF_ALTTEXT range, std,range). 46 47 The low level API allows for streaming, setting request headers and cookies, and other advanced features. 48 49 $(BOOKTABLE Cheat Sheet, 50 $(TR $(TH Function Name) $(TH Description) 51 ) 52 $(LEADINGROW High level) 53 $(TR $(TDNW $(LREF download)) $(TD $(D 54 download("ftp.digitalmars.com/sieve.ds", "/tmp/downloaded-ftp-file")) 55 downloads file from URL to file system.) 56 ) 57 $(TR $(TDNW $(LREF upload)) $(TD $(D 58 upload("/tmp/downloaded-ftp-file", "ftp.digitalmars.com/sieve.ds");) 59 uploads file from file system to URL.) 60 ) 61 $(TR $(TDNW $(LREF get)) $(TD $(D 62 get("dlang.org")) returns a char[] containing the dlang.org web page.) 63 ) 64 $(TR $(TDNW $(LREF put)) $(TD $(D 65 put("dlang.org", "Hi")) returns a char[] containing 66 the dlang.org web page. after a HTTP PUT of "hi") 67 ) 68 $(TR $(TDNW $(LREF post)) $(TD $(D 69 post("dlang.org", "Hi")) returns a char[] containing 70 the dlang.org web page. after a HTTP POST of "hi") 71 ) 72 $(TR $(TDNW $(LREF byLine)) $(TD $(D 73 byLine("dlang.org")) returns a range of char[] containing the 74 dlang.org web page.) 75 ) 76 $(TR $(TDNW $(LREF byChunk)) $(TD $(D 77 byChunk("dlang.org", 10)) returns a range of ubyte[10] containing the 78 dlang.org web page.) 79 ) 80 $(TR $(TDNW $(LREF byLineAsync)) $(TD $(D 81 byLineAsync("dlang.org")) asynchronously returns a range of char[] containing the dlang.org web 82 page.) 83 ) 84 $(TR $(TDNW $(LREF byChunkAsync)) $(TD $(D 85 byChunkAsync("dlang.org", 10)) asynchronously returns a range of ubyte[10] containing the 86 dlang.org web page.) 87 ) 88 $(LEADINGROW Low level 89 ) 90 $(TR $(TDNW $(LREF HTTP)) $(TD Struct for advanced HTTP usage)) 91 $(TR $(TDNW $(LREF FTP)) $(TD Struct for advanced FTP usage)) 92 $(TR $(TDNW $(LREF SMTP)) $(TD Struct for advanced SMTP usage)) 93 ) 94 95 96 Example: 97 --- 98 import std.net.curl, std.stdio; 99 100 // Return a char[] containing the content specified by a URL 101 auto content = get("dlang.org"); 102 103 // Post data and return a char[] containing the content specified by a URL 104 auto content = post("mydomain.com/here.cgi", ["name1" : "value1", "name2" : "value2"]); 105 106 // Get content of file from ftp server 107 auto content = get("ftp.digitalmars.com/sieve.ds"); 108 109 // Post and print out content line by line. The request is done in another thread. 110 foreach (line; byLineAsync("dlang.org", "Post data")) 111 writeln(line); 112 113 // Get using a line range and proxy settings 114 auto client = HTTP(); 115 client.proxy = "1.2.3.4"; 116 foreach (line; byLine("dlang.org", client)) 117 writeln(line); 118 --- 119 120 For more control than the high level functions provide, use the low level API: 121 122 Example: 123 --- 124 import std.net.curl, std.stdio; 125 126 // GET with custom data receivers 127 auto http = HTTP("dlang.org"); 128 http.onReceiveHeader = 129 (in char[] key, in char[] value) { writeln(key, ": ", value); }; 130 http.onReceive = (ubyte[] data) { /+ drop +/ return data.length; }; 131 http.perform(); 132 --- 133 134 First, an instance of the reference-counted HTTP struct is created. Then the 135 custom delegates are set. These will be called whenever the HTTP instance 136 receives a header and a data buffer, respectively. In this simple example, the 137 headers are written to stdout and the data is ignored. If the request is 138 stopped before it has finished then return something less than data.length from 139 the onReceive callback. See $(LREF onReceiveHeader)/$(LREF onReceive) for more 140 information. Finally, the HTTP request is performed by calling perform(), which is 141 synchronous. 142 143 Source: $(PHOBOSSRC std/net/curl.d) 144 145 Copyright: Copyright Jonas Drewsen 2011-2012 146 License: $(HTTP www.boost.org/LICENSE_1_0.txt, Boost License 1.0). 147 Authors: Jonas Drewsen. Some of the SMTP code contributed by Jimmy Cao. 148 149 Credits: The functionality is based on $(HTTP curl.haxx.se/libcurl, libcurl). 150 libcurl is licensed under an MIT/X derivative license. 151 */ 152 /* 153 Copyright Jonas Drewsen 2011 - 2012. 154 Distributed under the Boost Software License, Version 1.0. 155 (See accompanying file LICENSE_1_0.txt or copy at 156 http://www.boost.org/LICENSE_1_0.txt) 157 */ 158 module std.net.curl; 159 160 public import etc.c.curl : CurlOption; 161 import core.time : dur; 162 import etc.c.curl : CURLcode; 163 import std.range.primitives; 164 import std.encoding : EncodingScheme; 165 import std.traits : isSomeChar; 166 import std.typecons : Flag, Yes, No, Tuple; 167 168 version (iOS) 169 version = iOSDerived; 170 else version (TVOS) 171 version = iOSDerived; 172 else version (WatchOS) 173 version = iOSDerived; 174 175 version (iOSDerived) {} 176 else: 177 178 version (StdUnittest) 179 { 180 import std.socket : Socket, SocketShutdown; 181 182 private struct TestServer 183 { 184 import std.concurrency : Tid; 185 186 import std.socket : Socket, TcpSocket; 187 188 string addr() { return _addr; } 189 190 void handle(void function(Socket s) dg) 191 { 192 import std.concurrency : send; 193 tid.send(dg); 194 } 195 196 private: 197 string _addr; 198 Tid tid; 199 TcpSocket sock; 200 201 static void loop(shared TcpSocket listener) 202 { 203 import std.concurrency : OwnerTerminated, receiveOnly; 204 import std.stdio : stderr; 205 206 try while (true) 207 { 208 void function(Socket) handler = void; 209 try 210 handler = receiveOnly!(typeof(handler)); 211 catch (OwnerTerminated) 212 return; 213 handler((cast() listener).accept); 214 } 215 catch (Throwable e) 216 { 217 // https://issues.dlang.org/show_bug.cgi?id=7018 218 stderr.writeln(e); 219 } 220 } 221 } 222 223 private TestServer startServer() 224 { 225 import std.concurrency : spawn; 226 import std.socket : INADDR_LOOPBACK, InternetAddress, TcpSocket; 227 228 tlsInit = true; 229 auto sock = new TcpSocket; 230 sock.bind(new InternetAddress(INADDR_LOOPBACK, InternetAddress.PORT_ANY)); 231 sock.listen(1); 232 auto addr = sock.localAddress.toString(); 233 auto tid = spawn(&TestServer.loop, cast(shared) sock); 234 return TestServer(addr, tid, sock); 235 } 236 237 /** Test server */ 238 __gshared TestServer server; 239 /** Thread-local storage init */ 240 bool tlsInit; 241 242 private ref TestServer testServer() 243 { 244 import std.concurrency : initOnce; 245 return initOnce!server(startServer()); 246 } 247 248 static ~this() 249 { 250 // terminate server from a thread local dtor of the thread that started it, 251 // because thread_joinall is called before shared module dtors 252 if (tlsInit && server.sock) 253 { 254 server.sock.shutdown(SocketShutdown.RECEIVE); 255 server.sock.close(); 256 } 257 } 258 259 private struct Request(T) 260 { 261 string hdrs; 262 immutable(T)[] bdy; 263 } 264 265 private Request!T recvReq(T=char)(Socket s) 266 { 267 import std.algorithm.comparison : min; 268 import std.algorithm.searching : find, canFind; 269 import std.conv : to; 270 import std.regex : ctRegex, matchFirst; 271 272 ubyte[1024] tmp=void; 273 ubyte[] buf; 274 275 while (true) 276 { 277 auto nbytes = s.receive(tmp[]); 278 assert(nbytes >= 0); 279 280 immutable beg = buf.length > 3 ? buf.length - 3 : 0; 281 buf ~= tmp[0 .. nbytes]; 282 auto bdy = buf[beg .. $].find(cast(ubyte[])"\r\n\r\n"); 283 if (bdy.empty) 284 continue; 285 286 auto hdrs = cast(string) buf[0 .. $ - bdy.length]; 287 bdy.popFrontN(4); 288 // no support for chunked transfer-encoding 289 if (auto m = hdrs.matchFirst(ctRegex!(`Content-Length: ([0-9]+)`, "i"))) 290 { 291 import std.uni : asUpperCase; 292 if (hdrs.asUpperCase.canFind("EXPECT: 100-CONTINUE")) 293 s.send(httpContinue); 294 295 size_t remain = m.captures[1].to!size_t - bdy.length; 296 while (remain) 297 { 298 nbytes = s.receive(tmp[0 .. min(remain, $)]); 299 assert(nbytes >= 0); 300 buf ~= tmp[0 .. nbytes]; 301 remain -= nbytes; 302 } 303 } 304 else 305 { 306 assert(bdy.empty); 307 } 308 bdy = buf[hdrs.length + 4 .. $]; 309 return typeof(return)(hdrs, cast(immutable(T)[])bdy); 310 } 311 } 312 313 private string httpOK(string msg) 314 { 315 import std.conv : to; 316 317 return "HTTP/1.1 200 OK\r\n"~ 318 "Content-Type: text/plain\r\n"~ 319 "Content-Length: "~msg.length.to!string~"\r\n"~ 320 "\r\n"~ 321 msg; 322 } 323 324 private string httpOK() 325 { 326 return "HTTP/1.1 200 OK\r\n"~ 327 "Content-Length: 0\r\n"~ 328 "\r\n"; 329 } 330 331 private string httpNotFound() 332 { 333 return "HTTP/1.1 404 Not Found\r\n"~ 334 "Content-Length: 0\r\n"~ 335 "\r\n"; 336 } 337 338 private enum httpContinue = "HTTP/1.1 100 Continue\r\n\r\n"; 339 } 340 version (StdDdoc) import std.stdio; 341 342 // Default data timeout for Protocols 343 private enum _defaultDataTimeout = dur!"minutes"(2); 344 345 /** 346 Macros: 347 348 CALLBACK_PARAMS = $(TABLE , 349 $(DDOC_PARAM_ROW 350 $(DDOC_PARAM_ID $(DDOC_PARAM dlTotal)) 351 $(DDOC_PARAM_DESC total bytes to download) 352 ) 353 $(DDOC_PARAM_ROW 354 $(DDOC_PARAM_ID $(DDOC_PARAM dlNow)) 355 $(DDOC_PARAM_DESC currently downloaded bytes) 356 ) 357 $(DDOC_PARAM_ROW 358 $(DDOC_PARAM_ID $(DDOC_PARAM ulTotal)) 359 $(DDOC_PARAM_DESC total bytes to upload) 360 ) 361 $(DDOC_PARAM_ROW 362 $(DDOC_PARAM_ID $(DDOC_PARAM ulNow)) 363 $(DDOC_PARAM_DESC currently uploaded bytes) 364 ) 365 ) 366 */ 367 368 /** Connection type used when the URL should be used to auto detect the protocol. 369 * 370 * This struct is used as placeholder for the connection parameter when calling 371 * the high level API and the connection type (HTTP/FTP) should be guessed by 372 * inspecting the URL parameter. 373 * 374 * The rules for guessing the protocol are: 375 * 1, if URL starts with ftp://, ftps:// or ftp. then FTP connection is assumed. 376 * 2, HTTP connection otherwise. 377 * 378 * Example: 379 * --- 380 * import std.net.curl; 381 * // Two requests below will do the same. 382 * char[] content; 383 * 384 * // Explicit connection provided 385 * content = get!HTTP("dlang.org"); 386 * 387 * // Guess connection type by looking at the URL 388 * content = get!AutoProtocol("ftp://foo.com/file"); 389 * // and since AutoProtocol is default this is the same as 390 * content = get("ftp://foo.com/file"); 391 * // and will end up detecting FTP from the url and be the same as 392 * content = get!FTP("ftp://foo.com/file"); 393 * --- 394 */ 395 struct AutoProtocol { } 396 397 // Returns true if the url points to an FTP resource 398 private bool isFTPUrl(const(char)[] url) 399 { 400 import std.algorithm.searching : startsWith; 401 import std.uni : toLower; 402 403 return startsWith(url.toLower(), "ftp://", "ftps://", "ftp.") != 0; 404 } 405 406 // Is true if the Conn type is a valid Curl Connection type. 407 private template isCurlConn(Conn) 408 { 409 enum auto isCurlConn = is(Conn : HTTP) || 410 is(Conn : FTP) || is(Conn : AutoProtocol); 411 } 412 413 /** HTTP/FTP download to local file system. 414 * 415 * Params: 416 * url = resource to download 417 * saveToPath = path to store the downloaded content on local disk 418 * conn = connection to use e.g. FTP or HTTP. The default AutoProtocol will 419 * guess connection type and create a new instance for this call only. 420 * 421 * Example: 422 * ---- 423 * import std.net.curl; 424 * download("https://httpbin.org/get", "/tmp/downloaded-http-file"); 425 * ---- 426 */ 427 void download(Conn = AutoProtocol)(const(char)[] url, string saveToPath, Conn conn = Conn()) 428 if (isCurlConn!Conn) 429 { 430 static if (is(Conn : HTTP) || is(Conn : FTP)) 431 { 432 import std.stdio : File; 433 conn.url = url; 434 auto f = File(saveToPath, "wb"); 435 conn.onReceive = (ubyte[] data) { f.rawWrite(data); return data.length; }; 436 conn.perform(); 437 } 438 else 439 { 440 if (isFTPUrl(url)) 441 return download!FTP(url, saveToPath, FTP()); 442 else 443 return download!HTTP(url, saveToPath, HTTP()); 444 } 445 } 446 447 @system unittest 448 { 449 import std.algorithm.searching : canFind; 450 static import std.file; 451 452 foreach (host; [testServer.addr, "http://"~testServer.addr]) 453 { 454 testServer.handle((s) { 455 assert(s.recvReq.hdrs.canFind("GET /")); 456 s.send(httpOK("Hello world")); 457 }); 458 auto fn = std.file.deleteme; 459 scope (exit) 460 { 461 if (std.file.exists(fn)) 462 std.file.remove(fn); 463 } 464 download(host, fn); 465 assert(std.file.readText(fn) == "Hello world"); 466 } 467 } 468 469 /** Upload file from local files system using the HTTP or FTP protocol. 470 * 471 * Params: 472 * loadFromPath = path load data from local disk. 473 * url = resource to upload to 474 * conn = connection to use e.g. FTP or HTTP. The default AutoProtocol will 475 * guess connection type and create a new instance for this call only. 476 * 477 * Example: 478 * ---- 479 * import std.net.curl; 480 * upload("/tmp/downloaded-ftp-file", "ftp.digitalmars.com/sieve.ds"); 481 * upload("/tmp/downloaded-http-file", "https://httpbin.org/post"); 482 * ---- 483 */ 484 void upload(Conn = AutoProtocol)(string loadFromPath, const(char)[] url, Conn conn = Conn()) 485 if (isCurlConn!Conn) 486 { 487 static if (is(Conn : HTTP)) 488 { 489 conn.url = url; 490 conn.method = HTTP.Method.put; 491 } 492 else static if (is(Conn : FTP)) 493 { 494 conn.url = url; 495 conn.handle.set(CurlOption.upload, 1L); 496 } 497 else 498 { 499 if (isFTPUrl(url)) 500 return upload!FTP(loadFromPath, url, FTP()); 501 else 502 return upload!HTTP(loadFromPath, url, HTTP()); 503 } 504 505 static if (is(Conn : HTTP) || is(Conn : FTP)) 506 { 507 import std.stdio : File; 508 auto f = File(loadFromPath, "rb"); 509 conn.onSend = buf => f.rawRead(buf).length; 510 immutable sz = f.size; 511 if (sz != ulong.max) 512 conn.contentLength = sz; 513 conn.perform(); 514 } 515 } 516 517 @system unittest 518 { 519 import std.algorithm.searching : canFind; 520 static import std.file; 521 522 foreach (host; [testServer.addr, "http://"~testServer.addr]) 523 { 524 auto fn = std.file.deleteme; 525 scope (exit) 526 { 527 if (std.file.exists(fn)) 528 std.file.remove(fn); 529 } 530 std.file.write(fn, "upload data\n"); 531 testServer.handle((s) { 532 auto req = s.recvReq; 533 assert(req.hdrs.canFind("PUT /path")); 534 assert(req.bdy.canFind("upload data")); 535 s.send(httpOK()); 536 }); 537 upload(fn, host ~ "/path"); 538 } 539 } 540 541 /** HTTP/FTP get content. 542 * 543 * Params: 544 * url = resource to get 545 * conn = connection to use e.g. FTP or HTTP. The default AutoProtocol will 546 * guess connection type and create a new instance for this call only. 547 * 548 * The template parameter `T` specifies the type to return. Possible values 549 * are `char` and `ubyte` to return `char[]` or `ubyte[]`. If asking 550 * for `char`, content will be converted from the connection character set 551 * (specified in HTTP response headers or FTP connection properties, both ISO-8859-1 552 * by default) to UTF-8. 553 * 554 * Example: 555 * ---- 556 * import std.net.curl; 557 * auto content = get("https://httpbin.org/get"); 558 * ---- 559 * 560 * Returns: 561 * A T[] range containing the content of the resource pointed to by the URL. 562 * 563 * Throws: 564 * 565 * `CurlException` on error. 566 * 567 * See_Also: $(LREF HTTP.Method) 568 */ 569 T[] get(Conn = AutoProtocol, T = char)(const(char)[] url, Conn conn = Conn()) 570 if ( isCurlConn!Conn && (is(T == char) || is(T == ubyte)) ) 571 { 572 static if (is(Conn : HTTP)) 573 { 574 conn.method = HTTP.Method.get; 575 return _basicHTTP!(T)(url, "", conn); 576 577 } 578 else static if (is(Conn : FTP)) 579 { 580 return _basicFTP!(T)(url, "", conn); 581 } 582 else 583 { 584 if (isFTPUrl(url)) 585 return get!(FTP,T)(url, FTP()); 586 else 587 return get!(HTTP,T)(url, HTTP()); 588 } 589 } 590 591 @system unittest 592 { 593 import std.algorithm.searching : canFind; 594 595 foreach (host; [testServer.addr, "http://"~testServer.addr]) 596 { 597 testServer.handle((s) { 598 assert(s.recvReq.hdrs.canFind("GET /path")); 599 s.send(httpOK("GETRESPONSE")); 600 }); 601 auto res = get(host ~ "/path"); 602 assert(res == "GETRESPONSE"); 603 } 604 } 605 606 607 /** HTTP post content. 608 * 609 * Params: 610 * url = resource to post to 611 * postDict = data to send as the body of the request. An associative array 612 * of `string` is accepted and will be encoded using 613 * www-form-urlencoding 614 * postData = data to send as the body of the request. An array 615 * of an arbitrary type is accepted and will be cast to ubyte[] 616 * before sending it. 617 * conn = HTTP connection to use 618 * T = The template parameter `T` specifies the type to return. Possible values 619 * are `char` and `ubyte` to return `char[]` or `ubyte[]`. If asking 620 * for `char`, content will be converted from the connection character set 621 * (specified in HTTP response headers or FTP connection properties, both ISO-8859-1 622 * by default) to UTF-8. 623 * 624 * Examples: 625 * ---- 626 * import std.net.curl; 627 * 628 * auto content1 = post("https://httpbin.org/post", ["name1" : "value1", "name2" : "value2"]); 629 * auto content2 = post("https://httpbin.org/post", [1,2,3,4]); 630 * ---- 631 * 632 * Returns: 633 * A T[] range containing the content of the resource pointed to by the URL. 634 * 635 * See_Also: $(LREF HTTP.Method) 636 */ 637 T[] post(T = char, PostUnit)(const(char)[] url, const(PostUnit)[] postData, HTTP conn = HTTP()) 638 if (is(T == char) || is(T == ubyte)) 639 { 640 conn.method = HTTP.Method.post; 641 return _basicHTTP!(T)(url, postData, conn); 642 } 643 644 @system unittest 645 { 646 import std.algorithm.searching : canFind; 647 648 foreach (host; [testServer.addr, "http://"~testServer.addr]) 649 { 650 testServer.handle((s) { 651 auto req = s.recvReq; 652 assert(req.hdrs.canFind("POST /path")); 653 assert(req.bdy.canFind("POSTBODY")); 654 s.send(httpOK("POSTRESPONSE")); 655 }); 656 auto res = post(host ~ "/path", "POSTBODY"); 657 assert(res == "POSTRESPONSE"); 658 } 659 } 660 661 @system unittest 662 { 663 import std.algorithm.searching : canFind; 664 665 auto data = new ubyte[](256); 666 foreach (i, ref ub; data) 667 ub = cast(ubyte) i; 668 669 testServer.handle((s) { 670 auto req = s.recvReq!ubyte; 671 assert(req.bdy.canFind(cast(ubyte[])[0, 1, 2, 3, 4])); 672 assert(req.bdy.canFind(cast(ubyte[])[253, 254, 255])); 673 s.send(httpOK(cast(ubyte[])[17, 27, 35, 41])); 674 }); 675 auto res = post!ubyte(testServer.addr, data); 676 assert(res == cast(ubyte[])[17, 27, 35, 41]); 677 } 678 679 /// ditto 680 T[] post(T = char)(const(char)[] url, string[string] postDict, HTTP conn = HTTP()) 681 if (is(T == char) || is(T == ubyte)) 682 { 683 import std.uri : urlEncode; 684 685 return post!T(url, urlEncode(postDict), conn); 686 } 687 688 @system unittest 689 { 690 import std.algorithm.searching : canFind; 691 import std.meta : AliasSeq; 692 693 static immutable expected = ["name1=value1&name2=value2", "name2=value2&name1=value1"]; 694 695 foreach (host; [testServer.addr, "http://" ~ testServer.addr]) 696 { 697 foreach (T; AliasSeq!(char, ubyte)) 698 { 699 testServer.handle((s) { 700 auto req = s.recvReq!char; 701 s.send(httpOK(req.bdy)); 702 }); 703 auto res = post!T(host ~ "/path", ["name1" : "value1", "name2" : "value2"]); 704 assert(canFind(expected, res)); 705 } 706 } 707 } 708 709 /** HTTP/FTP put content. 710 * 711 * Params: 712 * url = resource to put 713 * putData = data to send as the body of the request. An array 714 * of an arbitrary type is accepted and will be cast to ubyte[] 715 * before sending it. 716 * conn = connection to use e.g. FTP or HTTP. The default AutoProtocol will 717 * guess connection type and create a new instance for this call only. 718 * 719 * The template parameter `T` specifies the type to return. Possible values 720 * are `char` and `ubyte` to return `char[]` or `ubyte[]`. If asking 721 * for `char`, content will be converted from the connection character set 722 * (specified in HTTP response headers or FTP connection properties, both ISO-8859-1 723 * by default) to UTF-8. 724 * 725 * Example: 726 * ---- 727 * import std.net.curl; 728 * auto content = put("https://httpbin.org/put", 729 * "Putting this data"); 730 * ---- 731 * 732 * Returns: 733 * A T[] range containing the content of the resource pointed to by the URL. 734 * 735 * See_Also: $(LREF HTTP.Method) 736 */ 737 T[] put(Conn = AutoProtocol, T = char, PutUnit)(const(char)[] url, const(PutUnit)[] putData, 738 Conn conn = Conn()) 739 if ( isCurlConn!Conn && (is(T == char) || is(T == ubyte)) ) 740 { 741 static if (is(Conn : HTTP)) 742 { 743 conn.method = HTTP.Method.put; 744 return _basicHTTP!(T)(url, putData, conn); 745 } 746 else static if (is(Conn : FTP)) 747 { 748 return _basicFTP!(T)(url, putData, conn); 749 } 750 else 751 { 752 if (isFTPUrl(url)) 753 return put!(FTP,T)(url, putData, FTP()); 754 else 755 return put!(HTTP,T)(url, putData, HTTP()); 756 } 757 } 758 759 @system unittest 760 { 761 import std.algorithm.searching : canFind; 762 763 foreach (host; [testServer.addr, "http://"~testServer.addr]) 764 { 765 testServer.handle((s) { 766 auto req = s.recvReq; 767 assert(req.hdrs.canFind("PUT /path")); 768 assert(req.bdy.canFind("PUTBODY")); 769 s.send(httpOK("PUTRESPONSE")); 770 }); 771 auto res = put(host ~ "/path", "PUTBODY"); 772 assert(res == "PUTRESPONSE"); 773 } 774 } 775 776 777 /** HTTP/FTP delete content. 778 * 779 * Params: 780 * url = resource to delete 781 * conn = connection to use e.g. FTP or HTTP. The default AutoProtocol will 782 * guess connection type and create a new instance for this call only. 783 * 784 * Example: 785 * ---- 786 * import std.net.curl; 787 * del("https://httpbin.org/delete"); 788 * ---- 789 * 790 * See_Also: $(LREF HTTP.Method) 791 */ 792 void del(Conn = AutoProtocol)(const(char)[] url, Conn conn = Conn()) 793 if (isCurlConn!Conn) 794 { 795 static if (is(Conn : HTTP)) 796 { 797 conn.method = HTTP.Method.del; 798 _basicHTTP!char(url, cast(void[]) null, conn); 799 } 800 else static if (is(Conn : FTP)) 801 { 802 import std.algorithm.searching : findSplitAfter; 803 import std.conv : text; 804 import std.exception : enforce; 805 806 auto trimmed = url.findSplitAfter("ftp://")[1]; 807 auto t = trimmed.findSplitAfter("/"); 808 enum minDomainNameLength = 3; 809 enforce!CurlException(t[0].length > minDomainNameLength, 810 text("Invalid FTP URL for delete ", url)); 811 conn.url = t[0]; 812 813 enforce!CurlException(!t[1].empty, 814 text("No filename specified to delete for URL ", url)); 815 conn.addCommand("DELE " ~ t[1]); 816 conn.perform(); 817 } 818 else 819 { 820 if (isFTPUrl(url)) 821 return del!FTP(url, FTP()); 822 else 823 return del!HTTP(url, HTTP()); 824 } 825 } 826 827 @system unittest 828 { 829 import std.algorithm.searching : canFind; 830 831 foreach (host; [testServer.addr, "http://"~testServer.addr]) 832 { 833 testServer.handle((s) { 834 auto req = s.recvReq; 835 assert(req.hdrs.canFind("DELETE /path")); 836 s.send(httpOK()); 837 }); 838 del(host ~ "/path"); 839 } 840 } 841 842 843 /** HTTP options request. 844 * 845 * Params: 846 * url = resource make a option call to 847 * conn = connection to use e.g. FTP or HTTP. The default AutoProtocol will 848 * guess connection type and create a new instance for this call only. 849 * 850 * The template parameter `T` specifies the type to return. Possible values 851 * are `char` and `ubyte` to return `char[]` or `ubyte[]`. 852 * 853 * Example: 854 * ---- 855 * import std.net.curl; 856 * auto http = HTTP(); 857 * options("https://httpbin.org/headers", http); 858 * writeln("Allow set to " ~ http.responseHeaders["Allow"]); 859 * ---- 860 * 861 * Returns: 862 * A T[] range containing the options of the resource pointed to by the URL. 863 * 864 * See_Also: $(LREF HTTP.Method) 865 */ 866 T[] options(T = char)(const(char)[] url, HTTP conn = HTTP()) 867 if (is(T == char) || is(T == ubyte)) 868 { 869 conn.method = HTTP.Method.options; 870 return _basicHTTP!(T)(url, null, conn); 871 } 872 873 @system unittest 874 { 875 import std.algorithm.searching : canFind; 876 877 testServer.handle((s) { 878 auto req = s.recvReq; 879 assert(req.hdrs.canFind("OPTIONS /path")); 880 s.send(httpOK("OPTIONSRESPONSE")); 881 }); 882 auto res = options(testServer.addr ~ "/path"); 883 assert(res == "OPTIONSRESPONSE"); 884 } 885 886 887 /** HTTP trace request. 888 * 889 * Params: 890 * url = resource make a trace call to 891 * conn = connection to use e.g. FTP or HTTP. The default AutoProtocol will 892 * guess connection type and create a new instance for this call only. 893 * 894 * The template parameter `T` specifies the type to return. Possible values 895 * are `char` and `ubyte` to return `char[]` or `ubyte[]`. 896 * 897 * Example: 898 * ---- 899 * import std.net.curl; 900 * trace("https://httpbin.org/headers"); 901 * ---- 902 * 903 * Returns: 904 * A T[] range containing the trace info of the resource pointed to by the URL. 905 * 906 * See_Also: $(LREF HTTP.Method) 907 */ 908 T[] trace(T = char)(const(char)[] url, HTTP conn = HTTP()) 909 if (is(T == char) || is(T == ubyte)) 910 { 911 conn.method = HTTP.Method.trace; 912 return _basicHTTP!(T)(url, cast(void[]) null, conn); 913 } 914 915 @system unittest 916 { 917 import std.algorithm.searching : canFind; 918 919 testServer.handle((s) { 920 auto req = s.recvReq; 921 assert(req.hdrs.canFind("TRACE /path")); 922 s.send(httpOK("TRACERESPONSE")); 923 }); 924 auto res = trace(testServer.addr ~ "/path"); 925 assert(res == "TRACERESPONSE"); 926 } 927 928 929 /** HTTP connect request. 930 * 931 * Params: 932 * url = resource make a connect to 933 * conn = HTTP connection to use 934 * 935 * The template parameter `T` specifies the type to return. Possible values 936 * are `char` and `ubyte` to return `char[]` or `ubyte[]`. 937 * 938 * Example: 939 * ---- 940 * import std.net.curl; 941 * connect("https://httpbin.org/headers"); 942 * ---- 943 * 944 * Returns: 945 * A T[] range containing the connect info of the resource pointed to by the URL. 946 * 947 * See_Also: $(LREF HTTP.Method) 948 */ 949 T[] connect(T = char)(const(char)[] url, HTTP conn = HTTP()) 950 if (is(T == char) || is(T == ubyte)) 951 { 952 conn.method = HTTP.Method.connect; 953 return _basicHTTP!(T)(url, cast(void[]) null, conn); 954 } 955 956 @system unittest 957 { 958 import std.algorithm.searching : canFind; 959 960 testServer.handle((s) { 961 auto req = s.recvReq; 962 assert(req.hdrs.canFind("CONNECT /path")); 963 s.send(httpOK("CONNECTRESPONSE")); 964 }); 965 auto res = connect(testServer.addr ~ "/path"); 966 assert(res == "CONNECTRESPONSE"); 967 } 968 969 970 /** HTTP patch content. 971 * 972 * Params: 973 * url = resource to patch 974 * patchData = data to send as the body of the request. An array 975 * of an arbitrary type is accepted and will be cast to ubyte[] 976 * before sending it. 977 * conn = HTTP connection to use 978 * 979 * The template parameter `T` specifies the type to return. Possible values 980 * are `char` and `ubyte` to return `char[]` or `ubyte[]`. 981 * 982 * Example: 983 * ---- 984 * auto http = HTTP(); 985 * http.addRequestHeader("Content-Type", "application/json"); 986 * auto content = patch("https://httpbin.org/patch", `{"title": "Patched Title"}`, http); 987 * ---- 988 * 989 * Returns: 990 * A T[] range containing the content of the resource pointed to by the URL. 991 * 992 * See_Also: $(LREF HTTP.Method) 993 */ 994 T[] patch(T = char, PatchUnit)(const(char)[] url, const(PatchUnit)[] patchData, 995 HTTP conn = HTTP()) 996 if (is(T == char) || is(T == ubyte)) 997 { 998 conn.method = HTTP.Method.patch; 999 return _basicHTTP!(T)(url, patchData, conn); 1000 } 1001 1002 @system unittest 1003 { 1004 import std.algorithm.searching : canFind; 1005 1006 testServer.handle((s) { 1007 auto req = s.recvReq; 1008 assert(req.hdrs.canFind("PATCH /path")); 1009 assert(req.bdy.canFind("PATCHBODY")); 1010 s.send(httpOK("PATCHRESPONSE")); 1011 }); 1012 auto res = patch(testServer.addr ~ "/path", "PATCHBODY"); 1013 assert(res == "PATCHRESPONSE"); 1014 } 1015 1016 1017 /* 1018 * Helper function for the high level interface. 1019 * 1020 * It performs an HTTP request using the client which must have 1021 * been setup correctly before calling this function. 1022 */ 1023 private auto _basicHTTP(T)(const(char)[] url, const(void)[] sendData, HTTP client) 1024 { 1025 import std.algorithm.comparison : min; 1026 import std.format : format; 1027 import std.exception : enforce; 1028 import etc.c.curl : CurlSeek, CurlSeekPos; 1029 1030 immutable doSend = sendData !is null && 1031 (client.method == HTTP.Method.post || 1032 client.method == HTTP.Method.put || 1033 client.method == HTTP.Method.patch); 1034 1035 scope (exit) 1036 { 1037 client.onReceiveHeader = null; 1038 client.onReceiveStatusLine = null; 1039 client.onReceive = null; 1040 1041 if (doSend) 1042 { 1043 client.onSend = null; 1044 client.handle.onSeek = null; 1045 client.contentLength = 0; 1046 } 1047 } 1048 client.url = url; 1049 HTTP.StatusLine statusLine; 1050 import std.array : appender; 1051 auto content = appender!(ubyte[])(); 1052 client.onReceive = (ubyte[] data) 1053 { 1054 content ~= data; 1055 return data.length; 1056 }; 1057 1058 if (doSend) 1059 { 1060 client.contentLength = sendData.length; 1061 auto remainingData = sendData; 1062 client.onSend = delegate size_t(void[] buf) 1063 { 1064 size_t minLen = min(buf.length, remainingData.length); 1065 if (minLen == 0) return 0; 1066 buf[0 .. minLen] = remainingData[0 .. minLen]; 1067 remainingData = remainingData[minLen..$]; 1068 return minLen; 1069 }; 1070 client.handle.onSeek = delegate(long offset, CurlSeekPos mode) 1071 { 1072 switch (mode) 1073 { 1074 case CurlSeekPos.set: 1075 remainingData = sendData[cast(size_t) offset..$]; 1076 return CurlSeek.ok; 1077 default: 1078 // As of curl 7.18.0, libcurl will not pass 1079 // anything other than CurlSeekPos.set. 1080 return CurlSeek.cantseek; 1081 } 1082 }; 1083 } 1084 1085 client.onReceiveHeader = (in char[] key, 1086 in char[] value) 1087 { 1088 if (key == "content-length") 1089 { 1090 import std.conv : to; 1091 content.reserve(value.to!size_t); 1092 } 1093 }; 1094 client.onReceiveStatusLine = (HTTP.StatusLine l) { statusLine = l; }; 1095 client.perform(); 1096 enforce(statusLine.code / 100 == 2, new HTTPStatusException(statusLine.code, 1097 format("HTTP request returned status code %d (%s)", statusLine.code, statusLine.reason))); 1098 1099 return _decodeContent!T(content.data, client.p.charset); 1100 } 1101 1102 @system unittest 1103 { 1104 import std.algorithm.searching : canFind; 1105 import std.exception : collectException; 1106 1107 testServer.handle((s) { 1108 auto req = s.recvReq; 1109 assert(req.hdrs.canFind("GET /path")); 1110 s.send(httpNotFound()); 1111 }); 1112 auto e = collectException!HTTPStatusException(get(testServer.addr ~ "/path")); 1113 assert(e.msg == "HTTP request returned status code 404 (Not Found)"); 1114 assert(e.status == 404); 1115 } 1116 1117 // Content length must be reset after post 1118 // https://issues.dlang.org/show_bug.cgi?id=14760 1119 @system unittest 1120 { 1121 import std.algorithm.searching : canFind; 1122 1123 testServer.handle((s) { 1124 auto req = s.recvReq; 1125 assert(req.hdrs.canFind("POST /")); 1126 assert(req.bdy.canFind("POSTBODY")); 1127 s.send(httpOK("POSTRESPONSE")); 1128 1129 req = s.recvReq; 1130 assert(req.hdrs.canFind("TRACE /")); 1131 assert(req.bdy.empty); 1132 s.blocking = false; 1133 ubyte[6] buf = void; 1134 assert(s.receive(buf[]) < 0); 1135 s.send(httpOK("TRACERESPONSE")); 1136 }); 1137 auto http = HTTP(); 1138 auto res = post(testServer.addr, "POSTBODY", http); 1139 assert(res == "POSTRESPONSE"); 1140 res = trace(testServer.addr, http); 1141 assert(res == "TRACERESPONSE"); 1142 } 1143 1144 @system unittest // charset detection and transcoding to T 1145 { 1146 testServer.handle((s) { 1147 s.send("HTTP/1.1 200 OK\r\n"~ 1148 "Content-Length: 4\r\n"~ 1149 "Content-Type: text/plain; charset=utf-8\r\n" ~ 1150 "\r\n" ~ 1151 "äbc"); 1152 }); 1153 auto client = HTTP(); 1154 auto result = _basicHTTP!char(testServer.addr, "", client); 1155 assert(result == "äbc"); 1156 1157 testServer.handle((s) { 1158 s.send("HTTP/1.1 200 OK\r\n"~ 1159 "Content-Length: 3\r\n"~ 1160 "Content-Type: text/plain; charset=iso-8859-1\r\n" ~ 1161 "\r\n" ~ 1162 0xE4 ~ "bc"); 1163 }); 1164 client = HTTP(); 1165 result = _basicHTTP!char(testServer.addr, "", client); 1166 assert(result == "äbc"); 1167 } 1168 1169 /* 1170 * Helper function for the high level interface. 1171 * 1172 * It performs an FTP request using the client which must have 1173 * been setup correctly before calling this function. 1174 */ 1175 private auto _basicFTP(T)(const(char)[] url, const(void)[] sendData, FTP client) 1176 { 1177 import std.algorithm.comparison : min; 1178 1179 scope (exit) 1180 { 1181 client.onReceive = null; 1182 if (!sendData.empty) 1183 client.onSend = null; 1184 } 1185 1186 ubyte[] content; 1187 1188 if (client.encoding.empty) 1189 client.encoding = "ISO-8859-1"; 1190 1191 client.url = url; 1192 client.onReceive = (ubyte[] data) 1193 { 1194 content ~= data; 1195 return data.length; 1196 }; 1197 1198 if (!sendData.empty) 1199 { 1200 client.handle.set(CurlOption.upload, 1L); 1201 client.onSend = delegate size_t(void[] buf) 1202 { 1203 size_t minLen = min(buf.length, sendData.length); 1204 if (minLen == 0) return 0; 1205 buf[0 .. minLen] = sendData[0 .. minLen]; 1206 sendData = sendData[minLen..$]; 1207 return minLen; 1208 }; 1209 } 1210 1211 client.perform(); 1212 1213 return _decodeContent!T(content, client.encoding); 1214 } 1215 1216 /* Used by _basicHTTP() and _basicFTP() to decode ubyte[] to 1217 * correct string format 1218 */ 1219 private auto _decodeContent(T)(ubyte[] content, string encoding) 1220 { 1221 static if (is(T == ubyte)) 1222 { 1223 return content; 1224 } 1225 else 1226 { 1227 import std.exception : enforce; 1228 import std.format : format; 1229 1230 // Optimally just return the utf8 encoded content 1231 if (encoding == "UTF-8") 1232 return cast(char[])(content); 1233 1234 // The content has to be re-encoded to utf8 1235 auto scheme = EncodingScheme.create(encoding); 1236 enforce!CurlException(scheme !is null, 1237 format("Unknown encoding '%s'", encoding)); 1238 1239 auto strInfo = decodeString(content, scheme); 1240 enforce!CurlException(strInfo[0] != size_t.max, 1241 format("Invalid encoding sequence for encoding '%s'", 1242 encoding)); 1243 1244 return strInfo[1]; 1245 } 1246 } 1247 1248 alias KeepTerminator = Flag!"keepTerminator"; 1249 /+ 1250 struct ByLineBuffer(Char) 1251 { 1252 bool linePresent; 1253 bool EOF; 1254 Char[] buffer; 1255 ubyte[] decodeRemainder; 1256 1257 bool append(const(ubyte)[] data) 1258 { 1259 byLineBuffer ~= data; 1260 } 1261 1262 @property bool linePresent() 1263 { 1264 return byLinePresent; 1265 } 1266 1267 Char[] get() 1268 { 1269 if (!linePresent) 1270 { 1271 // Decode ubyte[] into Char[] until a Terminator is found. 1272 // If not Terminator is found and EOF is false then raise an 1273 // exception. 1274 } 1275 return byLineBuffer; 1276 } 1277 1278 } 1279 ++/ 1280 /** HTTP/FTP fetch content as a range of lines. 1281 * 1282 * A range of lines is returned when the request is complete. If the method or 1283 * other request properties is to be customized then set the `conn` parameter 1284 * with a HTTP/FTP instance that has these properties set. 1285 * 1286 * Example: 1287 * ---- 1288 * import std.net.curl, std.stdio; 1289 * foreach (line; byLine("dlang.org")) 1290 * writeln(line); 1291 * ---- 1292 * 1293 * Params: 1294 * url = The url to receive content from 1295 * keepTerminator = `Yes.keepTerminator` signals that the line terminator should be 1296 * returned as part of the lines in the range. 1297 * terminator = The character that terminates a line 1298 * conn = The connection to use e.g. HTTP or FTP. 1299 * 1300 * Returns: 1301 * A range of Char[] with the content of the resource pointer to by the URL 1302 */ 1303 auto byLine(Conn = AutoProtocol, Terminator = char, Char = char) 1304 (const(char)[] url, KeepTerminator keepTerminator = No.keepTerminator, 1305 Terminator terminator = '\n', Conn conn = Conn()) 1306 if (isCurlConn!Conn && isSomeChar!Char && isSomeChar!Terminator) 1307 { 1308 static struct SyncLineInputRange 1309 { 1310 1311 private Char[] lines; 1312 private Char[] current; 1313 private bool currentValid; 1314 private bool keepTerminator; 1315 private Terminator terminator; 1316 1317 this(Char[] lines, bool kt, Terminator terminator) 1318 { 1319 this.lines = lines; 1320 this.keepTerminator = kt; 1321 this.terminator = terminator; 1322 currentValid = true; 1323 popFront(); 1324 } 1325 1326 @property @safe bool empty() 1327 { 1328 return !currentValid; 1329 } 1330 1331 @property @safe Char[] front() 1332 { 1333 import std.exception : enforce; 1334 enforce!CurlException(currentValid, "Cannot call front() on empty range"); 1335 return current; 1336 } 1337 1338 void popFront() 1339 { 1340 import std.algorithm.searching : findSplitAfter, findSplit; 1341 import std.exception : enforce; 1342 1343 enforce!CurlException(currentValid, "Cannot call popFront() on empty range"); 1344 if (lines.empty) 1345 { 1346 currentValid = false; 1347 return; 1348 } 1349 1350 if (keepTerminator) 1351 { 1352 auto r = findSplitAfter(lines, [ terminator ]); 1353 if (r[0].empty) 1354 { 1355 current = r[1]; 1356 lines = r[0]; 1357 } 1358 else 1359 { 1360 current = r[0]; 1361 lines = r[1]; 1362 } 1363 } 1364 else 1365 { 1366 auto r = findSplit(lines, [ terminator ]); 1367 current = r[0]; 1368 lines = r[2]; 1369 } 1370 } 1371 } 1372 1373 auto result = _getForRange!Char(url, conn); 1374 return SyncLineInputRange(result, keepTerminator == Yes.keepTerminator, terminator); 1375 } 1376 1377 @system unittest 1378 { 1379 import std.algorithm.comparison : equal; 1380 1381 foreach (host; [testServer.addr, "http://"~testServer.addr]) 1382 { 1383 testServer.handle((s) { 1384 auto req = s.recvReq; 1385 s.send(httpOK("Line1\nLine2\nLine3")); 1386 }); 1387 assert(byLine(host).equal(["Line1", "Line2", "Line3"])); 1388 } 1389 } 1390 1391 /** HTTP/FTP fetch content as a range of chunks. 1392 * 1393 * A range of chunks is returned when the request is complete. If the method or 1394 * other request properties is to be customized then set the `conn` parameter 1395 * with a HTTP/FTP instance that has these properties set. 1396 * 1397 * Example: 1398 * ---- 1399 * import std.net.curl, std.stdio; 1400 * foreach (chunk; byChunk("dlang.org", 100)) 1401 * writeln(chunk); // chunk is ubyte[100] 1402 * ---- 1403 * 1404 * Params: 1405 * url = The url to receive content from 1406 * chunkSize = The size of each chunk 1407 * conn = The connection to use e.g. HTTP or FTP. 1408 * 1409 * Returns: 1410 * A range of ubyte[chunkSize] with the content of the resource pointer to by the URL 1411 */ 1412 auto byChunk(Conn = AutoProtocol) 1413 (const(char)[] url, size_t chunkSize = 1024, Conn conn = Conn()) 1414 if (isCurlConn!(Conn)) 1415 { 1416 static struct SyncChunkInputRange 1417 { 1418 private size_t chunkSize; 1419 private ubyte[] _bytes; 1420 private size_t offset; 1421 1422 this(ubyte[] bytes, size_t chunkSize) 1423 { 1424 this._bytes = bytes; 1425 this.chunkSize = chunkSize; 1426 } 1427 1428 @property @safe auto empty() 1429 { 1430 return offset == _bytes.length; 1431 } 1432 1433 @property ubyte[] front() 1434 { 1435 size_t nextOffset = offset + chunkSize; 1436 if (nextOffset > _bytes.length) nextOffset = _bytes.length; 1437 return _bytes[offset .. nextOffset]; 1438 } 1439 1440 @safe void popFront() 1441 { 1442 offset += chunkSize; 1443 if (offset > _bytes.length) offset = _bytes.length; 1444 } 1445 } 1446 1447 auto result = _getForRange!ubyte(url, conn); 1448 return SyncChunkInputRange(result, chunkSize); 1449 } 1450 1451 @system unittest 1452 { 1453 import std.algorithm.comparison : equal; 1454 1455 foreach (host; [testServer.addr, "http://"~testServer.addr]) 1456 { 1457 testServer.handle((s) { 1458 auto req = s.recvReq; 1459 s.send(httpOK(cast(ubyte[])[0, 1, 2, 3, 4, 5])); 1460 }); 1461 assert(byChunk(host, 2).equal([[0, 1], [2, 3], [4, 5]])); 1462 } 1463 } 1464 1465 private T[] _getForRange(T,Conn)(const(char)[] url, Conn conn) 1466 { 1467 static if (is(Conn : HTTP)) 1468 { 1469 conn.method = conn.method == HTTP.Method.undefined ? HTTP.Method.get : conn.method; 1470 return _basicHTTP!(T)(url, null, conn); 1471 } 1472 else static if (is(Conn : FTP)) 1473 { 1474 return _basicFTP!(T)(url, null, conn); 1475 } 1476 else 1477 { 1478 if (isFTPUrl(url)) 1479 return get!(FTP,T)(url, FTP()); 1480 else 1481 return get!(HTTP,T)(url, HTTP()); 1482 } 1483 } 1484 1485 /* 1486 Main thread part of the message passing protocol used for all async 1487 curl protocols. 1488 */ 1489 private mixin template WorkerThreadProtocol(Unit, alias units) 1490 { 1491 import core.time : Duration; 1492 1493 @property bool empty() 1494 { 1495 tryEnsureUnits(); 1496 return state == State.done; 1497 } 1498 1499 @property Unit[] front() 1500 { 1501 import std.format : format; 1502 tryEnsureUnits(); 1503 assert(state == State.gotUnits, 1504 format("Expected %s but got $s", 1505 State.gotUnits, state)); 1506 return units; 1507 } 1508 1509 void popFront() 1510 { 1511 import std.concurrency : send; 1512 import std.format : format; 1513 1514 tryEnsureUnits(); 1515 assert(state == State.gotUnits, 1516 format("Expected %s but got $s", 1517 State.gotUnits, state)); 1518 state = State.needUnits; 1519 // Send to worker thread for buffer reuse 1520 workerTid.send(cast(immutable(Unit)[]) units); 1521 units = null; 1522 } 1523 1524 /** Wait for duration or until data is available and return true if data is 1525 available 1526 */ 1527 bool wait(Duration d) 1528 { 1529 import core.time : dur; 1530 import std.datetime.stopwatch : StopWatch; 1531 import std.concurrency : receiveTimeout; 1532 1533 if (state == State.gotUnits) 1534 return true; 1535 1536 enum noDur = dur!"hnsecs"(0); 1537 StopWatch sw; 1538 sw.start(); 1539 while (state != State.gotUnits && d > noDur) 1540 { 1541 final switch (state) 1542 { 1543 case State.needUnits: 1544 receiveTimeout(d, 1545 (Tid origin, CurlMessage!(immutable(Unit)[]) _data) 1546 { 1547 if (origin != workerTid) 1548 return false; 1549 units = cast(Unit[]) _data.data; 1550 state = State.gotUnits; 1551 return true; 1552 }, 1553 (Tid origin, CurlMessage!bool f) 1554 { 1555 if (origin != workerTid) 1556 return false; 1557 state = state.done; 1558 return true; 1559 } 1560 ); 1561 break; 1562 case State.gotUnits: return true; 1563 case State.done: 1564 return false; 1565 } 1566 d -= sw.peek(); 1567 sw.reset(); 1568 } 1569 return state == State.gotUnits; 1570 } 1571 1572 enum State 1573 { 1574 needUnits, 1575 gotUnits, 1576 done 1577 } 1578 State state; 1579 1580 void tryEnsureUnits() 1581 { 1582 import std.concurrency : receive; 1583 while (true) 1584 { 1585 final switch (state) 1586 { 1587 case State.needUnits: 1588 receive( 1589 (Tid origin, CurlMessage!(immutable(Unit)[]) _data) 1590 { 1591 if (origin != workerTid) 1592 return false; 1593 units = cast(Unit[]) _data.data; 1594 state = State.gotUnits; 1595 return true; 1596 }, 1597 (Tid origin, CurlMessage!bool f) 1598 { 1599 if (origin != workerTid) 1600 return false; 1601 state = state.done; 1602 return true; 1603 } 1604 ); 1605 break; 1606 case State.gotUnits: return; 1607 case State.done: 1608 return; 1609 } 1610 } 1611 } 1612 } 1613 1614 /** HTTP/FTP fetch content as a range of lines asynchronously. 1615 * 1616 * A range of lines is returned immediately and the request that fetches the 1617 * lines is performed in another thread. If the method or other request 1618 * properties is to be customized then set the `conn` parameter with a 1619 * HTTP/FTP instance that has these properties set. 1620 * 1621 * If `postData` is non-_null the method will be set to `post` for HTTP 1622 * requests. 1623 * 1624 * The background thread will buffer up to transmitBuffers number of lines 1625 * before it stops receiving data from network. When the main thread reads the 1626 * lines from the range it frees up buffers and allows for the background thread 1627 * to receive more data from the network. 1628 * 1629 * If no data is available and the main thread accesses the range it will block 1630 * until data becomes available. An exception to this is the `wait(Duration)` method on 1631 * the $(LREF LineInputRange). This method will wait at maximum for the 1632 * specified duration and return true if data is available. 1633 * 1634 * Example: 1635 * ---- 1636 * import std.net.curl, std.stdio; 1637 * // Get some pages in the background 1638 * auto range1 = byLineAsync("www.google.com"); 1639 * auto range2 = byLineAsync("www.wikipedia.org"); 1640 * foreach (line; byLineAsync("dlang.org")) 1641 * writeln(line); 1642 * 1643 * // Lines already fetched in the background and ready 1644 * foreach (line; range1) writeln(line); 1645 * foreach (line; range2) writeln(line); 1646 * ---- 1647 * 1648 * ---- 1649 * import std.net.curl, std.stdio; 1650 * // Get a line in a background thread and wait in 1651 * // main thread for 2 seconds for it to arrive. 1652 * auto range3 = byLineAsync("dlang.com"); 1653 * if (range3.wait(dur!"seconds"(2))) 1654 * writeln(range3.front); 1655 * else 1656 * writeln("No line received after 2 seconds!"); 1657 * ---- 1658 * 1659 * Params: 1660 * url = The url to receive content from 1661 * postData = Data to HTTP Post 1662 * keepTerminator = `Yes.keepTerminator` signals that the line terminator should be 1663 * returned as part of the lines in the range. 1664 * terminator = The character that terminates a line 1665 * transmitBuffers = The number of lines buffered asynchronously 1666 * conn = The connection to use e.g. HTTP or FTP. 1667 * 1668 * Returns: 1669 * A range of Char[] with the content of the resource pointer to by the 1670 * URL. 1671 */ 1672 auto byLineAsync(Conn = AutoProtocol, Terminator = char, Char = char, PostUnit) 1673 (const(char)[] url, const(PostUnit)[] postData, 1674 KeepTerminator keepTerminator = No.keepTerminator, 1675 Terminator terminator = '\n', 1676 size_t transmitBuffers = 10, Conn conn = Conn()) 1677 if (isCurlConn!Conn && isSomeChar!Char && isSomeChar!Terminator) 1678 { 1679 static if (is(Conn : AutoProtocol)) 1680 { 1681 if (isFTPUrl(url)) 1682 return byLineAsync(url, postData, keepTerminator, 1683 terminator, transmitBuffers, FTP()); 1684 else 1685 return byLineAsync(url, postData, keepTerminator, 1686 terminator, transmitBuffers, HTTP()); 1687 } 1688 else 1689 { 1690 import std.concurrency : OnCrowding, send, setMaxMailboxSize, spawn, thisTid, Tid; 1691 // 50 is just an arbitrary number for now 1692 setMaxMailboxSize(thisTid, 50, OnCrowding.block); 1693 auto tid = spawn(&_async!().spawn!(Conn, Char, Terminator)); 1694 tid.send(thisTid); 1695 tid.send(terminator); 1696 tid.send(keepTerminator == Yes.keepTerminator); 1697 1698 _async!().duplicateConnection(url, conn, postData, tid); 1699 1700 return _async!().LineInputRange!Char(tid, transmitBuffers, 1701 Conn.defaultAsyncStringBufferSize); 1702 } 1703 } 1704 1705 /// ditto 1706 auto byLineAsync(Conn = AutoProtocol, Terminator = char, Char = char) 1707 (const(char)[] url, KeepTerminator keepTerminator = No.keepTerminator, 1708 Terminator terminator = '\n', 1709 size_t transmitBuffers = 10, Conn conn = Conn()) 1710 { 1711 static if (is(Conn : AutoProtocol)) 1712 { 1713 if (isFTPUrl(url)) 1714 return byLineAsync(url, cast(void[]) null, keepTerminator, 1715 terminator, transmitBuffers, FTP()); 1716 else 1717 return byLineAsync(url, cast(void[]) null, keepTerminator, 1718 terminator, transmitBuffers, HTTP()); 1719 } 1720 else 1721 { 1722 return byLineAsync(url, cast(void[]) null, keepTerminator, 1723 terminator, transmitBuffers, conn); 1724 } 1725 } 1726 1727 @system unittest 1728 { 1729 import std.algorithm.comparison : equal; 1730 1731 foreach (host; [testServer.addr, "http://"~testServer.addr]) 1732 { 1733 testServer.handle((s) { 1734 auto req = s.recvReq; 1735 s.send(httpOK("Line1\nLine2\nLine3")); 1736 }); 1737 assert(byLineAsync(host).equal(["Line1", "Line2", "Line3"])); 1738 } 1739 } 1740 1741 /** HTTP/FTP fetch content as a range of chunks asynchronously. 1742 * 1743 * A range of chunks is returned immediately and the request that fetches the 1744 * chunks is performed in another thread. If the method or other request 1745 * properties is to be customized then set the `conn` parameter with a 1746 * HTTP/FTP instance that has these properties set. 1747 * 1748 * If `postData` is non-_null the method will be set to `post` for HTTP 1749 * requests. 1750 * 1751 * The background thread will buffer up to transmitBuffers number of chunks 1752 * before is stops receiving data from network. When the main thread reads the 1753 * chunks from the range it frees up buffers and allows for the background 1754 * thread to receive more data from the network. 1755 * 1756 * If no data is available and the main thread access the range it will block 1757 * until data becomes available. An exception to this is the `wait(Duration)` 1758 * method on the $(LREF ChunkInputRange). This method will wait at maximum for the specified 1759 * duration and return true if data is available. 1760 * 1761 * Example: 1762 * ---- 1763 * import std.net.curl, std.stdio; 1764 * // Get some pages in the background 1765 * auto range1 = byChunkAsync("www.google.com", 100); 1766 * auto range2 = byChunkAsync("www.wikipedia.org"); 1767 * foreach (chunk; byChunkAsync("dlang.org")) 1768 * writeln(chunk); // chunk is ubyte[100] 1769 * 1770 * // Chunks already fetched in the background and ready 1771 * foreach (chunk; range1) writeln(chunk); 1772 * foreach (chunk; range2) writeln(chunk); 1773 * ---- 1774 * 1775 * ---- 1776 * import std.net.curl, std.stdio; 1777 * // Get a line in a background thread and wait in 1778 * // main thread for 2 seconds for it to arrive. 1779 * auto range3 = byChunkAsync("dlang.com", 10); 1780 * if (range3.wait(dur!"seconds"(2))) 1781 * writeln(range3.front); 1782 * else 1783 * writeln("No chunk received after 2 seconds!"); 1784 * ---- 1785 * 1786 * Params: 1787 * url = The url to receive content from 1788 * postData = Data to HTTP Post 1789 * chunkSize = The size of the chunks 1790 * transmitBuffers = The number of chunks buffered asynchronously 1791 * conn = The connection to use e.g. HTTP or FTP. 1792 * 1793 * Returns: 1794 * A range of ubyte[chunkSize] with the content of the resource pointer to by 1795 * the URL. 1796 */ 1797 auto byChunkAsync(Conn = AutoProtocol, PostUnit) 1798 (const(char)[] url, const(PostUnit)[] postData, 1799 size_t chunkSize = 1024, size_t transmitBuffers = 10, 1800 Conn conn = Conn()) 1801 if (isCurlConn!(Conn)) 1802 { 1803 static if (is(Conn : AutoProtocol)) 1804 { 1805 if (isFTPUrl(url)) 1806 return byChunkAsync(url, postData, chunkSize, 1807 transmitBuffers, FTP()); 1808 else 1809 return byChunkAsync(url, postData, chunkSize, 1810 transmitBuffers, HTTP()); 1811 } 1812 else 1813 { 1814 import std.concurrency : OnCrowding, send, setMaxMailboxSize, spawn, thisTid, Tid; 1815 // 50 is just an arbitrary number for now 1816 setMaxMailboxSize(thisTid, 50, OnCrowding.block); 1817 auto tid = spawn(&_async!().spawn!(Conn, ubyte)); 1818 tid.send(thisTid); 1819 1820 _async!().duplicateConnection(url, conn, postData, tid); 1821 1822 return _async!().ChunkInputRange(tid, transmitBuffers, chunkSize); 1823 } 1824 } 1825 1826 /// ditto 1827 auto byChunkAsync(Conn = AutoProtocol) 1828 (const(char)[] url, 1829 size_t chunkSize = 1024, size_t transmitBuffers = 10, 1830 Conn conn = Conn()) 1831 if (isCurlConn!(Conn)) 1832 { 1833 static if (is(Conn : AutoProtocol)) 1834 { 1835 if (isFTPUrl(url)) 1836 return byChunkAsync(url, cast(void[]) null, chunkSize, 1837 transmitBuffers, FTP()); 1838 else 1839 return byChunkAsync(url, cast(void[]) null, chunkSize, 1840 transmitBuffers, HTTP()); 1841 } 1842 else 1843 { 1844 return byChunkAsync(url, cast(void[]) null, chunkSize, 1845 transmitBuffers, conn); 1846 } 1847 } 1848 1849 @system unittest 1850 { 1851 import std.algorithm.comparison : equal; 1852 1853 foreach (host; [testServer.addr, "http://"~testServer.addr]) 1854 { 1855 testServer.handle((s) { 1856 auto req = s.recvReq; 1857 s.send(httpOK(cast(ubyte[])[0, 1, 2, 3, 4, 5])); 1858 }); 1859 assert(byChunkAsync(host, 2).equal([[0, 1], [2, 3], [4, 5]])); 1860 } 1861 } 1862 1863 1864 /* 1865 Mixin template for all supported curl protocols. This is the commom 1866 functionallity such as timeouts and network interface settings. This should 1867 really be in the HTTP/FTP/SMTP structs but the documentation tool does not 1868 support a mixin to put its doc strings where a mixin is done. Therefore docs 1869 in this template is copied into each of HTTP/FTP/SMTP below. 1870 */ 1871 private mixin template Protocol() 1872 { 1873 import etc.c.curl : CurlReadFunc, RawCurlProxy = CurlProxy; 1874 import core.time : Duration; 1875 import std.socket : InternetAddress; 1876 1877 /// Value to return from `onSend`/`onReceive` delegates in order to 1878 /// pause a request 1879 alias requestPause = CurlReadFunc.pause; 1880 1881 /// Value to return from onSend delegate in order to abort a request 1882 alias requestAbort = CurlReadFunc.abort; 1883 1884 static uint defaultAsyncStringBufferSize = 100; 1885 1886 /** 1887 The curl handle used by this connection. 1888 */ 1889 @property ref Curl handle() return 1890 { 1891 return p.curl; 1892 } 1893 1894 /** 1895 True if the instance is stopped. A stopped instance is not usable. 1896 */ 1897 @property bool isStopped() 1898 { 1899 return p.curl.stopped; 1900 } 1901 1902 /// Stop and invalidate this instance. 1903 void shutdown() 1904 { 1905 p.curl.shutdown(); 1906 } 1907 1908 /** Set verbose. 1909 This will print request information to stderr. 1910 */ 1911 @property void verbose(bool on) 1912 { 1913 p.curl.set(CurlOption.verbose, on ? 1L : 0L); 1914 } 1915 1916 // Connection settings 1917 1918 /// Set timeout for activity on connection. 1919 @property void dataTimeout(Duration d) 1920 { 1921 p.curl.set(CurlOption.low_speed_limit, 1); 1922 p.curl.set(CurlOption.low_speed_time, d.total!"seconds"); 1923 } 1924 1925 /** Set maximum time an operation is allowed to take. 1926 This includes dns resolution, connecting, data transfer, etc. 1927 */ 1928 @property void operationTimeout(Duration d) 1929 { 1930 p.curl.set(CurlOption.timeout_ms, d.total!"msecs"); 1931 } 1932 1933 /// Set timeout for connecting. 1934 @property void connectTimeout(Duration d) 1935 { 1936 p.curl.set(CurlOption.connecttimeout_ms, d.total!"msecs"); 1937 } 1938 1939 // Network settings 1940 1941 /** Proxy 1942 * See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXY, _proxy) 1943 */ 1944 @property void proxy(const(char)[] host) 1945 { 1946 p.curl.set(CurlOption.proxy, host); 1947 } 1948 1949 /** Proxy port 1950 * See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXYPORT, _proxy_port) 1951 */ 1952 @property void proxyPort(ushort port) 1953 { 1954 p.curl.set(CurlOption.proxyport, cast(long) port); 1955 } 1956 1957 /// Type of proxy 1958 alias CurlProxy = RawCurlProxy; 1959 1960 /** Proxy type 1961 * See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXY, _proxy_type) 1962 */ 1963 @property void proxyType(CurlProxy type) 1964 { 1965 p.curl.set(CurlOption.proxytype, cast(long) type); 1966 } 1967 1968 /// DNS lookup timeout. 1969 @property void dnsTimeout(Duration d) 1970 { 1971 p.curl.set(CurlOption.dns_cache_timeout, d.total!"msecs"); 1972 } 1973 1974 /** 1975 * The network interface to use in form of the IP of the interface. 1976 * 1977 * Example: 1978 * ---- 1979 * theprotocol.netInterface = "192.168.1.32"; 1980 * theprotocol.netInterface = [ 192, 168, 1, 32 ]; 1981 * ---- 1982 * 1983 * See: $(REF InternetAddress, std,socket) 1984 */ 1985 @property void netInterface(const(char)[] i) 1986 { 1987 p.curl.set(CurlOption.intrface, i); 1988 } 1989 1990 /// ditto 1991 @property void netInterface(const(ubyte)[4] i) 1992 { 1993 import std.format : format; 1994 const str = format("%d.%d.%d.%d", i[0], i[1], i[2], i[3]); 1995 netInterface = str; 1996 } 1997 1998 /// ditto 1999 @property void netInterface(InternetAddress i) 2000 { 2001 netInterface = i.toAddrString(); 2002 } 2003 2004 /** 2005 Set the local outgoing port to use. 2006 Params: 2007 port = the first outgoing port number to try and use 2008 */ 2009 @property void localPort(ushort port) 2010 { 2011 p.curl.set(CurlOption.localport, cast(long) port); 2012 } 2013 2014 /** 2015 Set the no proxy flag for the specified host names. 2016 Params: 2017 test = a list of comma host names that do not require 2018 proxy to get reached 2019 */ 2020 void setNoProxy(string hosts) 2021 { 2022 p.curl.set(CurlOption.noproxy, hosts); 2023 } 2024 2025 /** 2026 Set the local outgoing port range to use. 2027 This can be used together with the localPort property. 2028 Params: 2029 range = if the first port is occupied then try this many 2030 port number forwards 2031 */ 2032 @property void localPortRange(ushort range) 2033 { 2034 p.curl.set(CurlOption.localportrange, cast(long) range); 2035 } 2036 2037 /** Set the tcp no-delay socket option on or off. 2038 See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTTCPNODELAY, nodelay) 2039 */ 2040 @property void tcpNoDelay(bool on) 2041 { 2042 p.curl.set(CurlOption.tcp_nodelay, cast(long) (on ? 1 : 0) ); 2043 } 2044 2045 /** Sets whether SSL peer certificates should be verified. 2046 See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTSSLVERIFYPEER, verifypeer) 2047 */ 2048 @property void verifyPeer(bool on) 2049 { 2050 p.curl.set(CurlOption.ssl_verifypeer, on ? 1 : 0); 2051 } 2052 2053 /** Sets whether the host within an SSL certificate should be verified. 2054 See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTSSLVERIFYHOST, verifypeer) 2055 */ 2056 @property void verifyHost(bool on) 2057 { 2058 p.curl.set(CurlOption.ssl_verifyhost, on ? 2 : 0); 2059 } 2060 2061 // Authentication settings 2062 2063 /** 2064 Set the user name, password and optionally domain for authentication 2065 purposes. 2066 2067 Some protocols may need authentication in some cases. Use this 2068 function to provide credentials. 2069 2070 Params: 2071 username = the username 2072 password = the password 2073 domain = used for NTLM authentication only and is set to the NTLM domain 2074 name 2075 */ 2076 void setAuthentication(const(char)[] username, const(char)[] password, 2077 const(char)[] domain = "") 2078 { 2079 import std.format : format; 2080 if (!domain.empty) 2081 username = format("%s/%s", domain, username); 2082 p.curl.set(CurlOption.userpwd, format("%s:%s", username, password)); 2083 } 2084 2085 @system unittest 2086 { 2087 import std.algorithm.searching : canFind; 2088 2089 testServer.handle((s) { 2090 auto req = s.recvReq; 2091 assert(req.hdrs.canFind("GET /")); 2092 assert(req.hdrs.canFind("Basic dXNlcjpwYXNz")); 2093 s.send(httpOK()); 2094 }); 2095 2096 auto http = HTTP(testServer.addr); 2097 http.onReceive = (ubyte[] data) { return data.length; }; 2098 http.setAuthentication("user", "pass"); 2099 http.perform(); 2100 2101 // https://issues.dlang.org/show_bug.cgi?id=17540 2102 http.setNoProxy("www.example.com"); 2103 } 2104 2105 /** 2106 Set the user name and password for proxy authentication. 2107 2108 Params: 2109 username = the username 2110 password = the password 2111 */ 2112 void setProxyAuthentication(const(char)[] username, const(char)[] password) 2113 { 2114 import std.array : replace; 2115 import std.format : format; 2116 2117 p.curl.set(CurlOption.proxyuserpwd, 2118 format("%s:%s", 2119 username.replace(":", "%3A"), 2120 password.replace(":", "%3A")) 2121 ); 2122 } 2123 2124 /** 2125 * The event handler that gets called when data is needed for sending. The 2126 * length of the `void[]` specifies the maximum number of bytes that can 2127 * be sent. 2128 * 2129 * Returns: 2130 * The callback returns the number of elements in the buffer that have been 2131 * filled and are ready to send. 2132 * The special value `.abortRequest` can be returned in order to abort the 2133 * current request. 2134 * The special value `.pauseRequest` can be returned in order to pause the 2135 * current request. 2136 * 2137 * Example: 2138 * ---- 2139 * import std.net.curl; 2140 * string msg = "Hello world"; 2141 * auto client = HTTP("dlang.org"); 2142 * client.onSend = delegate size_t(void[] data) 2143 * { 2144 * auto m = cast(void[]) msg; 2145 * size_t length = m.length > data.length ? data.length : m.length; 2146 * if (length == 0) return 0; 2147 * data[0 .. length] = m[0 .. length]; 2148 * msg = msg[length..$]; 2149 * return length; 2150 * }; 2151 * client.perform(); 2152 * ---- 2153 */ 2154 @property void onSend(size_t delegate(void[]) callback) 2155 { 2156 p.curl.clear(CurlOption.postfields); // cannot specify data when using callback 2157 p.curl.onSend = callback; 2158 } 2159 2160 /** 2161 * The event handler that receives incoming data. Be sure to copy the 2162 * incoming ubyte[] since it is not guaranteed to be valid after the 2163 * callback returns. 2164 * 2165 * Returns: 2166 * The callback returns the number of incoming bytes read. If the entire array is 2167 * not read the request will abort. 2168 * The special value .pauseRequest can be returned in order to pause the 2169 * current request. 2170 * 2171 * Example: 2172 * ---- 2173 * import std.net.curl, std.stdio, std.conv; 2174 * auto client = HTTP("dlang.org"); 2175 * client.onReceive = (ubyte[] data) 2176 * { 2177 * writeln("Got data", to!(const(char)[])(data)); 2178 * return data.length; 2179 * }; 2180 * client.perform(); 2181 * ---- 2182 */ 2183 @property void onReceive(size_t delegate(ubyte[]) callback) 2184 { 2185 p.curl.onReceive = callback; 2186 } 2187 2188 /** 2189 * The event handler that gets called to inform of upload/download progress. 2190 * 2191 * Params: 2192 * dlTotal = total bytes to download 2193 * dlNow = currently downloaded bytes 2194 * ulTotal = total bytes to upload 2195 * ulNow = currently uploaded bytes 2196 * 2197 * Returns: 2198 * Return 0 from the callback to signal success, return non-zero to abort 2199 * transfer 2200 * 2201 * Example: 2202 * ---- 2203 * import std.net.curl, std.stdio; 2204 * auto client = HTTP("dlang.org"); 2205 * client.onProgress = delegate int(size_t dl, size_t dln, size_t ul, size_t uln) 2206 * { 2207 * writeln("Progress: downloaded ", dln, " of ", dl); 2208 * writeln("Progress: uploaded ", uln, " of ", ul); 2209 * return 0; 2210 * }; 2211 * client.perform(); 2212 * ---- 2213 */ 2214 @property void onProgress(int delegate(size_t dlTotal, size_t dlNow, 2215 size_t ulTotal, size_t ulNow) callback) 2216 { 2217 p.curl.onProgress = callback; 2218 } 2219 } 2220 2221 /* 2222 Decode `ubyte[]` array using the provided EncodingScheme up to maxChars 2223 Returns: Tuple of ubytes read and the `Char[]` characters decoded. 2224 Not all ubytes are guaranteed to be read in case of decoding error. 2225 */ 2226 private Tuple!(size_t,Char[]) 2227 decodeString(Char = char)(const(ubyte)[] data, 2228 EncodingScheme scheme, 2229 size_t maxChars = size_t.max) 2230 { 2231 import std.encoding : INVALID_SEQUENCE; 2232 Char[] res; 2233 immutable startLen = data.length; 2234 size_t charsDecoded = 0; 2235 while (data.length && charsDecoded < maxChars) 2236 { 2237 immutable dchar dc = scheme.safeDecode(data); 2238 if (dc == INVALID_SEQUENCE) 2239 { 2240 return typeof(return)(size_t.max, cast(Char[]) null); 2241 } 2242 charsDecoded++; 2243 res ~= dc; 2244 } 2245 return typeof(return)(startLen-data.length, res); 2246 } 2247 2248 /* 2249 Decode `ubyte[]` array using the provided `EncodingScheme` until a the 2250 line terminator specified is found. The basesrc parameter is effectively 2251 prepended to src as the first thing. 2252 2253 This function is used for decoding as much of the src buffer as 2254 possible until either the terminator is found or decoding fails. If 2255 it fails as the last data in the src it may mean that the src buffer 2256 were missing some bytes in order to represent a correct code 2257 point. Upon the next call to this function more bytes have been 2258 received from net and the failing bytes should be given as the 2259 basesrc parameter. It is done this way to minimize data copying. 2260 2261 Returns: true if a terminator was found 2262 Not all ubytes are guaranteed to be read in case of decoding error. 2263 any decoded chars will be inserted into dst. 2264 */ 2265 private bool decodeLineInto(Terminator, Char = char)(ref const(ubyte)[] basesrc, 2266 ref const(ubyte)[] src, 2267 ref Char[] dst, 2268 EncodingScheme scheme, 2269 Terminator terminator) 2270 { 2271 import std.algorithm.searching : endsWith; 2272 import std.encoding : INVALID_SEQUENCE; 2273 import std.exception : enforce; 2274 2275 // if there is anything in the basesrc then try to decode that 2276 // first. 2277 if (basesrc.length != 0) 2278 { 2279 // Try to ensure 4 entries in the basesrc by copying from src. 2280 immutable blen = basesrc.length; 2281 immutable len = (basesrc.length + src.length) >= 4 ? 2282 4 : basesrc.length + src.length; 2283 basesrc.length = len; 2284 2285 immutable dchar dc = scheme.safeDecode(basesrc); 2286 if (dc == INVALID_SEQUENCE) 2287 { 2288 enforce!CurlException(len != 4, "Invalid code sequence"); 2289 return false; 2290 } 2291 dst ~= dc; 2292 src = src[len-basesrc.length-blen .. $]; // remove used ubytes from src 2293 basesrc.length = 0; 2294 } 2295 2296 while (src.length) 2297 { 2298 const lsrc = src; 2299 dchar dc = scheme.safeDecode(src); 2300 if (dc == INVALID_SEQUENCE) 2301 { 2302 if (src.empty) 2303 { 2304 // The invalid sequence was in the end of the src. Maybe there 2305 // just need to be more bytes available so these last bytes are 2306 // put back to src for later use. 2307 src = lsrc; 2308 return false; 2309 } 2310 dc = '?'; 2311 } 2312 dst ~= dc; 2313 2314 if (dst.endsWith(terminator)) 2315 return true; 2316 } 2317 return false; // no terminator found 2318 } 2319 2320 /** 2321 * HTTP client functionality. 2322 * 2323 * Example: 2324 * 2325 * Get with custom data receivers: 2326 * 2327 * --- 2328 * import std.net.curl, std.stdio; 2329 * 2330 * auto http = HTTP("https://dlang.org"); 2331 * http.onReceiveHeader = 2332 * (in char[] key, in char[] value) { writeln(key ~ ": " ~ value); }; 2333 * http.onReceive = (ubyte[] data) { /+ drop +/ return data.length; }; 2334 * http.perform(); 2335 * --- 2336 * 2337 */ 2338 2339 /** 2340 * Put with data senders: 2341 * 2342 * --- 2343 * import std.net.curl, std.stdio; 2344 * 2345 * auto http = HTTP("https://dlang.org"); 2346 * auto msg = "Hello world"; 2347 * http.contentLength = msg.length; 2348 * http.onSend = (void[] data) 2349 * { 2350 * auto m = cast(void[]) msg; 2351 * size_t len = m.length > data.length ? data.length : m.length; 2352 * if (len == 0) return len; 2353 * data[0 .. len] = m[0 .. len]; 2354 * msg = msg[len..$]; 2355 * return len; 2356 * }; 2357 * http.perform(); 2358 * --- 2359 * 2360 */ 2361 2362 /** 2363 * Tracking progress: 2364 * 2365 * --- 2366 * import std.net.curl, std.stdio; 2367 * 2368 * auto http = HTTP(); 2369 * http.method = HTTP.Method.get; 2370 * http.url = "http://upload.wikimedia.org/wikipedia/commons/" ~ 2371 * "5/53/Wikipedia-logo-en-big.png"; 2372 * http.onReceive = (ubyte[] data) { return data.length; }; 2373 * http.onProgress = (size_t dltotal, size_t dlnow, 2374 * size_t ultotal, size_t ulnow) 2375 * { 2376 * writeln("Progress ", dltotal, ", ", dlnow, ", ", ultotal, ", ", ulnow); 2377 * return 0; 2378 * }; 2379 * http.perform(); 2380 * --- 2381 * 2382 * See_Also: $(LINK2 http://www.ietf.org/rfc/rfc2616.txt, RFC2616) 2383 * 2384 */ 2385 struct HTTP 2386 { 2387 mixin Protocol; 2388 2389 import std.datetime.systime : SysTime; 2390 import std.typecons : RefCounted; 2391 import etc.c.curl : CurlAuth, CurlInfo, curl_slist, CURLVERSION_NOW, curl_off_t; 2392 2393 /// Authentication method equal to $(REF CurlAuth, etc,c,curl) 2394 alias AuthMethod = CurlAuth; 2395 2396 static private uint defaultMaxRedirects = 10; 2397 2398 private struct Impl 2399 { 2400 ~this() 2401 { 2402 if (headersOut !is null) 2403 Curl.curl.slist_free_all(headersOut); 2404 if (curl.handle !is null) // work around RefCounted/emplace bug 2405 curl.shutdown(); 2406 } 2407 Curl curl; 2408 curl_slist* headersOut; 2409 string[string] headersIn; 2410 string charset; 2411 2412 /// The status line of the final sub-request in a request. 2413 StatusLine status; 2414 private void delegate(StatusLine) onReceiveStatusLine; 2415 2416 /// The HTTP method to use. 2417 Method method = Method.undefined; 2418 2419 @system @property void onReceiveHeader(void delegate(in char[] key, 2420 in char[] value) callback) 2421 { 2422 import std.algorithm.searching : startsWith; 2423 import std.regex : regex, match; 2424 import std.uni : toLower; 2425 2426 // Wrap incoming callback in order to separate http status line from 2427 // http headers. On redirected requests there may be several such 2428 // status lines. The last one is the one recorded. 2429 auto dg = (in char[] header) 2430 { 2431 import std.utf : UTFException; 2432 try 2433 { 2434 if (header.empty) 2435 { 2436 // header delimiter 2437 return; 2438 } 2439 if (header.startsWith("HTTP/")) 2440 { 2441 headersIn.clear(); 2442 if (parseStatusLine(header, status)) 2443 { 2444 if (onReceiveStatusLine != null) 2445 onReceiveStatusLine(status); 2446 } 2447 return; 2448 } 2449 2450 // Normal http header 2451 auto m = match(cast(char[]) header, regex("(.*?): (.*)$")); 2452 2453 auto fieldName = m.captures[1].toLower().idup; 2454 if (fieldName == "content-type") 2455 { 2456 auto mct = match(cast(char[]) m.captures[2], 2457 regex("charset=([^;]*)", "i")); 2458 if (!mct.empty && mct.captures.length > 1) 2459 charset = mct.captures[1].idup; 2460 } 2461 2462 if (!m.empty && callback !is null) 2463 callback(fieldName, m.captures[2]); 2464 headersIn[fieldName] = m.captures[2].idup; 2465 } 2466 catch (UTFException e) 2467 { 2468 //munch it - a header should be all ASCII, any "wrong UTF" is broken header 2469 } 2470 }; 2471 2472 curl.onReceiveHeader = dg; 2473 } 2474 } 2475 2476 private RefCounted!Impl p; 2477 import etc.c.curl : CurlTimeCond; 2478 2479 /// Parse status line, as received from / generated by cURL. 2480 private static bool parseStatusLine(const char[] header, out StatusLine status) @safe 2481 { 2482 import std.conv : to; 2483 import std.regex : regex, match; 2484 2485 const m = match(header, regex(r"^HTTP/(\d+)(?:\.(\d+))? (\d+)(?: (.*))?$")); 2486 if (m.empty) 2487 return false; // Invalid status line 2488 else 2489 { 2490 status.majorVersion = to!ushort(m.captures[1]); 2491 status.minorVersion = m.captures[2].length ? to!ushort(m.captures[2]) : 0; 2492 status.code = to!ushort(m.captures[3]); 2493 status.reason = m.captures[4].idup; 2494 return true; 2495 } 2496 } 2497 2498 @safe unittest 2499 { 2500 StatusLine status; 2501 assert(parseStatusLine("HTTP/1.1 200 OK", status) 2502 && status == StatusLine(1, 1, 200, "OK")); 2503 assert(parseStatusLine("HTTP/1.0 304 Not Modified", status) 2504 && status == StatusLine(1, 0, 304, "Not Modified")); 2505 // The HTTP2 protocol is binary; cURL generates this fake text header. 2506 assert(parseStatusLine("HTTP/2 200", status) 2507 && status == StatusLine(2, 0, 200, null)); 2508 } 2509 2510 /** Time condition enumeration as an alias of $(REF CurlTimeCond, etc,c,curl) 2511 2512 $(HTTP www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.25, _RFC2616 Section 14.25) 2513 */ 2514 alias TimeCond = CurlTimeCond; 2515 2516 /** 2517 Constructor taking the url as parameter. 2518 */ 2519 static HTTP opCall(const(char)[] url) 2520 { 2521 HTTP http; 2522 http.initialize(); 2523 http.url = url; 2524 return http; 2525 } 2526 2527 /// 2528 static HTTP opCall() 2529 { 2530 HTTP http; 2531 http.initialize(); 2532 return http; 2533 } 2534 2535 /// 2536 HTTP dup() 2537 { 2538 HTTP copy; 2539 copy.initialize(); 2540 copy.p.method = p.method; 2541 curl_slist* cur = p.headersOut; 2542 curl_slist* newlist = null; 2543 while (cur) 2544 { 2545 newlist = Curl.curl.slist_append(newlist, cur.data); 2546 cur = cur.next; 2547 } 2548 copy.p.headersOut = newlist; 2549 copy.p.curl.set(CurlOption.httpheader, copy.p.headersOut); 2550 copy.p.curl = p.curl.dup(); 2551 copy.dataTimeout = _defaultDataTimeout; 2552 copy.onReceiveHeader = null; 2553 return copy; 2554 } 2555 2556 private void initialize() 2557 { 2558 p.curl.initialize(); 2559 maxRedirects = HTTP.defaultMaxRedirects; 2560 p.charset = "ISO-8859-1"; // Default charset defined in HTTP RFC 2561 p.method = Method.undefined; 2562 setUserAgent(HTTP.defaultUserAgent); 2563 dataTimeout = _defaultDataTimeout; 2564 onReceiveHeader = null; 2565 verifyPeer = true; 2566 verifyHost = true; 2567 } 2568 2569 /** 2570 Perform a http request. 2571 2572 After the HTTP client has been setup and possibly assigned callbacks the 2573 `perform()` method will start performing the request towards the 2574 specified server. 2575 2576 Params: 2577 throwOnError = whether to throw an exception or return a CurlCode on error 2578 */ 2579 CurlCode perform(ThrowOnError throwOnError = Yes.throwOnError) 2580 { 2581 p.status.reset(); 2582 2583 CurlOption opt; 2584 final switch (p.method) 2585 { 2586 case Method.head: 2587 p.curl.set(CurlOption.nobody, 1L); 2588 opt = CurlOption.nobody; 2589 break; 2590 case Method.undefined: 2591 case Method.get: 2592 p.curl.set(CurlOption.httpget, 1L); 2593 opt = CurlOption.httpget; 2594 break; 2595 case Method.post: 2596 p.curl.set(CurlOption.post, 1L); 2597 opt = CurlOption.post; 2598 break; 2599 case Method.put: 2600 p.curl.set(CurlOption.upload, 1L); 2601 opt = CurlOption.upload; 2602 break; 2603 case Method.del: 2604 p.curl.set(CurlOption.customrequest, "DELETE"); 2605 opt = CurlOption.customrequest; 2606 break; 2607 case Method.options: 2608 p.curl.set(CurlOption.customrequest, "OPTIONS"); 2609 opt = CurlOption.customrequest; 2610 break; 2611 case Method.trace: 2612 p.curl.set(CurlOption.customrequest, "TRACE"); 2613 opt = CurlOption.customrequest; 2614 break; 2615 case Method.connect: 2616 p.curl.set(CurlOption.customrequest, "CONNECT"); 2617 opt = CurlOption.customrequest; 2618 break; 2619 case Method.patch: 2620 p.curl.set(CurlOption.customrequest, "PATCH"); 2621 opt = CurlOption.customrequest; 2622 break; 2623 } 2624 2625 scope (exit) p.curl.clear(opt); 2626 return p.curl.perform(throwOnError); 2627 } 2628 2629 /// The URL to specify the location of the resource. 2630 @property void url(const(char)[] url) 2631 { 2632 import std.algorithm.searching : startsWith; 2633 import std.uni : toLower; 2634 if (!startsWith(url.toLower(), "http://", "https://")) 2635 url = "http://" ~ url; 2636 p.curl.set(CurlOption.url, url); 2637 } 2638 2639 /// Set the CA certificate bundle file to use for SSL peer verification 2640 @property void caInfo(const(char)[] caFile) 2641 { 2642 p.curl.set(CurlOption.cainfo, caFile); 2643 } 2644 2645 // This is a workaround for mixed in content not having its 2646 // docs mixed in. 2647 version (StdDdoc) 2648 { 2649 static import etc.c.curl; 2650 2651 /// Value to return from `onSend`/`onReceive` delegates in order to 2652 /// pause a request 2653 alias requestPause = CurlReadFunc.pause; 2654 2655 /// Value to return from onSend delegate in order to abort a request 2656 alias requestAbort = CurlReadFunc.abort; 2657 2658 /** 2659 True if the instance is stopped. A stopped instance is not usable. 2660 */ 2661 @property bool isStopped(); 2662 2663 /// Stop and invalidate this instance. 2664 void shutdown(); 2665 2666 /** Set verbose. 2667 This will print request information to stderr. 2668 */ 2669 @property void verbose(bool on); 2670 2671 // Connection settings 2672 2673 /// Set timeout for activity on connection. 2674 @property void dataTimeout(Duration d); 2675 2676 /** Set maximum time an operation is allowed to take. 2677 This includes dns resolution, connecting, data transfer, etc. 2678 */ 2679 @property void operationTimeout(Duration d); 2680 2681 /// Set timeout for connecting. 2682 @property void connectTimeout(Duration d); 2683 2684 // Network settings 2685 2686 /** Proxy 2687 * See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXY, _proxy) 2688 */ 2689 @property void proxy(const(char)[] host); 2690 2691 /** Proxy port 2692 * See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXYPORT, _proxy_port) 2693 */ 2694 @property void proxyPort(ushort port); 2695 2696 /// Type of proxy 2697 alias CurlProxy = etc.c.curl.CurlProxy; 2698 2699 /** Proxy type 2700 * See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXY, _proxy_type) 2701 */ 2702 @property void proxyType(CurlProxy type); 2703 2704 /// DNS lookup timeout. 2705 @property void dnsTimeout(Duration d); 2706 2707 /** 2708 * The network interface to use in form of the IP of the interface. 2709 * 2710 * Example: 2711 * ---- 2712 * theprotocol.netInterface = "192.168.1.32"; 2713 * theprotocol.netInterface = [ 192, 168, 1, 32 ]; 2714 * ---- 2715 * 2716 * See: $(REF InternetAddress, std,socket) 2717 */ 2718 @property void netInterface(const(char)[] i); 2719 2720 /// ditto 2721 @property void netInterface(const(ubyte)[4] i); 2722 2723 /// ditto 2724 @property void netInterface(InternetAddress i); 2725 2726 /** 2727 Set the local outgoing port to use. 2728 Params: 2729 port = the first outgoing port number to try and use 2730 */ 2731 @property void localPort(ushort port); 2732 2733 /** 2734 Set the local outgoing port range to use. 2735 This can be used together with the localPort property. 2736 Params: 2737 range = if the first port is occupied then try this many 2738 port number forwards 2739 */ 2740 @property void localPortRange(ushort range); 2741 2742 /** Set the tcp no-delay socket option on or off. 2743 See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTTCPNODELAY, nodelay) 2744 */ 2745 @property void tcpNoDelay(bool on); 2746 2747 // Authentication settings 2748 2749 /** 2750 Set the user name, password and optionally domain for authentication 2751 purposes. 2752 2753 Some protocols may need authentication in some cases. Use this 2754 function to provide credentials. 2755 2756 Params: 2757 username = the username 2758 password = the password 2759 domain = used for NTLM authentication only and is set to the NTLM domain 2760 name 2761 */ 2762 void setAuthentication(const(char)[] username, const(char)[] password, 2763 const(char)[] domain = ""); 2764 2765 /** 2766 Set the user name and password for proxy authentication. 2767 2768 Params: 2769 username = the username 2770 password = the password 2771 */ 2772 void setProxyAuthentication(const(char)[] username, const(char)[] password); 2773 2774 /** 2775 * The event handler that gets called when data is needed for sending. The 2776 * length of the `void[]` specifies the maximum number of bytes that can 2777 * be sent. 2778 * 2779 * Returns: 2780 * The callback returns the number of elements in the buffer that have been 2781 * filled and are ready to send. 2782 * The special value `.abortRequest` can be returned in order to abort the 2783 * current request. 2784 * The special value `.pauseRequest` can be returned in order to pause the 2785 * current request. 2786 * 2787 * Example: 2788 * ---- 2789 * import std.net.curl; 2790 * string msg = "Hello world"; 2791 * auto client = HTTP("dlang.org"); 2792 * client.onSend = delegate size_t(void[] data) 2793 * { 2794 * auto m = cast(void[]) msg; 2795 * size_t length = m.length > data.length ? data.length : m.length; 2796 * if (length == 0) return 0; 2797 * data[0 .. length] = m[0 .. length]; 2798 * msg = msg[length..$]; 2799 * return length; 2800 * }; 2801 * client.perform(); 2802 * ---- 2803 */ 2804 @property void onSend(size_t delegate(void[]) callback); 2805 2806 /** 2807 * The event handler that receives incoming data. Be sure to copy the 2808 * incoming ubyte[] since it is not guaranteed to be valid after the 2809 * callback returns. 2810 * 2811 * Returns: 2812 * The callback returns the incoming bytes read. If not the entire array is 2813 * the request will abort. 2814 * The special value .pauseRequest can be returned in order to pause the 2815 * current request. 2816 * 2817 * Example: 2818 * ---- 2819 * import std.net.curl, std.stdio, std.conv; 2820 * auto client = HTTP("dlang.org"); 2821 * client.onReceive = (ubyte[] data) 2822 * { 2823 * writeln("Got data", to!(const(char)[])(data)); 2824 * return data.length; 2825 * }; 2826 * client.perform(); 2827 * ---- 2828 */ 2829 @property void onReceive(size_t delegate(ubyte[]) callback); 2830 2831 /** 2832 * Register an event handler that gets called to inform of 2833 * upload/download progress. 2834 * 2835 * Callback_parameters: 2836 * $(CALLBACK_PARAMS) 2837 * 2838 * Callback_returns: Return 0 to signal success, return non-zero to 2839 * abort transfer. 2840 * 2841 * Example: 2842 * ---- 2843 * import std.net.curl, std.stdio; 2844 * auto client = HTTP("dlang.org"); 2845 * client.onProgress = delegate int(size_t dl, size_t dln, size_t ul, size_t uln) 2846 * { 2847 * writeln("Progress: downloaded ", dln, " of ", dl); 2848 * writeln("Progress: uploaded ", uln, " of ", ul); 2849 * return 0; 2850 * }; 2851 * client.perform(); 2852 * ---- 2853 */ 2854 @property void onProgress(int delegate(size_t dlTotal, size_t dlNow, 2855 size_t ulTotal, size_t ulNow) callback); 2856 } 2857 2858 /** Clear all outgoing headers. 2859 */ 2860 void clearRequestHeaders() 2861 { 2862 if (p.headersOut !is null) 2863 Curl.curl.slist_free_all(p.headersOut); 2864 p.headersOut = null; 2865 p.curl.clear(CurlOption.httpheader); 2866 } 2867 2868 /** Add a header e.g. "X-CustomField: Something is fishy". 2869 * 2870 * There is no remove header functionality. Do a $(LREF clearRequestHeaders) 2871 * and set the needed headers instead. 2872 * 2873 * Example: 2874 * --- 2875 * import std.net.curl; 2876 * auto client = HTTP(); 2877 * client.addRequestHeader("X-Custom-ABC", "This is the custom value"); 2878 * auto content = get("dlang.org", client); 2879 * --- 2880 */ 2881 void addRequestHeader(const(char)[] name, const(char)[] value) 2882 { 2883 import std.format : format; 2884 import std.internal.cstring : tempCString; 2885 import std.uni : icmp; 2886 2887 if (icmp(name, "User-Agent") == 0) 2888 return setUserAgent(value); 2889 string nv = format("%s: %s", name, value); 2890 p.headersOut = Curl.curl.slist_append(p.headersOut, 2891 nv.tempCString().buffPtr); 2892 p.curl.set(CurlOption.httpheader, p.headersOut); 2893 } 2894 2895 /** 2896 * The default "User-Agent" value send with a request. 2897 * It has the form "Phobos-std.net.curl/$(I PHOBOS_VERSION) (libcurl/$(I CURL_VERSION))" 2898 */ 2899 static string defaultUserAgent() @property 2900 { 2901 import std.compiler : version_major, version_minor; 2902 import std.format : format, sformat; 2903 2904 // http://curl.haxx.se/docs/versions.html 2905 enum fmt = "Phobos-std.net.curl/%d.%03d (libcurl/%d.%d.%d)"; 2906 enum maxLen = fmt.length - "%d%03d%d%d%d".length + 10 + 10 + 3 + 3 + 3; 2907 2908 static char[maxLen] buf = void; 2909 static string userAgent; 2910 2911 if (!userAgent.length) 2912 { 2913 auto curlVer = Curl.curl.version_info(CURLVERSION_NOW).version_num; 2914 userAgent = cast(immutable) sformat( 2915 buf, fmt, version_major, version_minor, 2916 curlVer >> 16 & 0xFF, curlVer >> 8 & 0xFF, curlVer & 0xFF); 2917 } 2918 return userAgent; 2919 } 2920 2921 /** Set the value of the user agent request header field. 2922 * 2923 * By default a request has it's "User-Agent" field set to $(LREF 2924 * defaultUserAgent) even if `setUserAgent` was never called. Pass 2925 * an empty string to suppress the "User-Agent" field altogether. 2926 */ 2927 void setUserAgent(const(char)[] userAgent) 2928 { 2929 p.curl.set(CurlOption.useragent, userAgent); 2930 } 2931 2932 /** 2933 * Get various timings defined in $(REF CurlInfo, etc, c, curl). 2934 * The value is usable only if the return value is equal to `etc.c.curl.CurlError.ok`. 2935 * 2936 * Params: 2937 * timing = one of the timings defined in $(REF CurlInfo, etc, c, curl). 2938 * The values are: 2939 * `etc.c.curl.CurlInfo.namelookup_time`, 2940 * `etc.c.curl.CurlInfo.connect_time`, 2941 * `etc.c.curl.CurlInfo.pretransfer_time`, 2942 * `etc.c.curl.CurlInfo.starttransfer_time`, 2943 * `etc.c.curl.CurlInfo.redirect_time`, 2944 * `etc.c.curl.CurlInfo.appconnect_time`, 2945 * `etc.c.curl.CurlInfo.total_time`. 2946 * val = the actual value of the inquired timing. 2947 * 2948 * Returns: 2949 * The return code of the operation. The value stored in val 2950 * should be used only if the return value is `etc.c.curl.CurlInfo.ok`. 2951 * 2952 * Example: 2953 * --- 2954 * import std.net.curl; 2955 * import etc.c.curl : CurlError, CurlInfo; 2956 * 2957 * auto client = HTTP("dlang.org"); 2958 * client.perform(); 2959 * 2960 * double val; 2961 * CurlCode code; 2962 * 2963 * code = client.getTiming(CurlInfo.namelookup_time, val); 2964 * assert(code == CurlError.ok); 2965 * --- 2966 */ 2967 CurlCode getTiming(CurlInfo timing, ref double val) 2968 { 2969 return p.curl.getTiming(timing, val); 2970 } 2971 2972 /** The headers read from a successful response. 2973 * 2974 */ 2975 @property string[string] responseHeaders() 2976 { 2977 return p.headersIn; 2978 } 2979 2980 /// HTTP method used. 2981 @property void method(Method m) 2982 { 2983 p.method = m; 2984 } 2985 2986 /// ditto 2987 @property Method method() 2988 { 2989 return p.method; 2990 } 2991 2992 /** 2993 HTTP status line of last response. One call to perform may 2994 result in several requests because of redirection. 2995 */ 2996 @property StatusLine statusLine() 2997 { 2998 return p.status; 2999 } 3000 3001 /// Set the active cookie string e.g. "name1=value1;name2=value2" 3002 void setCookie(const(char)[] cookie) 3003 { 3004 p.curl.set(CurlOption.cookie, cookie); 3005 } 3006 3007 /// Set a file path to where a cookie jar should be read/stored. 3008 void setCookieJar(const(char)[] path) 3009 { 3010 p.curl.set(CurlOption.cookiefile, path); 3011 if (path.length) 3012 p.curl.set(CurlOption.cookiejar, path); 3013 } 3014 3015 /// Flush cookie jar to disk. 3016 void flushCookieJar() 3017 { 3018 p.curl.set(CurlOption.cookielist, "FLUSH"); 3019 } 3020 3021 /// Clear session cookies. 3022 void clearSessionCookies() 3023 { 3024 p.curl.set(CurlOption.cookielist, "SESS"); 3025 } 3026 3027 /// Clear all cookies. 3028 void clearAllCookies() 3029 { 3030 p.curl.set(CurlOption.cookielist, "ALL"); 3031 } 3032 3033 /** 3034 Set time condition on the request. 3035 3036 Params: 3037 cond = `CurlTimeCond.{none,ifmodsince,ifunmodsince,lastmod}` 3038 timestamp = Timestamp for the condition 3039 3040 $(HTTP www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.25, _RFC2616 Section 14.25) 3041 */ 3042 void setTimeCondition(HTTP.TimeCond cond, SysTime timestamp) 3043 { 3044 p.curl.set(CurlOption.timecondition, cond); 3045 p.curl.set(CurlOption.timevalue, timestamp.toUnixTime()); 3046 } 3047 3048 /** Specifying data to post when not using the onSend callback. 3049 * 3050 * The data is NOT copied by the library. Content-Type will default to 3051 * application/octet-stream. Data is not converted or encoded by this 3052 * method. 3053 * 3054 * Example: 3055 * ---- 3056 * import std.net.curl, std.stdio, std.conv; 3057 * auto http = HTTP("http://www.mydomain.com"); 3058 * http.onReceive = (ubyte[] data) { writeln(to!(const(char)[])(data)); return data.length; }; 3059 * http.postData = [1,2,3,4,5]; 3060 * http.perform(); 3061 * ---- 3062 */ 3063 @property void postData(const(void)[] data) 3064 { 3065 setPostData(data, "application/octet-stream"); 3066 } 3067 3068 /** Specifying data to post when not using the onSend callback. 3069 * 3070 * The data is NOT copied by the library. Content-Type will default to 3071 * text/plain. Data is not converted or encoded by this method. 3072 * 3073 * Example: 3074 * ---- 3075 * import std.net.curl, std.stdio, std.conv; 3076 * auto http = HTTP("http://www.mydomain.com"); 3077 * http.onReceive = (ubyte[] data) { writeln(to!(const(char)[])(data)); return data.length; }; 3078 * http.postData = "The quick...."; 3079 * http.perform(); 3080 * ---- 3081 */ 3082 @property void postData(const(char)[] data) 3083 { 3084 setPostData(data, "text/plain"); 3085 } 3086 3087 /** 3088 * Specify data to post when not using the onSend callback, with 3089 * user-specified Content-Type. 3090 * Params: 3091 * data = Data to post. 3092 * contentType = MIME type of the data, for example, "text/plain" or 3093 * "application/octet-stream". See also: 3094 * $(LINK2 http://en.wikipedia.org/wiki/Internet_media_type, 3095 * Internet media type) on Wikipedia. 3096 * ----- 3097 * import std.net.curl; 3098 * auto http = HTTP("http://onlineform.example.com"); 3099 * auto data = "app=login&username=bob&password=s00perS3kret"; 3100 * http.setPostData(data, "application/x-www-form-urlencoded"); 3101 * http.onReceive = (ubyte[] data) { return data.length; }; 3102 * http.perform(); 3103 * ----- 3104 */ 3105 void setPostData(const(void)[] data, string contentType) 3106 { 3107 // cannot use callback when specifying data directly so it is disabled here. 3108 p.curl.clear(CurlOption.readfunction); 3109 addRequestHeader("Content-Type", contentType); 3110 p.curl.set(CurlOption.postfields, cast(void*) data.ptr); 3111 p.curl.set(CurlOption.postfieldsize, data.length); 3112 if (method == Method.undefined) 3113 method = Method.post; 3114 } 3115 3116 @system unittest 3117 { 3118 import std.algorithm.searching : canFind; 3119 3120 testServer.handle((s) { 3121 auto req = s.recvReq!ubyte; 3122 assert(req.hdrs.canFind("POST /path")); 3123 assert(req.bdy.canFind(cast(ubyte[])[0, 1, 2, 3, 4])); 3124 assert(req.bdy.canFind(cast(ubyte[])[253, 254, 255])); 3125 s.send(httpOK(cast(ubyte[])[17, 27, 35, 41])); 3126 }); 3127 auto data = new ubyte[](256); 3128 foreach (i, ref ub; data) 3129 ub = cast(ubyte) i; 3130 3131 auto http = HTTP(testServer.addr~"/path"); 3132 http.postData = data; 3133 ubyte[] res; 3134 http.onReceive = (data) { res ~= data; return data.length; }; 3135 http.perform(); 3136 assert(res == cast(ubyte[])[17, 27, 35, 41]); 3137 } 3138 3139 /** 3140 * Set the event handler that receives incoming headers. 3141 * 3142 * The callback will receive a header field key, value as parameter. The 3143 * `const(char)[]` arrays are not valid after the delegate has returned. 3144 * 3145 * Example: 3146 * ---- 3147 * import std.net.curl, std.stdio, std.conv; 3148 * auto http = HTTP("dlang.org"); 3149 * http.onReceive = (ubyte[] data) { writeln(to!(const(char)[])(data)); return data.length; }; 3150 * http.onReceiveHeader = (in char[] key, in char[] value) { writeln(key, " = ", value); }; 3151 * http.perform(); 3152 * ---- 3153 */ 3154 @property void onReceiveHeader(void delegate(in char[] key, 3155 in char[] value) callback) 3156 { 3157 p.onReceiveHeader = callback; 3158 } 3159 3160 /** 3161 Callback for each received StatusLine. 3162 3163 Notice that several callbacks can be done for each call to 3164 `perform()` due to redirections. 3165 3166 See_Also: $(LREF StatusLine) 3167 */ 3168 @property void onReceiveStatusLine(void delegate(StatusLine) callback) 3169 { 3170 p.onReceiveStatusLine = callback; 3171 } 3172 3173 /** 3174 The content length in bytes when using request that has content 3175 e.g. POST/PUT and not using chunked transfer. Is set as the 3176 "Content-Length" header. Set to ulong.max to reset to chunked transfer. 3177 */ 3178 @property void contentLength(ulong len) 3179 { 3180 import std.conv : to; 3181 3182 CurlOption lenOpt; 3183 3184 // Force post if necessary 3185 if (p.method != Method.put && p.method != Method.post && 3186 p.method != Method.patch) 3187 p.method = Method.post; 3188 3189 if (p.method == Method.post || p.method == Method.patch) 3190 lenOpt = CurlOption.postfieldsize_large; 3191 else 3192 lenOpt = CurlOption.infilesize_large; 3193 3194 if (size_t.max != ulong.max && len == size_t.max) 3195 len = ulong.max; // check size_t.max for backwards compat, turn into error 3196 3197 if (len == ulong.max) 3198 { 3199 // HTTP 1.1 supports requests with no length header set. 3200 addRequestHeader("Transfer-Encoding", "chunked"); 3201 addRequestHeader("Expect", "100-continue"); 3202 } 3203 else 3204 { 3205 p.curl.set(lenOpt, to!curl_off_t(len)); 3206 } 3207 } 3208 3209 /** 3210 Authentication method as specified in $(LREF AuthMethod). 3211 */ 3212 @property void authenticationMethod(AuthMethod authMethod) 3213 { 3214 p.curl.set(CurlOption.httpauth, cast(long) authMethod); 3215 } 3216 3217 /** 3218 Set max allowed redirections using the location header. 3219 uint.max for infinite. 3220 */ 3221 @property void maxRedirects(uint maxRedirs) 3222 { 3223 if (maxRedirs == uint.max) 3224 { 3225 // Disable 3226 p.curl.set(CurlOption.followlocation, 0); 3227 } 3228 else 3229 { 3230 p.curl.set(CurlOption.followlocation, 1); 3231 p.curl.set(CurlOption.maxredirs, maxRedirs); 3232 } 3233 } 3234 3235 /** <a name="HTTP.Method"/>The standard HTTP methods : 3236 * $(HTTP www.w3.org/Protocols/rfc2616/rfc2616-sec5.html#sec5.1.1, _RFC2616 Section 5.1.1) 3237 */ 3238 enum Method 3239 { 3240 undefined, 3241 head, /// 3242 get, /// 3243 post, /// 3244 put, /// 3245 del, /// 3246 options, /// 3247 trace, /// 3248 connect, /// 3249 patch, /// 3250 } 3251 3252 /** 3253 HTTP status line ie. the first line returned in an HTTP response. 3254 3255 If authentication or redirections are done then the status will be for 3256 the last response received. 3257 */ 3258 struct StatusLine 3259 { 3260 ushort majorVersion; /// Major HTTP version ie. 1 in HTTP/1.0. 3261 ushort minorVersion; /// Minor HTTP version ie. 0 in HTTP/1.0. 3262 ushort code; /// HTTP status line code e.g. 200. 3263 string reason; /// HTTP status line reason string. 3264 3265 /// Reset this status line 3266 @safe void reset() 3267 { 3268 majorVersion = 0; 3269 minorVersion = 0; 3270 code = 0; 3271 reason = ""; 3272 } 3273 3274 /// 3275 string toString() const 3276 { 3277 import std.format : format; 3278 return format("%s %s (%s.%s)", 3279 code, reason, majorVersion, minorVersion); 3280 } 3281 } 3282 3283 } // HTTP 3284 3285 @system unittest // charset/Charset/CHARSET/... 3286 { 3287 import etc.c.curl; 3288 3289 static foreach (c; ["charset", "Charset", "CHARSET", "CharSet", "charSet", 3290 "ChArSeT", "cHaRsEt"]) 3291 {{ 3292 testServer.handle((s) { 3293 s.send("HTTP/1.1 200 OK\r\n"~ 3294 "Content-Length: 0\r\n"~ 3295 "Content-Type: text/plain; " ~ c ~ "=foo\r\n" ~ 3296 "\r\n"); 3297 }); 3298 3299 auto http = HTTP(testServer.addr); 3300 http.perform(); 3301 assert(http.p.charset == "foo"); 3302 3303 // https://issues.dlang.org/show_bug.cgi?id=16736 3304 double val; 3305 CurlCode code; 3306 3307 code = http.getTiming(CurlInfo.total_time, val); 3308 assert(code == CurlError.ok); 3309 code = http.getTiming(CurlInfo.namelookup_time, val); 3310 assert(code == CurlError.ok); 3311 code = http.getTiming(CurlInfo.connect_time, val); 3312 assert(code == CurlError.ok); 3313 code = http.getTiming(CurlInfo.pretransfer_time, val); 3314 assert(code == CurlError.ok); 3315 code = http.getTiming(CurlInfo.starttransfer_time, val); 3316 assert(code == CurlError.ok); 3317 code = http.getTiming(CurlInfo.redirect_time, val); 3318 assert(code == CurlError.ok); 3319 code = http.getTiming(CurlInfo.appconnect_time, val); 3320 assert(code == CurlError.ok); 3321 }} 3322 } 3323 3324 /** 3325 FTP client functionality. 3326 3327 See_Also: $(HTTP tools.ietf.org/html/rfc959, RFC959) 3328 */ 3329 struct FTP 3330 { 3331 3332 mixin Protocol; 3333 3334 import std.typecons : RefCounted; 3335 import etc.c.curl : CurlError, CurlInfo, curl_off_t, curl_slist; 3336 3337 private struct Impl 3338 { 3339 ~this() 3340 { 3341 if (commands !is null) 3342 Curl.curl.slist_free_all(commands); 3343 if (curl.handle !is null) // work around RefCounted/emplace bug 3344 curl.shutdown(); 3345 } 3346 curl_slist* commands; 3347 Curl curl; 3348 string encoding; 3349 } 3350 3351 private RefCounted!Impl p; 3352 3353 /** 3354 FTP access to the specified url. 3355 */ 3356 static FTP opCall(const(char)[] url) 3357 { 3358 FTP ftp; 3359 ftp.initialize(); 3360 ftp.url = url; 3361 return ftp; 3362 } 3363 3364 /// 3365 static FTP opCall() 3366 { 3367 FTP ftp; 3368 ftp.initialize(); 3369 return ftp; 3370 } 3371 3372 /// 3373 FTP dup() 3374 { 3375 FTP copy = FTP(); 3376 copy.initialize(); 3377 copy.p.encoding = p.encoding; 3378 copy.p.curl = p.curl.dup(); 3379 curl_slist* cur = p.commands; 3380 curl_slist* newlist = null; 3381 while (cur) 3382 { 3383 newlist = Curl.curl.slist_append(newlist, cur.data); 3384 cur = cur.next; 3385 } 3386 copy.p.commands = newlist; 3387 copy.p.curl.set(CurlOption.postquote, copy.p.commands); 3388 copy.dataTimeout = _defaultDataTimeout; 3389 return copy; 3390 } 3391 3392 private void initialize() 3393 { 3394 p.curl.initialize(); 3395 p.encoding = "ISO-8859-1"; 3396 dataTimeout = _defaultDataTimeout; 3397 } 3398 3399 /** 3400 Performs the ftp request as it has been configured. 3401 3402 After a FTP client has been setup and possibly assigned callbacks the $(D 3403 perform()) method will start performing the actual communication with the 3404 server. 3405 3406 Params: 3407 throwOnError = whether to throw an exception or return a CurlCode on error 3408 */ 3409 CurlCode perform(ThrowOnError throwOnError = Yes.throwOnError) 3410 { 3411 return p.curl.perform(throwOnError); 3412 } 3413 3414 /// The URL to specify the location of the resource. 3415 @property void url(const(char)[] url) 3416 { 3417 import std.algorithm.searching : startsWith; 3418 import std.uni : toLower; 3419 3420 if (!startsWith(url.toLower(), "ftp://", "ftps://")) 3421 url = "ftp://" ~ url; 3422 p.curl.set(CurlOption.url, url); 3423 } 3424 3425 // This is a workaround for mixed in content not having its 3426 // docs mixed in. 3427 version (StdDdoc) 3428 { 3429 static import etc.c.curl; 3430 3431 /// Value to return from `onSend`/`onReceive` delegates in order to 3432 /// pause a request 3433 alias requestPause = CurlReadFunc.pause; 3434 3435 /// Value to return from onSend delegate in order to abort a request 3436 alias requestAbort = CurlReadFunc.abort; 3437 3438 /** 3439 True if the instance is stopped. A stopped instance is not usable. 3440 */ 3441 @property bool isStopped(); 3442 3443 /// Stop and invalidate this instance. 3444 void shutdown(); 3445 3446 /** Set verbose. 3447 This will print request information to stderr. 3448 */ 3449 @property void verbose(bool on); 3450 3451 // Connection settings 3452 3453 /// Set timeout for activity on connection. 3454 @property void dataTimeout(Duration d); 3455 3456 /** Set maximum time an operation is allowed to take. 3457 This includes dns resolution, connecting, data transfer, etc. 3458 */ 3459 @property void operationTimeout(Duration d); 3460 3461 /// Set timeout for connecting. 3462 @property void connectTimeout(Duration d); 3463 3464 // Network settings 3465 3466 /** Proxy 3467 * See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXY, _proxy) 3468 */ 3469 @property void proxy(const(char)[] host); 3470 3471 /** Proxy port 3472 * See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXYPORT, _proxy_port) 3473 */ 3474 @property void proxyPort(ushort port); 3475 3476 /// Type of proxy 3477 alias CurlProxy = etc.c.curl.CurlProxy; 3478 3479 /** Proxy type 3480 * See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXY, _proxy_type) 3481 */ 3482 @property void proxyType(CurlProxy type); 3483 3484 /// DNS lookup timeout. 3485 @property void dnsTimeout(Duration d); 3486 3487 /** 3488 * The network interface to use in form of the IP of the interface. 3489 * 3490 * Example: 3491 * ---- 3492 * theprotocol.netInterface = "192.168.1.32"; 3493 * theprotocol.netInterface = [ 192, 168, 1, 32 ]; 3494 * ---- 3495 * 3496 * See: $(REF InternetAddress, std,socket) 3497 */ 3498 @property void netInterface(const(char)[] i); 3499 3500 /// ditto 3501 @property void netInterface(const(ubyte)[4] i); 3502 3503 /// ditto 3504 @property void netInterface(InternetAddress i); 3505 3506 /** 3507 Set the local outgoing port to use. 3508 Params: 3509 port = the first outgoing port number to try and use 3510 */ 3511 @property void localPort(ushort port); 3512 3513 /** 3514 Set the local outgoing port range to use. 3515 This can be used together with the localPort property. 3516 Params: 3517 range = if the first port is occupied then try this many 3518 port number forwards 3519 */ 3520 @property void localPortRange(ushort range); 3521 3522 /** Set the tcp no-delay socket option on or off. 3523 See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTTCPNODELAY, nodelay) 3524 */ 3525 @property void tcpNoDelay(bool on); 3526 3527 // Authentication settings 3528 3529 /** 3530 Set the user name, password and optionally domain for authentication 3531 purposes. 3532 3533 Some protocols may need authentication in some cases. Use this 3534 function to provide credentials. 3535 3536 Params: 3537 username = the username 3538 password = the password 3539 domain = used for NTLM authentication only and is set to the NTLM domain 3540 name 3541 */ 3542 void setAuthentication(const(char)[] username, const(char)[] password, 3543 const(char)[] domain = ""); 3544 3545 /** 3546 Set the user name and password for proxy authentication. 3547 3548 Params: 3549 username = the username 3550 password = the password 3551 */ 3552 void setProxyAuthentication(const(char)[] username, const(char)[] password); 3553 3554 /** 3555 * The event handler that gets called when data is needed for sending. The 3556 * length of the `void[]` specifies the maximum number of bytes that can 3557 * be sent. 3558 * 3559 * Returns: 3560 * The callback returns the number of elements in the buffer that have been 3561 * filled and are ready to send. 3562 * The special value `.abortRequest` can be returned in order to abort the 3563 * current request. 3564 * The special value `.pauseRequest` can be returned in order to pause the 3565 * current request. 3566 * 3567 */ 3568 @property void onSend(size_t delegate(void[]) callback); 3569 3570 /** 3571 * The event handler that receives incoming data. Be sure to copy the 3572 * incoming ubyte[] since it is not guaranteed to be valid after the 3573 * callback returns. 3574 * 3575 * Returns: 3576 * The callback returns the incoming bytes read. If not the entire array is 3577 * the request will abort. 3578 * The special value .pauseRequest can be returned in order to pause the 3579 * current request. 3580 * 3581 */ 3582 @property void onReceive(size_t delegate(ubyte[]) callback); 3583 3584 /** 3585 * The event handler that gets called to inform of upload/download progress. 3586 * 3587 * Callback_parameters: 3588 * $(CALLBACK_PARAMS) 3589 * 3590 * Callback_returns: 3591 * Return 0 from the callback to signal success, return non-zero to 3592 * abort transfer. 3593 */ 3594 @property void onProgress(int delegate(size_t dlTotal, size_t dlNow, 3595 size_t ulTotal, size_t ulNow) callback); 3596 } 3597 3598 /** Clear all commands send to ftp server. 3599 */ 3600 void clearCommands() 3601 { 3602 if (p.commands !is null) 3603 Curl.curl.slist_free_all(p.commands); 3604 p.commands = null; 3605 p.curl.clear(CurlOption.postquote); 3606 } 3607 3608 /** Add a command to send to ftp server. 3609 * 3610 * There is no remove command functionality. Do a $(LREF clearCommands) and 3611 * set the needed commands instead. 3612 * 3613 * Example: 3614 * --- 3615 * import std.net.curl; 3616 * auto client = FTP(); 3617 * client.addCommand("RNFR my_file.txt"); 3618 * client.addCommand("RNTO my_renamed_file.txt"); 3619 * upload("my_file.txt", "ftp.digitalmars.com", client); 3620 * --- 3621 */ 3622 void addCommand(const(char)[] command) 3623 { 3624 import std.internal.cstring : tempCString; 3625 p.commands = Curl.curl.slist_append(p.commands, 3626 command.tempCString().buffPtr); 3627 p.curl.set(CurlOption.postquote, p.commands); 3628 } 3629 3630 /// Connection encoding. Defaults to ISO-8859-1. 3631 @property void encoding(string name) 3632 { 3633 p.encoding = name; 3634 } 3635 3636 /// ditto 3637 @property string encoding() 3638 { 3639 return p.encoding; 3640 } 3641 3642 /** 3643 The content length in bytes of the ftp data. 3644 */ 3645 @property void contentLength(ulong len) 3646 { 3647 import std.conv : to; 3648 p.curl.set(CurlOption.infilesize_large, to!curl_off_t(len)); 3649 } 3650 3651 /** 3652 * Get various timings defined in $(REF CurlInfo, etc, c, curl). 3653 * The value is usable only if the return value is equal to `etc.c.curl.CurlError.ok`. 3654 * 3655 * Params: 3656 * timing = one of the timings defined in $(REF CurlInfo, etc, c, curl). 3657 * The values are: 3658 * `etc.c.curl.CurlInfo.namelookup_time`, 3659 * `etc.c.curl.CurlInfo.connect_time`, 3660 * `etc.c.curl.CurlInfo.pretransfer_time`, 3661 * `etc.c.curl.CurlInfo.starttransfer_time`, 3662 * `etc.c.curl.CurlInfo.redirect_time`, 3663 * `etc.c.curl.CurlInfo.appconnect_time`, 3664 * `etc.c.curl.CurlInfo.total_time`. 3665 * val = the actual value of the inquired timing. 3666 * 3667 * Returns: 3668 * The return code of the operation. The value stored in val 3669 * should be used only if the return value is `etc.c.curl.CurlInfo.ok`. 3670 * 3671 * Example: 3672 * --- 3673 * import std.net.curl; 3674 * import etc.c.curl : CurlError, CurlInfo; 3675 * 3676 * auto client = FTP(); 3677 * client.addCommand("RNFR my_file.txt"); 3678 * client.addCommand("RNTO my_renamed_file.txt"); 3679 * upload("my_file.txt", "ftp.digitalmars.com", client); 3680 * 3681 * double val; 3682 * CurlCode code; 3683 * 3684 * code = client.getTiming(CurlInfo.namelookup_time, val); 3685 * assert(code == CurlError.ok); 3686 * --- 3687 */ 3688 CurlCode getTiming(CurlInfo timing, ref double val) 3689 { 3690 return p.curl.getTiming(timing, val); 3691 } 3692 3693 @system unittest 3694 { 3695 auto client = FTP(); 3696 3697 double val; 3698 CurlCode code; 3699 3700 code = client.getTiming(CurlInfo.total_time, val); 3701 assert(code == CurlError.ok); 3702 code = client.getTiming(CurlInfo.namelookup_time, val); 3703 assert(code == CurlError.ok); 3704 code = client.getTiming(CurlInfo.connect_time, val); 3705 assert(code == CurlError.ok); 3706 code = client.getTiming(CurlInfo.pretransfer_time, val); 3707 assert(code == CurlError.ok); 3708 code = client.getTiming(CurlInfo.starttransfer_time, val); 3709 assert(code == CurlError.ok); 3710 code = client.getTiming(CurlInfo.redirect_time, val); 3711 assert(code == CurlError.ok); 3712 code = client.getTiming(CurlInfo.appconnect_time, val); 3713 assert(code == CurlError.ok); 3714 } 3715 } 3716 3717 /** 3718 * Basic SMTP protocol support. 3719 * 3720 * Example: 3721 * --- 3722 * import std.net.curl; 3723 * 3724 * // Send an email with SMTPS 3725 * auto smtp = SMTP("smtps://smtp.gmail.com"); 3726 * smtp.setAuthentication("from.addr@gmail.com", "password"); 3727 * smtp.mailTo = ["<to.addr@gmail.com>"]; 3728 * smtp.mailFrom = "<from.addr@gmail.com>"; 3729 * smtp.message = "Example Message"; 3730 * smtp.perform(); 3731 * --- 3732 * 3733 * See_Also: $(HTTP www.ietf.org/rfc/rfc2821.txt, RFC2821) 3734 */ 3735 struct SMTP 3736 { 3737 mixin Protocol; 3738 import std.typecons : RefCounted; 3739 import etc.c.curl : CurlUseSSL, curl_slist; 3740 3741 private struct Impl 3742 { 3743 ~this() 3744 { 3745 if (curl.handle !is null) // work around RefCounted/emplace bug 3746 curl.shutdown(); 3747 } 3748 Curl curl; 3749 3750 @property void message(string msg) 3751 { 3752 import std.algorithm.comparison : min; 3753 3754 auto _message = msg; 3755 /** 3756 This delegate reads the message text and copies it. 3757 */ 3758 curl.onSend = delegate size_t(void[] data) 3759 { 3760 if (!msg.length) return 0; 3761 size_t to_copy = min(data.length, _message.length); 3762 data[0 .. to_copy] = (cast(void[])_message)[0 .. to_copy]; 3763 _message = _message[to_copy..$]; 3764 return to_copy; 3765 }; 3766 } 3767 } 3768 3769 private RefCounted!Impl p; 3770 3771 /** 3772 Sets to the URL of the SMTP server. 3773 */ 3774 static SMTP opCall(const(char)[] url) 3775 { 3776 SMTP smtp; 3777 smtp.initialize(); 3778 smtp.url = url; 3779 return smtp; 3780 } 3781 3782 /// 3783 static SMTP opCall() 3784 { 3785 SMTP smtp; 3786 smtp.initialize(); 3787 return smtp; 3788 } 3789 3790 /+ TODO: The other structs have this function. 3791 SMTP dup() 3792 { 3793 SMTP copy = SMTP(); 3794 copy.initialize(); 3795 copy.p.encoding = p.encoding; 3796 copy.p.curl = p.curl.dup(); 3797 curl_slist* cur = p.commands; 3798 curl_slist* newlist = null; 3799 while (cur) 3800 { 3801 newlist = Curl.curl.slist_append(newlist, cur.data); 3802 cur = cur.next; 3803 } 3804 copy.p.commands = newlist; 3805 copy.p.curl.set(CurlOption.postquote, copy.p.commands); 3806 copy.dataTimeout = _defaultDataTimeout; 3807 return copy; 3808 } 3809 +/ 3810 3811 /** 3812 Performs the request as configured. 3813 Params: 3814 throwOnError = whether to throw an exception or return a CurlCode on error 3815 */ 3816 CurlCode perform(ThrowOnError throwOnError = Yes.throwOnError) 3817 { 3818 return p.curl.perform(throwOnError); 3819 } 3820 3821 /// The URL to specify the location of the resource. 3822 @property void url(const(char)[] url) 3823 { 3824 import std.algorithm.searching : startsWith; 3825 import std.exception : enforce; 3826 import std.uni : toLower; 3827 3828 auto lowered = url.toLower(); 3829 3830 if (lowered.startsWith("smtps://")) 3831 { 3832 p.curl.set(CurlOption.use_ssl, CurlUseSSL.all); 3833 } 3834 else 3835 { 3836 enforce!CurlException(lowered.startsWith("smtp://"), 3837 "The url must be for the smtp protocol."); 3838 } 3839 p.curl.set(CurlOption.url, url); 3840 } 3841 3842 private void initialize() 3843 { 3844 p.curl.initialize(); 3845 p.curl.set(CurlOption.upload, 1L); 3846 dataTimeout = _defaultDataTimeout; 3847 verifyPeer = true; 3848 verifyHost = true; 3849 } 3850 3851 // This is a workaround for mixed in content not having its 3852 // docs mixed in. 3853 version (StdDdoc) 3854 { 3855 static import etc.c.curl; 3856 3857 /// Value to return from `onSend`/`onReceive` delegates in order to 3858 /// pause a request 3859 alias requestPause = CurlReadFunc.pause; 3860 3861 /// Value to return from onSend delegate in order to abort a request 3862 alias requestAbort = CurlReadFunc.abort; 3863 3864 /** 3865 True if the instance is stopped. A stopped instance is not usable. 3866 */ 3867 @property bool isStopped(); 3868 3869 /// Stop and invalidate this instance. 3870 void shutdown(); 3871 3872 /** Set verbose. 3873 This will print request information to stderr. 3874 */ 3875 @property void verbose(bool on); 3876 3877 // Connection settings 3878 3879 /// Set timeout for activity on connection. 3880 @property void dataTimeout(Duration d); 3881 3882 /** Set maximum time an operation is allowed to take. 3883 This includes dns resolution, connecting, data transfer, etc. 3884 */ 3885 @property void operationTimeout(Duration d); 3886 3887 /// Set timeout for connecting. 3888 @property void connectTimeout(Duration d); 3889 3890 // Network settings 3891 3892 /** Proxy 3893 * See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXY, _proxy) 3894 */ 3895 @property void proxy(const(char)[] host); 3896 3897 /** Proxy port 3898 * See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXYPORT, _proxy_port) 3899 */ 3900 @property void proxyPort(ushort port); 3901 3902 /// Type of proxy 3903 alias CurlProxy = etc.c.curl.CurlProxy; 3904 3905 /** Proxy type 3906 * See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTPROXY, _proxy_type) 3907 */ 3908 @property void proxyType(CurlProxy type); 3909 3910 /// DNS lookup timeout. 3911 @property void dnsTimeout(Duration d); 3912 3913 /** 3914 * The network interface to use in form of the IP of the interface. 3915 * 3916 * Example: 3917 * ---- 3918 * theprotocol.netInterface = "192.168.1.32"; 3919 * theprotocol.netInterface = [ 192, 168, 1, 32 ]; 3920 * ---- 3921 * 3922 * See: $(REF InternetAddress, std,socket) 3923 */ 3924 @property void netInterface(const(char)[] i); 3925 3926 /// ditto 3927 @property void netInterface(const(ubyte)[4] i); 3928 3929 /// ditto 3930 @property void netInterface(InternetAddress i); 3931 3932 /** 3933 Set the local outgoing port to use. 3934 Params: 3935 port = the first outgoing port number to try and use 3936 */ 3937 @property void localPort(ushort port); 3938 3939 /** 3940 Set the local outgoing port range to use. 3941 This can be used together with the localPort property. 3942 Params: 3943 range = if the first port is occupied then try this many 3944 port number forwards 3945 */ 3946 @property void localPortRange(ushort range); 3947 3948 /** Set the tcp no-delay socket option on or off. 3949 See: $(HTTP curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTTCPNODELAY, nodelay) 3950 */ 3951 @property void tcpNoDelay(bool on); 3952 3953 // Authentication settings 3954 3955 /** 3956 Set the user name, password and optionally domain for authentication 3957 purposes. 3958 3959 Some protocols may need authentication in some cases. Use this 3960 function to provide credentials. 3961 3962 Params: 3963 username = the username 3964 password = the password 3965 domain = used for NTLM authentication only and is set to the NTLM domain 3966 name 3967 */ 3968 void setAuthentication(const(char)[] username, const(char)[] password, 3969 const(char)[] domain = ""); 3970 3971 /** 3972 Set the user name and password for proxy authentication. 3973 3974 Params: 3975 username = the username 3976 password = the password 3977 */ 3978 void setProxyAuthentication(const(char)[] username, const(char)[] password); 3979 3980 /** 3981 * The event handler that gets called when data is needed for sending. The 3982 * length of the `void[]` specifies the maximum number of bytes that can 3983 * be sent. 3984 * 3985 * Returns: 3986 * The callback returns the number of elements in the buffer that have been 3987 * filled and are ready to send. 3988 * The special value `.abortRequest` can be returned in order to abort the 3989 * current request. 3990 * The special value `.pauseRequest` can be returned in order to pause the 3991 * current request. 3992 */ 3993 @property void onSend(size_t delegate(void[]) callback); 3994 3995 /** 3996 * The event handler that receives incoming data. Be sure to copy the 3997 * incoming ubyte[] since it is not guaranteed to be valid after the 3998 * callback returns. 3999 * 4000 * Returns: 4001 * The callback returns the incoming bytes read. If not the entire array is 4002 * the request will abort. 4003 * The special value .pauseRequest can be returned in order to pause the 4004 * current request. 4005 */ 4006 @property void onReceive(size_t delegate(ubyte[]) callback); 4007 4008 /** 4009 * The event handler that gets called to inform of upload/download progress. 4010 * 4011 * Callback_parameters: 4012 * $(CALLBACK_PARAMS) 4013 * 4014 * Callback_returns: 4015 * Return 0 from the callback to signal success, return non-zero to 4016 * abort transfer. 4017 */ 4018 @property void onProgress(int delegate(size_t dlTotal, size_t dlNow, 4019 size_t ulTotal, size_t ulNow) callback); 4020 } 4021 4022 /** 4023 Setter for the sender's email address. 4024 */ 4025 @property void mailFrom()(const(char)[] sender) 4026 { 4027 assert(!sender.empty, "Sender must not be empty"); 4028 p.curl.set(CurlOption.mail_from, sender); 4029 } 4030 4031 /** 4032 Setter for the recipient email addresses. 4033 */ 4034 void mailTo()(const(char)[][] recipients...) 4035 { 4036 import std.internal.cstring : tempCString; 4037 assert(!recipients.empty, "Recipient must not be empty"); 4038 curl_slist* recipients_list = null; 4039 foreach (recipient; recipients) 4040 { 4041 recipients_list = 4042 Curl.curl.slist_append(recipients_list, 4043 recipient.tempCString().buffPtr); 4044 } 4045 p.curl.set(CurlOption.mail_rcpt, recipients_list); 4046 } 4047 4048 /** 4049 Sets the message body text. 4050 */ 4051 4052 @property void message(string msg) 4053 { 4054 p.message = msg; 4055 } 4056 } 4057 4058 @system unittest 4059 { 4060 import std.net.curl; 4061 4062 // Send an email with SMTPS 4063 auto smtp = SMTP("smtps://smtp.gmail.com"); 4064 smtp.setAuthentication("from.addr@gmail.com", "password"); 4065 smtp.mailTo = ["<to.addr@gmail.com>"]; 4066 smtp.mailFrom = "<from.addr@gmail.com>"; 4067 smtp.message = "Example Message"; 4068 //smtp.perform(); 4069 } 4070 4071 4072 /++ 4073 Exception thrown on errors in std.net.curl functions. 4074 +/ 4075 class CurlException : Exception 4076 { 4077 /++ 4078 Params: 4079 msg = The message for the exception. 4080 file = The file where the exception occurred. 4081 line = The line number where the exception occurred. 4082 next = The previous exception in the chain of exceptions, if any. 4083 +/ 4084 @safe pure nothrow 4085 this(string msg, 4086 string file = __FILE__, 4087 size_t line = __LINE__, 4088 Throwable next = null) 4089 { 4090 super(msg, file, line, next); 4091 } 4092 } 4093 4094 /++ 4095 Exception thrown on timeout errors in std.net.curl functions. 4096 +/ 4097 class CurlTimeoutException : CurlException 4098 { 4099 /++ 4100 Params: 4101 msg = The message for the exception. 4102 file = The file where the exception occurred. 4103 line = The line number where the exception occurred. 4104 next = The previous exception in the chain of exceptions, if any. 4105 +/ 4106 @safe pure nothrow 4107 this(string msg, 4108 string file = __FILE__, 4109 size_t line = __LINE__, 4110 Throwable next = null) 4111 { 4112 super(msg, file, line, next); 4113 } 4114 } 4115 4116 /++ 4117 Exception thrown on HTTP request failures, e.g. 404 Not Found. 4118 +/ 4119 class HTTPStatusException : CurlException 4120 { 4121 /++ 4122 Params: 4123 status = The HTTP status code. 4124 msg = The message for the exception. 4125 file = The file where the exception occurred. 4126 line = The line number where the exception occurred. 4127 next = The previous exception in the chain of exceptions, if any. 4128 +/ 4129 @safe pure nothrow 4130 this(int status, 4131 string msg, 4132 string file = __FILE__, 4133 size_t line = __LINE__, 4134 Throwable next = null) 4135 { 4136 super(msg, file, line, next); 4137 this.status = status; 4138 } 4139 4140 immutable int status; /// The HTTP status code 4141 } 4142 4143 /// Equal to $(REF CURLcode, etc,c,curl) 4144 alias CurlCode = CURLcode; 4145 4146 /// Flag to specify whether or not an exception is thrown on error. 4147 alias ThrowOnError = Flag!"throwOnError"; 4148 4149 private struct CurlAPI 4150 { 4151 import etc.c.curl : CurlGlobal; 4152 static struct API 4153 { 4154 import etc.c.curl : curl_version_info, curl_version_info_data, 4155 CURL, CURLcode, CURLINFO, CURLoption, CURLversion, curl_slist; 4156 extern(C): 4157 import core.stdc.config : c_long; 4158 CURLcode function(c_long flags) global_init; 4159 void function() global_cleanup; 4160 curl_version_info_data * function(CURLversion) version_info; 4161 CURL* function() easy_init; 4162 CURLcode function(CURL *curl, CURLoption option,...) easy_setopt; 4163 CURLcode function(CURL *curl) easy_perform; 4164 CURLcode function(CURL *curl, CURLINFO info,...) easy_getinfo; 4165 CURL* function(CURL *curl) easy_duphandle; 4166 char* function(CURLcode) easy_strerror; 4167 CURLcode function(CURL *handle, int bitmask) easy_pause; 4168 void function(CURL *curl) easy_cleanup; 4169 curl_slist* function(curl_slist *, char *) slist_append; 4170 void function(curl_slist *) slist_free_all; 4171 } 4172 __gshared API _api; 4173 __gshared void* _handle; 4174 4175 static ref API instance() @property 4176 { 4177 import std.concurrency : initOnce; 4178 initOnce!_handle(loadAPI()); 4179 return _api; 4180 } 4181 4182 static void* loadAPI() 4183 { 4184 import std.exception : enforce; 4185 4186 version (Posix) 4187 { 4188 import core.sys.posix.dlfcn : dlsym, dlopen, dlclose, RTLD_LAZY; 4189 alias loadSym = dlsym; 4190 } 4191 else version (Windows) 4192 { 4193 import core.sys.windows.winbase : GetProcAddress, GetModuleHandleA, 4194 LoadLibraryA; 4195 alias loadSym = GetProcAddress; 4196 } 4197 else 4198 static assert(0, "unimplemented"); 4199 4200 void* handle; 4201 version (Posix) 4202 handle = dlopen(null, RTLD_LAZY); 4203 else version (Windows) 4204 handle = GetModuleHandleA(null); 4205 assert(handle !is null); 4206 4207 // try to load curl from the executable to allow static linking 4208 if (loadSym(handle, "curl_global_init") is null) 4209 { 4210 import std.format : format; 4211 version (Posix) 4212 dlclose(handle); 4213 4214 version (LibcurlPath) 4215 { 4216 import std.string : strip; 4217 static immutable names = [strip(import("LibcurlPathFile"))]; 4218 } 4219 else version (OSX) 4220 static immutable names = ["libcurl.4.dylib"]; 4221 else version (Posix) 4222 { 4223 static immutable names = ["libcurl.so", "libcurl.so.4", 4224 "libcurl-gnutls.so.4", "libcurl-nss.so.4", "libcurl.so.3"]; 4225 } 4226 else version (Windows) 4227 static immutable names = ["libcurl.dll", "curl.dll"]; 4228 4229 foreach (name; names) 4230 { 4231 version (Posix) 4232 handle = dlopen(name.ptr, RTLD_LAZY); 4233 else version (Windows) 4234 handle = LoadLibraryA(name.ptr); 4235 if (handle !is null) break; 4236 } 4237 4238 enforce!CurlException(handle !is null, "Failed to load curl, tried %(%s, %).".format(names)); 4239 } 4240 4241 foreach (i, FP; typeof(API.tupleof)) 4242 { 4243 enum name = __traits(identifier, _api.tupleof[i]); 4244 auto p = enforce!CurlException(loadSym(handle, "curl_"~name), 4245 "Couldn't load curl_"~name~" from libcurl."); 4246 _api.tupleof[i] = cast(FP) p; 4247 } 4248 4249 enforce!CurlException(!_api.global_init(CurlGlobal.all), 4250 "Failed to initialize libcurl"); 4251 4252 static extern(C) void cleanup() 4253 { 4254 if (_handle is null) return; 4255 _api.global_cleanup(); 4256 version (Posix) 4257 { 4258 import core.sys.posix.dlfcn : dlclose; 4259 dlclose(_handle); 4260 } 4261 else version (Windows) 4262 { 4263 import core.sys.windows.winbase : FreeLibrary; 4264 FreeLibrary(_handle); 4265 } 4266 else 4267 static assert(0, "unimplemented"); 4268 _api = API.init; 4269 _handle = null; 4270 } 4271 4272 import core.stdc.stdlib : atexit; 4273 atexit(&cleanup); 4274 4275 return handle; 4276 } 4277 } 4278 4279 /** 4280 Wrapper to provide a better interface to libcurl than using the plain C API. 4281 It is recommended to use the `HTTP`/`FTP` etc. structs instead unless 4282 raw access to libcurl is needed. 4283 4284 Warning: This struct uses interior pointers for callbacks. Only allocate it 4285 on the stack if you never move or copy it. This also means passing by reference 4286 when passing Curl to other functions. Otherwise always allocate on 4287 the heap. 4288 */ 4289 struct Curl 4290 { 4291 import etc.c.curl : CURL, CurlError, CurlPause, CurlSeek, CurlSeekPos, 4292 curl_socket_t, CurlSockType, 4293 CurlReadFunc, CurlInfo, curlsocktype, curl_off_t, 4294 LIBCURL_VERSION_MAJOR, LIBCURL_VERSION_MINOR, LIBCURL_VERSION_PATCH; 4295 4296 alias OutData = void[]; 4297 alias InData = ubyte[]; 4298 private bool _stopped; 4299 4300 private static auto ref curl() @property { return CurlAPI.instance; } 4301 4302 // A handle should not be used by two threads simultaneously 4303 private CURL* handle; 4304 4305 // May also return `CURL_READFUNC_ABORT` or `CURL_READFUNC_PAUSE` 4306 private size_t delegate(OutData) _onSend; 4307 private size_t delegate(InData) _onReceive; 4308 private void delegate(in char[]) _onReceiveHeader; 4309 private CurlSeek delegate(long,CurlSeekPos) _onSeek; 4310 private int delegate(curl_socket_t,CurlSockType) _onSocketOption; 4311 private int delegate(size_t dltotal, size_t dlnow, 4312 size_t ultotal, size_t ulnow) _onProgress; 4313 4314 alias requestPause = CurlReadFunc.pause; 4315 alias requestAbort = CurlReadFunc.abort; 4316 4317 /** 4318 Initialize the instance by creating a working curl handle. 4319 */ 4320 void initialize() 4321 { 4322 import std.exception : enforce; 4323 enforce!CurlException(!handle, "Curl instance already initialized"); 4324 handle = curl.easy_init(); 4325 enforce!CurlException(handle, "Curl instance couldn't be initialized"); 4326 _stopped = false; 4327 set(CurlOption.nosignal, 1); 4328 } 4329 4330 /// 4331 @property bool stopped() const 4332 { 4333 return _stopped; 4334 } 4335 4336 /** 4337 Duplicate this handle. 4338 4339 The new handle will have all options set as the one it was duplicated 4340 from. An exception to this is that all options that cannot be shared 4341 across threads are reset thereby making it safe to use the duplicate 4342 in a new thread. 4343 */ 4344 Curl dup() 4345 { 4346 import std.meta : AliasSeq; 4347 Curl copy; 4348 copy.handle = curl.easy_duphandle(handle); 4349 copy._stopped = false; 4350 4351 with (CurlOption) { 4352 auto tt = AliasSeq!(file, writefunction, writeheader, 4353 headerfunction, infile, readfunction, ioctldata, ioctlfunction, 4354 seekdata, seekfunction, sockoptdata, sockoptfunction, 4355 opensocketdata, opensocketfunction, progressdata, 4356 progressfunction, debugdata, debugfunction, interleavedata, 4357 interleavefunction, chunk_data, chunk_bgn_function, 4358 chunk_end_function, fnmatch_data, fnmatch_function, cookiejar, postfields); 4359 4360 foreach (option; tt) 4361 copy.clear(option); 4362 } 4363 4364 // The options are only supported by libcurl when it has been built 4365 // against certain versions of OpenSSL - if your libcurl uses an old 4366 // OpenSSL, or uses an entirely different SSL engine, attempting to 4367 // clear these normally will raise an exception 4368 copy.clearIfSupported(CurlOption.ssl_ctx_function); 4369 copy.clearIfSupported(CurlOption.ssh_keydata); 4370 4371 // Enable for curl version > 7.21.7 4372 static if (LIBCURL_VERSION_MAJOR >= 7 && 4373 LIBCURL_VERSION_MINOR >= 21 && 4374 LIBCURL_VERSION_PATCH >= 7) 4375 { 4376 copy.clear(CurlOption.closesocketdata); 4377 copy.clear(CurlOption.closesocketfunction); 4378 } 4379 4380 copy.set(CurlOption.nosignal, 1); 4381 4382 // copy.clear(CurlOption.ssl_ctx_data); Let ssl function be shared 4383 // copy.clear(CurlOption.ssh_keyfunction); Let key function be shared 4384 4385 /* 4386 Allow sharing of conv functions 4387 copy.clear(CurlOption.conv_to_network_function); 4388 copy.clear(CurlOption.conv_from_network_function); 4389 copy.clear(CurlOption.conv_from_utf8_function); 4390 */ 4391 4392 return copy; 4393 } 4394 4395 private void _check(CurlCode code) 4396 { 4397 import std.exception : enforce; 4398 enforce!CurlTimeoutException(code != CurlError.operation_timedout, 4399 errorString(code)); 4400 4401 enforce!CurlException(code == CurlError.ok, 4402 errorString(code)); 4403 } 4404 4405 private string errorString(CurlCode code) 4406 { 4407 import core.stdc.string : strlen; 4408 import std.format : format; 4409 4410 auto msgZ = curl.easy_strerror(code); 4411 // doing the following (instead of just using std.conv.to!string) avoids 1 allocation 4412 return format("%s on handle %s", msgZ[0 .. strlen(msgZ)], handle); 4413 } 4414 4415 private void throwOnStopped(string message = null) 4416 { 4417 import std.exception : enforce; 4418 auto def = "Curl instance called after being cleaned up"; 4419 enforce!CurlException(!stopped, 4420 message == null ? def : message); 4421 } 4422 4423 /** 4424 Stop and invalidate this curl instance. 4425 Warning: Do not call this from inside a callback handler e.g. `onReceive`. 4426 */ 4427 void shutdown() 4428 { 4429 throwOnStopped(); 4430 _stopped = true; 4431 curl.easy_cleanup(this.handle); 4432 this.handle = null; 4433 } 4434 4435 /** 4436 Pausing and continuing transfers. 4437 */ 4438 void pause(bool sendingPaused, bool receivingPaused) 4439 { 4440 throwOnStopped(); 4441 _check(curl.easy_pause(this.handle, 4442 (sendingPaused ? CurlPause.send_cont : CurlPause.send) | 4443 (receivingPaused ? CurlPause.recv_cont : CurlPause.recv))); 4444 } 4445 4446 /** 4447 Set a string curl option. 4448 Params: 4449 option = A $(REF CurlOption, etc,c,curl) as found in the curl documentation 4450 value = The string 4451 */ 4452 void set(CurlOption option, const(char)[] value) 4453 { 4454 import std.internal.cstring : tempCString; 4455 throwOnStopped(); 4456 _check(curl.easy_setopt(this.handle, option, value.tempCString().buffPtr)); 4457 } 4458 4459 /** 4460 Set a long curl option. 4461 Params: 4462 option = A $(REF CurlOption, etc,c,curl) as found in the curl documentation 4463 value = The long 4464 */ 4465 void set(CurlOption option, long value) 4466 { 4467 throwOnStopped(); 4468 _check(curl.easy_setopt(this.handle, option, value)); 4469 } 4470 4471 /** 4472 Set a void* curl option. 4473 Params: 4474 option = A $(REF CurlOption, etc,c,curl) as found in the curl documentation 4475 value = The pointer 4476 */ 4477 void set(CurlOption option, void* value) 4478 { 4479 throwOnStopped(); 4480 _check(curl.easy_setopt(this.handle, option, value)); 4481 } 4482 4483 /** 4484 Clear a pointer option. 4485 Params: 4486 option = A $(REF CurlOption, etc,c,curl) as found in the curl documentation 4487 */ 4488 void clear(CurlOption option) 4489 { 4490 throwOnStopped(); 4491 _check(curl.easy_setopt(this.handle, option, null)); 4492 } 4493 4494 /** 4495 Clear a pointer option. Does not raise an exception if the underlying 4496 libcurl does not support the option. Use sparingly. 4497 Params: 4498 option = A $(REF CurlOption, etc,c,curl) as found in the curl documentation 4499 */ 4500 void clearIfSupported(CurlOption option) 4501 { 4502 throwOnStopped(); 4503 auto rval = curl.easy_setopt(this.handle, option, null); 4504 if (rval != CurlError.unknown_option && rval != CurlError.not_built_in) 4505 _check(rval); 4506 } 4507 4508 /** 4509 perform the curl request by doing the HTTP,FTP etc. as it has 4510 been setup beforehand. 4511 4512 Params: 4513 throwOnError = whether to throw an exception or return a CurlCode on error 4514 */ 4515 CurlCode perform(ThrowOnError throwOnError = Yes.throwOnError) 4516 { 4517 throwOnStopped(); 4518 CurlCode code = curl.easy_perform(this.handle); 4519 if (throwOnError) 4520 _check(code); 4521 return code; 4522 } 4523 4524 /** 4525 Get the various timings like name lookup time, total time, connect time etc. 4526 The timed category is passed through the timing parameter while the timing 4527 value is stored at val. The value is usable only if res is equal to 4528 `etc.c.curl.CurlError.ok`. 4529 */ 4530 CurlCode getTiming(CurlInfo timing, ref double val) 4531 { 4532 CurlCode code; 4533 code = curl.easy_getinfo(handle, timing, &val); 4534 return code; 4535 } 4536 4537 /** 4538 * The event handler that receives incoming data. 4539 * 4540 * Params: 4541 * callback = the callback that receives the `ubyte[]` data. 4542 * Be sure to copy the incoming data and not store 4543 * a slice. 4544 * 4545 * Returns: 4546 * The callback returns the incoming bytes read. If not the entire array is 4547 * the request will abort. 4548 * The special value HTTP.pauseRequest can be returned in order to pause the 4549 * current request. 4550 * 4551 * Example: 4552 * ---- 4553 * import std.net.curl, std.stdio, std.conv; 4554 * Curl curl; 4555 * curl.initialize(); 4556 * curl.set(CurlOption.url, "http://dlang.org"); 4557 * curl.onReceive = (ubyte[] data) { writeln("Got data", to!(const(char)[])(data)); return data.length;}; 4558 * curl.perform(); 4559 * ---- 4560 */ 4561 @property void onReceive(size_t delegate(InData) callback) 4562 { 4563 _onReceive = (InData id) 4564 { 4565 throwOnStopped("Receive callback called on cleaned up Curl instance"); 4566 return callback(id); 4567 }; 4568 set(CurlOption.file, cast(void*) &this); 4569 set(CurlOption.writefunction, cast(void*) &Curl._receiveCallback); 4570 } 4571 4572 /** 4573 * The event handler that receives incoming headers for protocols 4574 * that uses headers. 4575 * 4576 * Params: 4577 * callback = the callback that receives the header string. 4578 * Make sure the callback copies the incoming params if 4579 * it needs to store it because they are references into 4580 * the backend and may very likely change. 4581 * 4582 * Example: 4583 * ---- 4584 * import std.net.curl, std.stdio; 4585 * Curl curl; 4586 * curl.initialize(); 4587 * curl.set(CurlOption.url, "http://dlang.org"); 4588 * curl.onReceiveHeader = (in char[] header) { writeln(header); }; 4589 * curl.perform(); 4590 * ---- 4591 */ 4592 @property void onReceiveHeader(void delegate(in char[]) callback) 4593 { 4594 _onReceiveHeader = (in char[] od) 4595 { 4596 throwOnStopped("Receive header callback called on "~ 4597 "cleaned up Curl instance"); 4598 callback(od); 4599 }; 4600 set(CurlOption.writeheader, cast(void*) &this); 4601 set(CurlOption.headerfunction, 4602 cast(void*) &Curl._receiveHeaderCallback); 4603 } 4604 4605 /** 4606 * The event handler that gets called when data is needed for sending. 4607 * 4608 * Params: 4609 * callback = the callback that has a `void[]` buffer to be filled 4610 * 4611 * Returns: 4612 * The callback returns the number of elements in the buffer that have been 4613 * filled and are ready to send. 4614 * The special value `Curl.abortRequest` can be returned in 4615 * order to abort the current request. 4616 * The special value `Curl.pauseRequest` can be returned in order to 4617 * pause the current request. 4618 * 4619 * Example: 4620 * ---- 4621 * import std.net.curl; 4622 * Curl curl; 4623 * curl.initialize(); 4624 * curl.set(CurlOption.url, "http://dlang.org"); 4625 * 4626 * string msg = "Hello world"; 4627 * curl.onSend = (void[] data) 4628 * { 4629 * auto m = cast(void[]) msg; 4630 * size_t length = m.length > data.length ? data.length : m.length; 4631 * if (length == 0) return 0; 4632 * data[0 .. length] = m[0 .. length]; 4633 * msg = msg[length..$]; 4634 * return length; 4635 * }; 4636 * curl.perform(); 4637 * ---- 4638 */ 4639 @property void onSend(size_t delegate(OutData) callback) 4640 { 4641 _onSend = (OutData od) 4642 { 4643 throwOnStopped("Send callback called on cleaned up Curl instance"); 4644 return callback(od); 4645 }; 4646 set(CurlOption.infile, cast(void*) &this); 4647 set(CurlOption.readfunction, cast(void*) &Curl._sendCallback); 4648 } 4649 4650 /** 4651 * The event handler that gets called when the curl backend needs to seek 4652 * the data to be sent. 4653 * 4654 * Params: 4655 * callback = the callback that receives a seek offset and a seek position 4656 * $(REF CurlSeekPos, etc,c,curl) 4657 * 4658 * Returns: 4659 * The callback returns the success state of the seeking 4660 * $(REF CurlSeek, etc,c,curl) 4661 * 4662 * Example: 4663 * ---- 4664 * import std.net.curl; 4665 * Curl curl; 4666 * curl.initialize(); 4667 * curl.set(CurlOption.url, "http://dlang.org"); 4668 * curl.onSeek = (long p, CurlSeekPos sp) 4669 * { 4670 * return CurlSeek.cantseek; 4671 * }; 4672 * curl.perform(); 4673 * ---- 4674 */ 4675 @property void onSeek(CurlSeek delegate(long, CurlSeekPos) callback) 4676 { 4677 _onSeek = (long ofs, CurlSeekPos sp) 4678 { 4679 throwOnStopped("Seek callback called on cleaned up Curl instance"); 4680 return callback(ofs, sp); 4681 }; 4682 set(CurlOption.seekdata, cast(void*) &this); 4683 set(CurlOption.seekfunction, cast(void*) &Curl._seekCallback); 4684 } 4685 4686 /** 4687 * The event handler that gets called when the net socket has been created 4688 * but a `connect()` call has not yet been done. This makes it possible to set 4689 * misc. socket options. 4690 * 4691 * Params: 4692 * callback = the callback that receives the socket and socket type 4693 * $(REF CurlSockType, etc,c,curl) 4694 * 4695 * Returns: 4696 * Return 0 from the callback to signal success, return 1 to signal error 4697 * and make curl close the socket 4698 * 4699 * Example: 4700 * ---- 4701 * import std.net.curl; 4702 * Curl curl; 4703 * curl.initialize(); 4704 * curl.set(CurlOption.url, "http://dlang.org"); 4705 * curl.onSocketOption = delegate int(curl_socket_t s, CurlSockType t) { /+ do stuff +/ }; 4706 * curl.perform(); 4707 * ---- 4708 */ 4709 @property void onSocketOption(int delegate(curl_socket_t, 4710 CurlSockType) callback) 4711 { 4712 _onSocketOption = (curl_socket_t sock, CurlSockType st) 4713 { 4714 throwOnStopped("Socket option callback called on "~ 4715 "cleaned up Curl instance"); 4716 return callback(sock, st); 4717 }; 4718 set(CurlOption.sockoptdata, cast(void*) &this); 4719 set(CurlOption.sockoptfunction, 4720 cast(void*) &Curl._socketOptionCallback); 4721 } 4722 4723 /** 4724 * The event handler that gets called to inform of upload/download progress. 4725 * 4726 * Params: 4727 * callback = the callback that receives the (total bytes to download, 4728 * currently downloaded bytes, total bytes to upload, currently uploaded 4729 * bytes). 4730 * 4731 * Returns: 4732 * Return 0 from the callback to signal success, return non-zero to abort 4733 * transfer 4734 * 4735 * Example: 4736 * ---- 4737 * import std.net.curl, std.stdio; 4738 * Curl curl; 4739 * curl.initialize(); 4740 * curl.set(CurlOption.url, "http://dlang.org"); 4741 * curl.onProgress = delegate int(size_t dltotal, size_t dlnow, size_t ultotal, size_t ulnow) 4742 * { 4743 * writeln("Progress: downloaded bytes ", dlnow, " of ", dltotal); 4744 * writeln("Progress: uploaded bytes ", ulnow, " of ", ultotal); 4745 * return 0; 4746 * }; 4747 * curl.perform(); 4748 * ---- 4749 */ 4750 @property void onProgress(int delegate(size_t dlTotal, 4751 size_t dlNow, 4752 size_t ulTotal, 4753 size_t ulNow) callback) 4754 { 4755 _onProgress = (size_t dlt, size_t dln, size_t ult, size_t uln) 4756 { 4757 throwOnStopped("Progress callback called on cleaned "~ 4758 "up Curl instance"); 4759 return callback(dlt, dln, ult, uln); 4760 }; 4761 set(CurlOption.noprogress, 0); 4762 set(CurlOption.progressdata, cast(void*) &this); 4763 set(CurlOption.progressfunction, cast(void*) &Curl._progressCallback); 4764 } 4765 4766 // Internal C callbacks to register with libcurl 4767 extern (C) private static 4768 size_t _receiveCallback(const char* str, 4769 size_t size, size_t nmemb, void* ptr) 4770 { 4771 auto b = cast(Curl*) ptr; 4772 if (b._onReceive != null) 4773 return b._onReceive(cast(InData)(str[0 .. size*nmemb])); 4774 return size*nmemb; 4775 } 4776 4777 extern (C) private static 4778 size_t _receiveHeaderCallback(const char* str, 4779 size_t size, size_t nmemb, void* ptr) 4780 { 4781 import std.string : chomp; 4782 4783 auto b = cast(Curl*) ptr; 4784 auto s = str[0 .. size*nmemb].chomp(); 4785 if (b._onReceiveHeader != null) 4786 b._onReceiveHeader(s); 4787 4788 return size*nmemb; 4789 } 4790 4791 extern (C) private static 4792 size_t _sendCallback(char *str, size_t size, size_t nmemb, void *ptr) 4793 { 4794 Curl* b = cast(Curl*) ptr; 4795 auto a = cast(void[]) str[0 .. size*nmemb]; 4796 if (b._onSend == null) 4797 return 0; 4798 return b._onSend(a); 4799 } 4800 4801 extern (C) private static 4802 int _seekCallback(void *ptr, curl_off_t offset, int origin) 4803 { 4804 auto b = cast(Curl*) ptr; 4805 if (b._onSeek == null) 4806 return CurlSeek.cantseek; 4807 4808 // origin: CurlSeekPos.set/current/end 4809 // return: CurlSeek.ok/fail/cantseek 4810 return b._onSeek(cast(long) offset, cast(CurlSeekPos) origin); 4811 } 4812 4813 extern (C) private static 4814 int _socketOptionCallback(void *ptr, 4815 curl_socket_t curlfd, curlsocktype purpose) 4816 { 4817 auto b = cast(Curl*) ptr; 4818 if (b._onSocketOption == null) 4819 return 0; 4820 4821 // return: 0 ok, 1 fail 4822 return b._onSocketOption(curlfd, cast(CurlSockType) purpose); 4823 } 4824 4825 extern (C) private static 4826 int _progressCallback(void *ptr, 4827 double dltotal, double dlnow, 4828 double ultotal, double ulnow) 4829 { 4830 auto b = cast(Curl*) ptr; 4831 if (b._onProgress == null) 4832 return 0; 4833 4834 // return: 0 ok, 1 fail 4835 return b._onProgress(cast(size_t) dltotal, cast(size_t) dlnow, 4836 cast(size_t) ultotal, cast(size_t) ulnow); 4837 } 4838 4839 } 4840 4841 // Internal messages send between threads. 4842 // The data is wrapped in this struct in order to ensure that 4843 // other std.concurrency.receive calls does not pick up our messages 4844 // by accident. 4845 private struct CurlMessage(T) 4846 { 4847 public T data; 4848 } 4849 4850 private static CurlMessage!T curlMessage(T)(T data) 4851 { 4852 return CurlMessage!T(data); 4853 } 4854 4855 // Pool of to be used for reusing buffers 4856 private struct Pool(Data) 4857 { 4858 private struct Entry 4859 { 4860 Data data; 4861 Entry* next; 4862 } 4863 private Entry* root; 4864 private Entry* freeList; 4865 4866 @safe @property bool empty() 4867 { 4868 return root == null; 4869 } 4870 4871 @safe nothrow void push(Data d) 4872 { 4873 if (freeList == null) 4874 { 4875 // Allocate new Entry since there is no one 4876 // available in the freeList 4877 freeList = new Entry; 4878 } 4879 freeList.data = d; 4880 Entry* oldroot = root; 4881 root = freeList; 4882 freeList = freeList.next; 4883 root.next = oldroot; 4884 } 4885 4886 @safe Data pop() 4887 { 4888 import std.exception : enforce; 4889 enforce!Exception(root != null, "pop() called on empty pool"); 4890 auto d = root.data; 4891 auto n = root.next; 4892 root.next = freeList; 4893 freeList = root; 4894 root = n; 4895 return d; 4896 } 4897 } 4898 4899 // Lazily-instantiated namespace to avoid importing std.concurrency until needed. 4900 private struct _async() 4901 { 4902 static: 4903 // https://issues.dlang.org/show_bug.cgi?id=15831 4904 // this should be inside byLineAsync 4905 // Range that reads one chunk at a time asynchronously. 4906 private struct ChunkInputRange 4907 { 4908 import std.concurrency : Tid, send; 4909 4910 private ubyte[] chunk; 4911 mixin WorkerThreadProtocol!(ubyte, chunk); 4912 4913 private Tid workerTid; 4914 private State running; 4915 4916 private this(Tid tid, size_t transmitBuffers, size_t chunkSize) 4917 { 4918 workerTid = tid; 4919 state = State.needUnits; 4920 4921 // Send buffers to other thread for it to use. Since no mechanism is in 4922 // place for moving ownership a cast to shared is done here and a cast 4923 // back to non-shared in the receiving end. 4924 foreach (i ; 0 .. transmitBuffers) 4925 { 4926 ubyte[] arr = new ubyte[](chunkSize); 4927 workerTid.send(cast(immutable(ubyte[]))arr); 4928 } 4929 } 4930 } 4931 4932 // https://issues.dlang.org/show_bug.cgi?id=15831 4933 // this should be inside byLineAsync 4934 // Range that reads one line at a time asynchronously. 4935 private static struct LineInputRange(Char) 4936 { 4937 private Char[] line; 4938 mixin WorkerThreadProtocol!(Char, line); 4939 4940 private Tid workerTid; 4941 private State running; 4942 4943 private this(Tid tid, size_t transmitBuffers, size_t bufferSize) 4944 { 4945 import std.concurrency : send; 4946 4947 workerTid = tid; 4948 state = State.needUnits; 4949 4950 // Send buffers to other thread for it to use. Since no mechanism is in 4951 // place for moving ownership a cast to shared is done here and casted 4952 // back to non-shared in the receiving end. 4953 foreach (i ; 0 .. transmitBuffers) 4954 { 4955 auto arr = new Char[](bufferSize); 4956 workerTid.send(cast(immutable(Char[]))arr); 4957 } 4958 } 4959 } 4960 4961 import std.concurrency : Tid; 4962 4963 // Shared function for reading incoming chunks of data and 4964 // sending the to a parent thread 4965 private size_t receiveChunks(ubyte[] data, ref ubyte[] outdata, 4966 Pool!(ubyte[]) freeBuffers, 4967 ref ubyte[] buffer, Tid fromTid, 4968 ref bool aborted) 4969 { 4970 import std.concurrency : receive, send, thisTid; 4971 4972 immutable datalen = data.length; 4973 4974 // Copy data to fill active buffer 4975 while (!data.empty) 4976 { 4977 4978 // Make sure a buffer is present 4979 while ( outdata.empty && freeBuffers.empty) 4980 { 4981 // Active buffer is invalid and there are no 4982 // available buffers in the pool. Wait for buffers 4983 // to return from main thread in order to reuse 4984 // them. 4985 receive((immutable(ubyte)[] buf) 4986 { 4987 buffer = cast(ubyte[]) buf; 4988 outdata = buffer[]; 4989 }, 4990 (bool flag) { aborted = true; } 4991 ); 4992 if (aborted) return cast(size_t) 0; 4993 } 4994 if (outdata.empty) 4995 { 4996 buffer = freeBuffers.pop(); 4997 outdata = buffer[]; 4998 } 4999 5000 // Copy data 5001 auto copyBytes = outdata.length < data.length ? 5002 outdata.length : data.length; 5003 5004 outdata[0 .. copyBytes] = data[0 .. copyBytes]; 5005 outdata = outdata[copyBytes..$]; 5006 data = data[copyBytes..$]; 5007 5008 if (outdata.empty) 5009 fromTid.send(thisTid, curlMessage(cast(immutable(ubyte)[])buffer)); 5010 } 5011 5012 return datalen; 5013 } 5014 5015 // ditto 5016 private void finalizeChunks(ubyte[] outdata, ref ubyte[] buffer, 5017 Tid fromTid) 5018 { 5019 import std.concurrency : send, thisTid; 5020 if (!outdata.empty) 5021 { 5022 // Resize the last buffer 5023 buffer.length = buffer.length - outdata.length; 5024 fromTid.send(thisTid, curlMessage(cast(immutable(ubyte)[])buffer)); 5025 } 5026 } 5027 5028 5029 // Shared function for reading incoming lines of data and sending the to a 5030 // parent thread 5031 private static size_t receiveLines(Terminator, Unit) 5032 (const(ubyte)[] data, ref EncodingScheme encodingScheme, 5033 bool keepTerminator, Terminator terminator, 5034 ref const(ubyte)[] leftOverBytes, ref bool bufferValid, 5035 ref Pool!(Unit[]) freeBuffers, ref Unit[] buffer, 5036 Tid fromTid, ref bool aborted) 5037 { 5038 import std.concurrency : prioritySend, receive, send, thisTid; 5039 import std.exception : enforce; 5040 import std.format : format; 5041 import std.traits : isArray; 5042 5043 immutable datalen = data.length; 5044 5045 // Terminator is specified and buffers should be resized as determined by 5046 // the terminator 5047 5048 // Copy data to active buffer until terminator is found. 5049 5050 // Decode as many lines as possible 5051 while (true) 5052 { 5053 5054 // Make sure a buffer is present 5055 while (!bufferValid && freeBuffers.empty) 5056 { 5057 // Active buffer is invalid and there are no available buffers in 5058 // the pool. Wait for buffers to return from main thread in order to 5059 // reuse them. 5060 receive((immutable(Unit)[] buf) 5061 { 5062 buffer = cast(Unit[]) buf; 5063 buffer.length = 0; 5064 buffer.assumeSafeAppend(); 5065 bufferValid = true; 5066 }, 5067 (bool flag) { aborted = true; } 5068 ); 5069 if (aborted) return cast(size_t) 0; 5070 } 5071 if (!bufferValid) 5072 { 5073 buffer = freeBuffers.pop(); 5074 bufferValid = true; 5075 } 5076 5077 // Try to read a line from left over bytes from last onReceive plus the 5078 // newly received bytes. 5079 try 5080 { 5081 if (decodeLineInto(leftOverBytes, data, buffer, 5082 encodingScheme, terminator)) 5083 { 5084 if (keepTerminator) 5085 { 5086 fromTid.send(thisTid, 5087 curlMessage(cast(immutable(Unit)[])buffer)); 5088 } 5089 else 5090 { 5091 static if (isArray!Terminator) 5092 fromTid.send(thisTid, 5093 curlMessage(cast(immutable(Unit)[]) 5094 buffer[0..$-terminator.length])); 5095 else 5096 fromTid.send(thisTid, 5097 curlMessage(cast(immutable(Unit)[]) 5098 buffer[0..$-1])); 5099 } 5100 bufferValid = false; 5101 } 5102 else 5103 { 5104 // Could not decode an entire line. Save 5105 // bytes left in data for next call to 5106 // onReceive. Can be up to a max of 4 bytes. 5107 enforce!CurlException(data.length <= 4, 5108 format( 5109 "Too many bytes left not decoded %s"~ 5110 " > 4. Maybe the charset specified in"~ 5111 " headers does not match "~ 5112 "the actual content downloaded?", 5113 data.length)); 5114 leftOverBytes ~= data; 5115 break; 5116 } 5117 } 5118 catch (CurlException ex) 5119 { 5120 prioritySend(fromTid, cast(immutable(CurlException))ex); 5121 return cast(size_t) 0; 5122 } 5123 } 5124 return datalen; 5125 } 5126 5127 // ditto 5128 private static 5129 void finalizeLines(Unit)(bool bufferValid, Unit[] buffer, Tid fromTid) 5130 { 5131 import std.concurrency : send, thisTid; 5132 if (bufferValid && buffer.length != 0) 5133 fromTid.send(thisTid, curlMessage(cast(immutable(Unit)[])buffer[0..$])); 5134 } 5135 5136 /* Used by byLineAsync/byChunkAsync to duplicate an existing connection 5137 * that can be used exclusively in a spawned thread. 5138 */ 5139 private void duplicateConnection(Conn, PostData) 5140 (const(char)[] url, Conn conn, PostData postData, Tid tid) 5141 { 5142 import std.concurrency : send; 5143 import std.exception : enforce; 5144 5145 // no move semantic available in std.concurrency ie. must use casting. 5146 auto connDup = conn.dup(); 5147 connDup.url = url; 5148 5149 static if ( is(Conn : HTTP) ) 5150 { 5151 connDup.p.headersOut = null; 5152 connDup.method = conn.method == HTTP.Method.undefined ? 5153 HTTP.Method.get : conn.method; 5154 if (postData !is null) 5155 { 5156 if (connDup.method == HTTP.Method.put) 5157 { 5158 connDup.handle.set(CurlOption.infilesize_large, 5159 postData.length); 5160 } 5161 else 5162 { 5163 // post 5164 connDup.method = HTTP.Method.post; 5165 connDup.handle.set(CurlOption.postfieldsize_large, 5166 postData.length); 5167 } 5168 connDup.handle.set(CurlOption.copypostfields, 5169 cast(void*) postData.ptr); 5170 } 5171 tid.send(cast(ulong) connDup.handle.handle); 5172 tid.send(connDup.method); 5173 } 5174 else 5175 { 5176 enforce!CurlException(postData is null, 5177 "Cannot put ftp data using byLineAsync()"); 5178 tid.send(cast(ulong) connDup.handle.handle); 5179 tid.send(HTTP.Method.undefined); 5180 } 5181 connDup.p.curl.handle = null; // make sure handle is not freed 5182 } 5183 5184 // Spawn a thread for handling the reading of incoming data in the 5185 // background while the delegate is executing. This will optimize 5186 // throughput by allowing simultaneous input (this struct) and 5187 // output (e.g. AsyncHTTPLineOutputRange). 5188 private static void spawn(Conn, Unit, Terminator = void)() 5189 { 5190 import std.concurrency : Tid, prioritySend, receiveOnly, send, thisTid; 5191 import etc.c.curl : CURL, CurlError; 5192 Tid fromTid = receiveOnly!Tid(); 5193 5194 // Get buffer to read into 5195 Pool!(Unit[]) freeBuffers; // Free list of buffer objects 5196 5197 // Number of bytes filled into active buffer 5198 Unit[] buffer; 5199 bool aborted = false; 5200 5201 EncodingScheme encodingScheme; 5202 static if ( !is(Terminator == void)) 5203 { 5204 // Only lines reading will receive a terminator 5205 const terminator = receiveOnly!Terminator(); 5206 const keepTerminator = receiveOnly!bool(); 5207 5208 // max number of bytes to carry over from an onReceive 5209 // callback. This is 4 because it is the max code units to 5210 // decode a code point in the supported encodings. 5211 auto leftOverBytes = new const(ubyte)[4]; 5212 leftOverBytes.length = 0; 5213 auto bufferValid = false; 5214 } 5215 else 5216 { 5217 Unit[] outdata; 5218 } 5219 5220 // no move semantic available in std.concurrency ie. must use casting. 5221 auto connDup = cast(CURL*) receiveOnly!ulong(); 5222 auto client = Conn(); 5223 client.p.curl.handle = connDup; 5224 5225 // receive a method for both ftp and http but just use it for http 5226 auto method = receiveOnly!(HTTP.Method)(); 5227 5228 client.onReceive = (ubyte[] data) 5229 { 5230 // If no terminator is specified the chunk size is fixed. 5231 static if ( is(Terminator == void) ) 5232 return receiveChunks(data, outdata, freeBuffers, buffer, 5233 fromTid, aborted); 5234 else 5235 return receiveLines(data, encodingScheme, 5236 keepTerminator, terminator, leftOverBytes, 5237 bufferValid, freeBuffers, buffer, 5238 fromTid, aborted); 5239 }; 5240 5241 static if ( is(Conn == HTTP) ) 5242 { 5243 client.method = method; 5244 // register dummy header handler 5245 client.onReceiveHeader = (in char[] key, in char[] value) 5246 { 5247 if (key == "content-type") 5248 encodingScheme = EncodingScheme.create(client.p.charset); 5249 }; 5250 } 5251 else 5252 { 5253 encodingScheme = EncodingScheme.create(client.encoding); 5254 } 5255 5256 // Start the request 5257 CurlCode code; 5258 try 5259 { 5260 code = client.perform(No.throwOnError); 5261 } 5262 catch (Exception ex) 5263 { 5264 prioritySend(fromTid, cast(immutable(Exception)) ex); 5265 fromTid.send(thisTid, curlMessage(true)); // signal done 5266 return; 5267 } 5268 5269 if (code != CurlError.ok) 5270 { 5271 if (aborted && (code == CurlError.aborted_by_callback || 5272 code == CurlError.write_error)) 5273 { 5274 fromTid.send(thisTid, curlMessage(true)); // signal done 5275 return; 5276 } 5277 prioritySend(fromTid, cast(immutable(CurlException)) 5278 new CurlException(client.p.curl.errorString(code))); 5279 5280 fromTid.send(thisTid, curlMessage(true)); // signal done 5281 return; 5282 } 5283 5284 // Send remaining data that is not a full chunk size 5285 static if ( is(Terminator == void) ) 5286 finalizeChunks(outdata, buffer, fromTid); 5287 else 5288 finalizeLines(bufferValid, buffer, fromTid); 5289 5290 fromTid.send(thisTid, curlMessage(true)); // signal done 5291 } 5292 }