PageRenderTime 65ms CodeModel.GetById 11ms app.highlight 47ms RepoModel.GetById 1ms app.codeStats 0ms

/src/lhttpc_client.erl

https://bitbucket.org/zajda/lhttpc
Erlang | 685 lines | 563 code | 41 blank | 81 comment | 0 complexity | acbe0500eafc4c800288c972f35cff51 MD5 | raw file
  1%%% ----------------------------------------------------------------------------
  2%%% Copyright (c) 2009, Erlang Training and Consulting Ltd.
  3%%% All rights reserved.
  4%%%
  5%%% Redistribution and use in source and binary forms, with or without
  6%%% modification, are permitted provided that the following conditions are met:
  7%%%    * Redistributions of source code must retain the above copyright
  8%%%      notice, this list of conditions and the following disclaimer.
  9%%%    * Redistributions in binary form must reproduce the above copyright
 10%%%      notice, this list of conditions and the following disclaimer in the
 11%%%      documentation and/or other materials provided with the distribution.
 12%%%    * Neither the name of Erlang Training and Consulting Ltd. nor the
 13%%%      names of its contributors may be used to endorse or promote products
 14%%%      derived from this software without specific prior written permission.
 15%%%
 16%%% THIS SOFTWARE IS PROVIDED BY Erlang Training and Consulting Ltd. ''AS IS''
 17%%% AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 18%%% IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 19%%% ARE DISCLAIMED. IN NO EVENT SHALL Erlang Training and Consulting Ltd. BE
 20%%% LIABLE SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
 21%%% BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
 22%%% WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
 23%%% OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
 24%%% ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 25%%% ----------------------------------------------------------------------------
 26
 27%%% @private
 28%%% @author Oscar Hellstr??m <oscar@erlang-consulting.com>
 29%%% @doc
 30%%% This module implements the HTTP request handling. This should normally
 31%%% not be called directly since it should be spawned by the lhttpc module.
 32%%% @end
 33%%% @type boolean() = bool().
 34%%% @type iolist() = [] | binary() | [char() | binary() | iolist()].
 35-module(lhttpc_client).
 36
 37-export([request/9]).
 38
 39-include("lhttpc_types.hrl").
 40
 41-record(client_state, {
 42          host :: string(),
 43          port = 80 :: integer(),
 44          ssl = false :: true | false,
 45          method :: string(),
 46          request :: iolist(),
 47          request_headers :: headers(),
 48          socket,
 49          connect_timeout = infinity :: timeout(),
 50          connect_options = [] :: [any()],
 51          attempts :: integer(),
 52          requester :: pid(),
 53          partial_upload = false :: true | false,
 54          chunked_upload = false :: true | false,
 55          upload_window :: non_neg_integer() | infinity,
 56          partial_download = false :: true | false,
 57          download_window = infinity :: timeout(),
 58          part_size :: non_neg_integer() | infinity,
 59          %% in case of infinity we read whatever data we can get from
 60          %% the wire at that point or in case of chunked one chunk
 61          proxy_host :: string(),
 62          proxy_port :: integer(),
 63          proxy_auth :: {string(),string()},
 64          ignore_proxy :: [{string(),integer()}]
 65         }).
 66
 67-define(CONNECTION_HDR(HDRS, DEFAULT),
 68    string:to_lower(lhttpc_lib:header_value("connection", HDRS, DEFAULT))).
 69
 70-spec request(pid(), string(), 1..65535, true | false, string(),
 71        string() | atom(), headers(), iolist(), [option()]) -> no_return().
 72%% @spec (From, Host, Port, Ssl, Path, Method, Hdrs, RequestBody, Options) -> ok
 73%%    From = pid()
 74%%    Host = string()
 75%%    Port = integer()
 76%%    Ssl = boolean()
 77%%    Method = atom() | string()
 78%%    Hdrs = [Header]
 79%%    Header = {string() | atom(), string()}
 80%%    Body = iolist()
 81%%    Options = [Option]
 82%%    Option = {connect_timeout, Milliseconds}
 83%% @end
 84request(From, Host, Port, Ssl, Path, Method, Hdrs, Body, Options) ->
 85    Result = try
 86        execute(From, Host, Port, Ssl, Path, Method, Hdrs, Body, Options)
 87    catch
 88        Reason ->
 89            {response, self(), {error, Reason}};
 90        error:closed ->
 91            {response, self(), {error, connection_closed}};
 92        error:Error ->
 93            {exit, self(), {Error, erlang:get_stacktrace()}}
 94    end,
 95    case Result of
 96        {response, _, {ok, {no_return, _}}} -> ok;
 97        _Else                               -> From ! Result
 98    end,
 99    % Don't send back {'EXIT', self(), normal} if the process
100    % calling us is trapping exits
101    unlink(From),
102    ok.
103
104execute(From, Host, Port, Ssl, Path, Method, Hdrs, Body, Options) ->
105    UploadWindowSize = proplists:get_value(partial_upload, Options),
106    PartialUpload = proplists:is_defined(partial_upload, Options),
107    PartialDownload = proplists:is_defined(partial_download, Options),
108    PartialDownloadOptions = proplists:get_value(partial_download, Options, []),
109    NormalizedMethod = lhttpc_lib:normalize_method(Method),
110    {ChunkedUpload, Request} = lhttpc_lib:format_request(Path, NormalizedMethod,
111                                                         Hdrs, Host, Port, Body, 
112							 PartialUpload, Options),
113    ProxyAuth = proplists:get_value(proxy_auth, Options),
114    ProxyHost = proplists:get_value(proxy_host, Options),
115    ProxyPort = proplists:get_value(proxy_port, Options, 80),
116    IgnoreProxy =  proplists:get_value(ignore_proxy, Options),
117    {SocketHost, SocketPort} = determine_destination({Host,Port},
118                                                     {ProxyHost,ProxyPort},
119                                                     IgnoreProxy
120                                                    ),
121    SocketRequest = {socket, self(), SocketHost, SocketPort, Ssl},
122    Socket = case gen_server:call(lhttpc_manager, SocketRequest, infinity) of
123        {ok, S}   -> S; % Re-using HTTP/1.1 connections
124        no_socket -> undefined % Opening a new HTTP/1.1 connection
125    end,
126    State = #client_state{
127        host = Host,
128        port = Port,
129        ssl = Ssl,
130        method = NormalizedMethod,
131        request = Request,
132        requester = From,
133        request_headers = Hdrs,
134        socket = Socket,
135        connect_timeout = proplists:get_value(connect_timeout, Options,
136            infinity),
137        connect_options = proplists:get_value(connect_options, Options, []),
138        attempts = 1 + proplists:get_value(send_retry, Options, 1),
139        partial_upload = PartialUpload,
140        upload_window = UploadWindowSize,
141        chunked_upload = ChunkedUpload,
142        partial_download = PartialDownload,
143        download_window = proplists:get_value(window_size,
144            PartialDownloadOptions, infinity),
145        part_size = proplists:get_value(part_size,
146            PartialDownloadOptions, infinity),
147        proxy_auth = ProxyAuth,
148        proxy_host = ProxyHost, 
149        proxy_port = ProxyPort,
150        ignore_proxy = IgnoreProxy
151     },
152    Response = case send_request(State) of
153        {R, undefined} ->
154            {ok, R};
155        {R, NewSocket} ->
156            % The socket we ended up doing the request over is returned
157            % here, it might be the same as Socket, but we don't know.
158            % I've noticed that we don't want to give send sockets that we
159            % can't change the controlling process for to the manager. This
160            % really shouldn't fail, but it could do if:
161            % * The socket was closed remotely already
162            % * Due to an error in this module (returning dead sockets for
163            %   instance)
164            ManagerPid = whereis(lhttpc_manager),
165            case lhttpc_sock:controlling_process(NewSocket, ManagerPid, Ssl) of
166                ok ->
167                    gen_server:cast(lhttpc_manager,
168                        {done, Host, Port, Ssl, NewSocket});
169                _ ->
170                    ok
171            end,
172            {ok, R}
173    end,
174    {response, self(), Response}.
175
176send_request(#client_state{attempts = 0}) ->
177    % Don't try again if the number of allowed attempts is 0.
178    throw(connection_closed);
179send_request(#client_state{socket = undefined} = State) ->
180    Host = State#client_state.host,
181    Port = State#client_state.port,
182    Ssl = State#client_state.ssl,
183    Timeout = State#client_state.connect_timeout,
184    ConnectOptions = State#client_state.connect_options,
185    SocketOptions = [binary, {packet, http}, {active, false} | ConnectOptions],
186    ProxyHost = State#client_state.proxy_host,
187    ProxyPort = State#client_state.proxy_port,
188    IgnoreProxy = State#client_state.ignore_proxy,
189    {SocketHost, SocketPort} = determine_destination({Host,Port},
190                                                     {ProxyHost,ProxyPort},
191                                                     IgnoreProxy
192                                                    ),
193    case lhttpc_sock:connect(SocketHost, SocketPort, SocketOptions, Timeout, Ssl) of
194        {ok, Socket} ->
195            send_request(State#client_state{socket = Socket});
196        {error, etimedout} ->
197            % TCP stack decided to give up
198            throw(connect_timeout);
199        {error, timeout} ->
200            throw(connect_timeout);
201        {error, Reason} ->
202            erlang:error(Reason)
203    end;
204send_request(State) ->
205    Socket = State#client_state.socket,
206    Ssl = State#client_state.ssl,
207    Request = State#client_state.request,
208    case lhttpc_sock:send(Socket, Request, Ssl) of
209        ok ->
210            if
211                State#client_state.partial_upload     -> partial_upload(State);
212                not State#client_state.partial_upload -> read_response(State)
213            end;
214        {error, closed} ->
215            lhttpc_sock:close(Socket, Ssl),
216            NewState = State#client_state{
217                socket = undefined,
218                attempts = State#client_state.attempts - 1
219            },
220            send_request(NewState);
221        {error, Reason} ->
222            lhttpc_sock:close(Socket, Ssl),
223            erlang:error(Reason)
224    end.
225
226partial_upload(State) ->
227    Response = {ok, {self(), State#client_state.upload_window}},
228    State#client_state.requester ! {response, self(), Response},
229    partial_upload_loop(State#client_state{attempts = 1, request = undefined}).
230
231partial_upload_loop(State = #client_state{requester = Pid}) ->
232    receive
233        {trailers, Pid, Trailers} ->
234            send_trailers(State, Trailers),
235            read_response(State);
236        {body_part, Pid, http_eob} ->
237            send_body_part(State, http_eob),
238            read_response(State);
239        {body_part, Pid, Data} ->
240            send_body_part(State, Data),
241            Pid ! {ack, self()},
242            partial_upload_loop(State)
243    end.
244
245send_body_part(State = #client_state{socket = Socket, ssl = Ssl}, BodyPart) ->
246    Data = encode_body_part(State, BodyPart),
247    check_send_result(State, lhttpc_sock:send(Socket, Data, Ssl)).
248
249send_trailers(State = #client_state{chunked_upload = true}, Trailers) ->
250    Socket = State#client_state.socket,
251    Ssl = State#client_state.ssl,
252    Data = [<<"0\r\n">>, lhttpc_lib:format_hdrs(Trailers)],
253    check_send_result(State, lhttpc_sock:send(Socket, Data, Ssl));
254send_trailers(#client_state{chunked_upload = false}, _Trailers) ->
255    erlang:error(trailers_not_allowed).
256
257encode_body_part(#client_state{chunked_upload = true}, http_eob) ->
258    <<"0\r\n\r\n">>; % We don't send trailers after http_eob
259encode_body_part(#client_state{chunked_upload = false}, http_eob) ->
260    <<>>;
261encode_body_part(#client_state{chunked_upload = true}, Data) ->
262    Size = list_to_binary(erlang:integer_to_list(iolist_size(Data), 16)),
263    [Size, <<"\r\n">>, Data, <<"\r\n">>];
264encode_body_part(#client_state{chunked_upload = false}, Data) ->
265    Data.
266
267check_send_result(_State, ok) ->
268    ok;
269check_send_result(#client_state{socket = Sock, ssl = Ssl}, {error, Reason}) ->
270    lhttpc_sock:close(Sock, Ssl),
271    throw(Reason).
272
273read_response(#client_state{socket = Socket, ssl = Ssl} = State) ->
274    lhttpc_sock:setopts(Socket, [{packet, http}], Ssl),
275    read_response(State, nil, {nil, nil}, []).
276
277read_response(State, Vsn, {StatusCode, _} = Status, Hdrs) ->
278    Socket = State#client_state.socket,
279    Ssl = State#client_state.ssl,
280    case lhttpc_sock:recv(Socket, Ssl) of
281        {ok, {http_response, NewVsn, NewStatusCode, Reason}} ->
282            NewStatus = {NewStatusCode, Reason},
283            read_response(State, NewVsn, NewStatus, Hdrs);
284        {ok, {http_header, _, Name, _, Value}} ->
285            Header = {lhttpc_lib:maybe_atom_to_list(Name), Value},
286            read_response(State, Vsn, Status, [Header | Hdrs]);
287        {ok, http_eoh} when StatusCode >= 100, StatusCode =< 199 ->
288            % RFC 2616, section 10.1:
289            % A client MUST be prepared to accept one or more
290            % 1xx status responses prior to a regular
291            % response, even if the client does not expect a
292            % 100 (Continue) status message. Unexpected 1xx
293            % status responses MAY be ignored by a user agent.
294            read_response(State, nil, {nil, nil}, []);
295        {ok, http_eoh} ->
296            lhttpc_sock:setopts(Socket, [{packet, raw}], Ssl),
297            Response = handle_response_body(State, Vsn, Status, Hdrs),
298            NewHdrs = element(2, Response),
299            ReqHdrs = State#client_state.request_headers,
300            NewSocket = maybe_close_socket(Socket, Ssl, Vsn, ReqHdrs, NewHdrs),
301            {Response, NewSocket};
302        {error, closed} ->
303            % Either we only noticed that the socket was closed after we
304            % sent the request, the server closed it just after we put
305            % the request on the wire or the server has some issues and is
306            % closing connections without sending responses.
307            % If this the first attempt to send the request, we will try again.
308            lhttpc_sock:close(Socket, Ssl),
309            NewState = State#client_state{
310                socket = undefined,
311                attempts = State#client_state.attempts - 1
312            },
313            send_request(NewState);
314        {error, Reason} ->
315            erlang:error(Reason)
316    end.
317
318handle_response_body(#client_state{partial_download = false} = State, Vsn,
319        Status, Hdrs) ->
320    Socket = State#client_state.socket,
321    Ssl = State#client_state.ssl,
322    Method = State#client_state.method,
323    {Body, NewHdrs} = case has_body(Method, element(1, Status), Hdrs) of
324        true  -> read_body(Vsn, Hdrs, Ssl, Socket, body_type(Hdrs));
325        false -> {<<>>, Hdrs}
326    end,
327    {Status, NewHdrs, Body};
328handle_response_body(#client_state{partial_download = true} = State, Vsn,
329        Status, Hdrs) ->
330    Method = State#client_state.method,
331    case has_body(Method, element(1, Status), Hdrs) of
332        true ->
333            Response = {ok, {Status, Hdrs, self()}},
334            State#client_state.requester ! {response, self(), Response},
335            MonRef = erlang:monitor(process, State#client_state.requester),
336            Res = read_partial_body(State, Vsn, Hdrs, body_type(Hdrs)),
337            erlang:demonitor(MonRef, [flush]),
338            Res;
339        false ->
340            {Status, Hdrs, undefined}
341    end.
342
343has_body("HEAD", _, _) ->
344    % HEAD responses aren't allowed to include a body
345    false;
346has_body("OPTIONS", _, Hdrs) ->
347    % OPTIONS can include a body, if Content-Length or Transfer-Encoding
348    % indicates it
349    ContentLength = lhttpc_lib:header_value("content-length", Hdrs),
350    TransferEncoding = lhttpc_lib:header_value("transfer-encoding", Hdrs),
351    case {ContentLength, TransferEncoding} of
352        {undefined, undefined} -> false;
353        {_, _}                 -> true
354    end;
355has_body(_, 204, _) ->
356    false; % RFC 2616 10.2.5: 204 No Content
357has_body(_, 304, _) ->
358    false; % RFC 2616 10.3.5: 304 Not Modified
359has_body(_, _, _) ->
360    true. % All other responses are assumed to have a body
361
362body_type(Hdrs) ->
363    % Find out how to read the entity body from the request.
364    % * If we have a Content-Length, just use that and read the complete
365    %   entity.
366    % * If Transfer-Encoding is set to chunked, we should read one chunk at
367    %   the time
368    % * If neither of this is true, we need to read until the socket is
369    %   closed (AFAIK, this was common in versions before 1.1).
370    case lhttpc_lib:header_value("content-length", Hdrs) of
371        undefined ->
372            TransferEncoding = string:to_lower(
373                lhttpc_lib:header_value("transfer-encoding", Hdrs, "undefined")
374            ),
375            case TransferEncoding of
376                "chunked" -> chunked;
377                _         -> infinite
378            end;
379        ContentLength ->
380            {fixed_length, list_to_integer(ContentLength)}
381    end.
382
383read_partial_body(State, _Vsn, Hdrs, chunked) ->
384    Window = State#client_state.download_window,
385    read_partial_chunked_body(State, Hdrs, Window, 0, [], 0);
386read_partial_body(State, Vsn, Hdrs, infinite) ->
387    check_infinite_response(Vsn, Hdrs),
388    read_partial_infinite_body(State, Hdrs, State#client_state.download_window);
389read_partial_body(State, _Vsn, Hdrs, {fixed_length, ContentLength}) ->
390    read_partial_finite_body(State, Hdrs, ContentLength,
391        State#client_state.download_window).
392
393read_body(_Vsn, Hdrs, Ssl, Socket, chunked) ->
394    read_chunked_body(Socket, Ssl, Hdrs, []);
395read_body(Vsn, Hdrs, Ssl, Socket, infinite) ->
396    check_infinite_response(Vsn, Hdrs),
397    read_infinite_body(Socket, Hdrs, Ssl);
398read_body(_Vsn, Hdrs, Ssl, Socket, {fixed_length, ContentLength}) ->
399    read_length(Hdrs, Ssl, Socket, ContentLength).
400
401read_partial_finite_body(State = #client_state{}, Hdrs, 0, _Window) ->
402    reply_end_of_body(State, [], Hdrs);
403read_partial_finite_body(State = #client_state{requester = To}, Hdrs,
404        ContentLength, 0) ->
405    receive
406        {ack, To} ->
407            read_partial_finite_body(State, Hdrs, ContentLength, 1);
408        {'DOWN', _, process, To, _} ->
409            exit(normal)
410    end;
411read_partial_finite_body(State, Hdrs, ContentLength, Window) when Window >= 0->
412    Bin = read_body_part(State, ContentLength),
413    State#client_state.requester ! {body_part, self(), Bin},
414    To = State#client_state.requester,
415    receive
416        {ack, To} ->
417            Length = ContentLength - iolist_size(Bin),
418            read_partial_finite_body(State, Hdrs, Length, Window);
419        {'DOWN', _, process, To, _} ->
420            exit(normal)
421    after 0 ->
422            Length = ContentLength - iolist_size(Bin),
423        read_partial_finite_body(State, Hdrs, Length, lhttpc_lib:dec(Window))
424    end.
425
426read_body_part(#client_state{part_size = infinity} = State, _ContentLength) ->
427    case lhttpc_sock:recv(State#client_state.socket, State#client_state.ssl) of
428        {ok, Data} ->
429            Data;
430        {error, Reason} ->
431            erlang:error(Reason)
432    end;
433read_body_part(#client_state{part_size = PartSize} = State, ContentLength)
434        when PartSize =< ContentLength ->
435    Socket = State#client_state.socket, 
436    Ssl = State#client_state.ssl,
437    PartSize = State#client_state.part_size,
438    case lhttpc_sock:recv(Socket, PartSize, Ssl) of
439        {ok, Data} ->
440            Data;
441        {error, Reason} ->
442            erlang:error(Reason)
443    end;
444read_body_part(#client_state{part_size = PartSize} = State, ContentLength)
445        when PartSize > ContentLength ->
446    Socket = State#client_state.socket, 
447    Ssl = State#client_state.ssl,
448    case lhttpc_sock:recv(Socket, ContentLength, Ssl) of
449        {ok, Data} ->
450            Data;
451        {error, Reason} ->
452            erlang:error(Reason)
453    end.
454
455read_length(Hdrs, Ssl, Socket, Length) ->
456    case lhttpc_sock:recv(Socket, Length, Ssl) of
457        {ok, Data} ->
458            {Data, Hdrs};
459        {error, Reason} ->
460            erlang:error(Reason)
461    end.
462
463read_partial_chunked_body(State, Hdrs, Window, BufferSize, Buffer, 0) ->
464    Socket = State#client_state.socket,
465    Ssl = State#client_state.ssl,
466    PartSize = State#client_state.part_size,
467    case read_chunk_size(Socket, Ssl) of
468        0 ->
469            reply_chunked_part(State, Buffer, Window),
470            {Trailers, NewHdrs} = read_trailers(Socket, Ssl, [], Hdrs),
471            reply_end_of_body(State, Trailers, NewHdrs);
472        ChunkSize when PartSize =:= infinity ->
473            Chunk = read_chunk(Socket, Ssl, ChunkSize),
474            NewWindow = reply_chunked_part(State, [Chunk | Buffer], Window),
475            read_partial_chunked_body(State, Hdrs, NewWindow, 0, [], 0);
476        ChunkSize when BufferSize + ChunkSize >= PartSize ->
477            {Chunk, RemSize} = read_partial_chunk(Socket, Ssl,
478                PartSize - BufferSize, ChunkSize),
479            NewWindow = reply_chunked_part(State, [Chunk | Buffer], Window),
480            read_partial_chunked_body(State, Hdrs, NewWindow, 0, [], RemSize);
481        ChunkSize ->
482            Chunk = read_chunk(Socket, Ssl, ChunkSize),
483            read_partial_chunked_body(State, Hdrs, Window,
484                BufferSize + ChunkSize, [Chunk | Buffer], 0)
485    end;
486read_partial_chunked_body(State, Hdrs, Window, BufferSize, Buffer, RemSize) ->
487    Socket = State#client_state.socket,
488    Ssl = State#client_state.ssl,
489    PartSize = State#client_state.part_size,
490    if
491        BufferSize + RemSize >= PartSize ->
492            {Chunk, NewRemSize} =
493                read_partial_chunk(Socket, Ssl, PartSize - BufferSize, RemSize),
494            NewWindow = reply_chunked_part(State, [Chunk | Buffer], Window),
495            read_partial_chunked_body(State, Hdrs, NewWindow, 0, [],
496                NewRemSize);
497        BufferSize + RemSize < PartSize ->
498            Chunk = read_chunk(Socket, Ssl, RemSize),
499            read_partial_chunked_body(State, Hdrs, Window, BufferSize + RemSize,
500                [Chunk | Buffer], 0)
501    end.
502
503read_chunk_size(Socket, Ssl) ->
504    lhttpc_sock:setopts(Socket, [{packet, line}], Ssl),
505    case lhttpc_sock:recv(Socket, Ssl) of
506        {ok, ChunkSizeExt} ->
507            chunk_size(ChunkSizeExt);
508        {error, Reason} ->
509            erlang:error(Reason)
510    end.
511
512reply_chunked_part(_State, [], Window) ->
513    Window;
514reply_chunked_part(State = #client_state{requester = Pid}, Buff, 0) ->
515    receive
516        {ack, Pid} ->
517            reply_chunked_part(State, Buff, 1);
518        {'DOWN', _, process, Pid, _} ->
519            exit(normal)
520    end;
521reply_chunked_part(#client_state{requester = Pid}, Buffer, Window) ->
522    Pid ! {body_part, self(), list_to_binary(lists:reverse(Buffer))},
523    receive
524        {ack, Pid} ->  Window;
525        {'DOWN', _, process, Pid, _} -> exit(normal)
526    after 0 ->
527        lhttpc_lib:dec(Window)
528    end.
529
530read_chunked_body(Socket, Ssl, Hdrs, Chunks) ->
531    case read_chunk_size(Socket, Ssl) of
532        0 ->
533            Body = list_to_binary(lists:reverse(Chunks)),
534            {_, NewHdrs} = read_trailers(Socket, Ssl, [], Hdrs),
535            {Body, NewHdrs};
536        Size ->
537            Chunk = read_chunk(Socket, Ssl, Size),
538            read_chunked_body(Socket, Ssl, Hdrs, [Chunk | Chunks])
539    end.
540
541chunk_size(Bin) ->
542    erlang:list_to_integer(lists:reverse(chunk_size(Bin, [])), 16).
543
544chunk_size(<<$;, _/binary>>, Chars) ->
545    Chars;
546chunk_size(<<"\r\n", _/binary>>, Chars) ->
547    Chars;
548chunk_size(<<$\s, Binary/binary>>, Chars) ->
549    %% Facebook's HTTP server returns a chunk size like "6  \r\n"
550    chunk_size(Binary, Chars);
551chunk_size(<<Char, Binary/binary>>, Chars) ->
552    chunk_size(Binary, [Char | Chars]).
553
554read_partial_chunk(Socket, Ssl, ChunkSize, ChunkSize) ->
555    {read_chunk(Socket, Ssl, ChunkSize), 0};
556read_partial_chunk(Socket, Ssl, Size, ChunkSize) ->
557    lhttpc_sock:setopts(Socket, [{packet, raw}], Ssl),
558    case lhttpc_sock:recv(Socket, Size, Ssl) of
559        {ok, Chunk} ->
560            {Chunk, ChunkSize - Size};
561        {error, Reason} ->
562            erlang:error(Reason)
563    end.
564
565read_chunk(Socket, Ssl, Size) ->
566    lhttpc_sock:setopts(Socket, [{packet, raw}], Ssl),
567    case lhttpc_sock:recv(Socket, Size + 2, Ssl) of
568        {ok, <<Chunk:Size/binary, "\r\n">>} ->
569            Chunk;
570        {ok, Data} ->
571            erlang:error({invalid_chunk, Data});
572        {error, Reason} ->
573            erlang:error(Reason)
574    end.
575
576read_trailers(Socket, Ssl, Trailers, Hdrs) ->
577    lhttpc_sock:setopts(Socket, [{packet, httph}], Ssl),
578    case lhttpc_sock:recv(Socket, Ssl) of
579        {ok, http_eoh} ->
580            {Trailers, Hdrs};
581        {ok, {http_header, _, Name, _, Value}} ->
582            Header = {lhttpc_lib:maybe_atom_to_list(Name), Value},
583            read_trailers(Socket, Ssl, [Header | Trailers], [Header | Hdrs]);
584        {error, {http_error, Data}} ->
585            erlang:error({bad_trailer, Data})
586    end.
587
588reply_end_of_body(#client_state{requester = Requester}, Trailers, Hdrs) ->
589    Requester ! {http_eob, self(), Trailers},
590    {no_return, Hdrs}.
591
592read_partial_infinite_body(State = #client_state{requester = To}, Hdrs, 0) ->
593    receive
594        {ack, To} ->
595            read_partial_infinite_body(State, Hdrs, 1);
596        {'DOWN', _, process, To, _} ->
597            exit(normal)
598    end;
599read_partial_infinite_body(State = #client_state{requester = To}, Hdrs, Window)
600        when Window >= 0 ->
601    case read_infinite_body_part(State) of
602        http_eob -> reply_end_of_body(State, [], Hdrs);
603        Bin ->
604            State#client_state.requester ! {body_part, self(), Bin},
605            receive
606                {ack, To} ->
607                    read_partial_infinite_body(State, Hdrs, Window);
608                {'DOWN', _, process, To, _} ->
609                    exit(normal)
610            after 0 ->
611                read_partial_infinite_body(State, Hdrs, lhttpc_lib:dec(Window))
612            end
613    end.
614
615read_infinite_body_part(#client_state{socket = Socket, ssl = Ssl}) ->
616    case lhttpc_sock:recv(Socket, Ssl) of
617        {ok, Data} ->
618            Data;
619        {error, closed} ->
620            http_eob;
621        {error, Reason} ->
622            erlang:error(Reason)
623    end.
624
625check_infinite_response({1, Minor}, Hdrs) when Minor >= 1 ->
626    HdrValue = lhttpc_lib:header_value("connection", Hdrs, "keep-alive"),
627    case string:to_lower(HdrValue) of
628        "close" -> ok;
629        _       -> erlang:error(no_content_length)
630    end;
631check_infinite_response(_, Hdrs) ->
632    HdrValue = lhttpc_lib:header_value("connection", Hdrs, "close"),
633    case string:to_lower(HdrValue) of
634        "keep-alive" -> erlang:error(no_content_length);
635        _            -> ok
636    end.
637
638read_infinite_body(Socket, Hdrs, Ssl) ->
639    read_until_closed(Socket, <<>>, Hdrs, Ssl).
640
641read_until_closed(Socket, Acc, Hdrs, Ssl) ->
642    case lhttpc_sock:recv(Socket, Ssl) of
643        {ok, Body} ->
644            NewAcc = <<Acc/binary, Body/binary>>,
645            read_until_closed(Socket, NewAcc, Hdrs, Ssl);
646        {error, closed} ->
647            {Acc, Hdrs};
648        {error, Reason} ->
649            erlang:error(Reason)
650    end.
651
652maybe_close_socket(Socket, Ssl, {1, Minor}, ReqHdrs, RespHdrs) when Minor >= 1->
653    ClientConnection = ?CONNECTION_HDR(ReqHdrs, "keep-alive"),
654    ServerConnection = ?CONNECTION_HDR(RespHdrs, "keep-alive"),
655    if
656        ClientConnection =:= "close"; ServerConnection =:= "close" ->
657            lhttpc_sock:close(Socket, Ssl),
658            undefined;
659        ClientConnection =/= "close", ServerConnection =/= "close" ->
660            Socket
661    end;
662maybe_close_socket(Socket, Ssl, _, ReqHdrs, RespHdrs) ->
663    ClientConnection = ?CONNECTION_HDR(ReqHdrs, "keep-alive"),
664    ServerConnection = ?CONNECTION_HDR(RespHdrs, "close"),
665    if
666        ClientConnection =:= "close"; ServerConnection =/= "keep-alive" ->
667            lhttpc_sock:close(Socket, Ssl),
668            undefined;
669        ClientConnection =/= "close", ServerConnection =:= "keep-alive" ->
670            Socket
671    end.
672
673determine_destination({ReqHost, ReqPort}, {undefined, _}, _) ->
674    {ReqHost, ReqPort};
675determine_destination(_, {ProxyHost, ProxyPort}, []) ->
676    {ProxyHost, ProxyPort};
677determine_destination(_, {ProxyHost, ProxyPort}, undefined) ->
678    {ProxyHost, ProxyPort};
679determine_destination(OrigAddr, Proxy, NoProxy) ->
680   case lists:member(Proxy, NoProxy) of
681       true ->
682           OrigAddr;
683       false ->
684           Proxy
685   end.