PageRenderTime 58ms CodeModel.GetById 2ms app.highlight 50ms RepoModel.GetById 1ms app.codeStats 0ms

/src/dlhttpc_client.erl

https://github.com/ferd/dlhttpc
Erlang | 655 lines | 540 code | 40 blank | 75 comment | 0 complexity | 6bfc2b9b3a4a2555d41a3645d1327a4f MD5 | raw file
  1%%% ----------------------------------------------------------------------------
  2%%% Copyright (c) 2009, Erlang Training and Consulting Ltd.
  3%%% Copyright (c) 2012, Frederic Trottier-Hebert
  4%%% All rights reserved.
  5%%%
  6%%% Redistribution and use in source and binary forms, with or without
  7%%% modification, are permitted provided that the following conditions are met:
  8%%%    * Redistributions of source code must retain the above copyright
  9%%%      notice, this list of conditions and the following disclaimer.
 10%%%    * Redistributions in binary form must reproduce the above copyright
 11%%%      notice, this list of conditions and the following disclaimer in the
 12%%%      documentation and/or other materials provided with the distribution.
 13%%%    * Neither the name of Erlang Training and Consulting Ltd. nor the
 14%%%      names of its contributors may be used to endorse or promote products
 15%%%      derived from this software without specific prior written permission.
 16%%%
 17%%% THIS SOFTWARE IS PROVIDED BY Erlang Training and Consulting Ltd. ''AS IS''
 18%%% AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 19%%% IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 20%%% ARE DISCLAIMED. IN NO EVENT SHALL Erlang Training and Consulting Ltd. BE
 21%%% LIABLE SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
 22%%% BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
 23%%% WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
 24%%% OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
 25%%% ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 26%%% ----------------------------------------------------------------------------
 27
 28%%% @private
 29%%% @author Oscar Hellström <oscar@hellstrom.st>
 30%%% @author Fred Hebert <mononcqc@ferd.ca>
 31%%% @doc
 32%%% This module implements the HTTP request handling. This should normally
 33%%% not be called directly since it should be spawned by the dlhttpc module.
 34-module(dlhttpc_client).
 35
 36-export([request/10]).
 37
 38-include("dlhttpc_types.hrl").
 39
 40-record(client_state, {
 41        req_id :: term(),
 42        host :: string(),
 43        port = 80 :: integer(),
 44        ssl = false :: true | false,
 45        method :: string(),
 46        request :: iolist(),
 47        request_headers :: headers(),
 48        socket_ref,
 49        socket,
 50        connect_timeout = infinity :: timeout(),
 51        connect_options = [] :: [any()],
 52        attempts :: integer(),
 53        requester :: pid(),
 54        partial_upload = false :: true | false,
 55        chunked_upload = false :: true | false,
 56        upload_window :: non_neg_integer() | infinity,
 57        partial_download = false :: true | false,
 58        download_window = infinity :: timeout(),
 59        part_size :: non_neg_integer() | infinity
 60        %% in case of infinity we read whatever data we can get from
 61        %% the wire at that point or in case of chunked one chunk
 62    }).
 63
 64-define(CONNECTION_HDR(HDRS, DEFAULT),
 65    string:to_lower(dlhttpc_lib:header_value("connection", HDRS, DEFAULT))).
 66
 67-spec request(term(), pid(), string(), 1..65535, true | false, string(),
 68        string() | atom(), headers(), iolist(), [option()]) -> no_return().
 69%% @spec (ReqId, From, Host, Port, Ssl, Path, Method, Hdrs, RequestBody, Options) -> ok
 70%%    ReqId = term()
 71%%    From = pid()
 72%%    Host = string()
 73%%    Port = integer()
 74%%    Ssl = boolean()
 75%%    Method = atom() | string()
 76%%    Hdrs = [Header]
 77%%    Header = {string() | atom(), string()}
 78%%    Body = iolist()
 79%%    Options = [Option]
 80%%    Option = {connect_timeout, Milliseconds}
 81%% @end
 82request(ReqId, From, Host, Port, Ssl, Path, Method, Hdrs, Body, Options) ->
 83    Result = try
 84        execute(ReqId, From, Host, Port, Ssl, Path, Method, Hdrs, Body, Options)
 85    catch
 86        Reason ->
 87            {response, ReqId, self(), {error, Reason}};
 88        error:closed ->
 89            {response, ReqId, self(), {error, connection_closed}};
 90        error:Error ->
 91            {exit, ReqId, self(), {Error, erlang:get_stacktrace()}}
 92    end,
 93    case Result of
 94        {response, _, _, {ok, {no_return, _}}} -> ok;
 95        _Else                               -> From ! Result
 96    end,
 97    % Don't send back {'EXIT', self(), normal} if the process
 98    % calling us is trapping exits
 99    unlink(From),
100    ok.
101
102execute(ReqId, From, Host, Port, Ssl, Path, Method, Hdrs, Body, Options) ->
103    UploadWindowSize = proplists:get_value(partial_upload, Options),
104    PartialUpload = proplists:is_defined(partial_upload, Options),
105    PartialDownload = proplists:is_defined(partial_download, Options),
106    PartialDownloadOptions = proplists:get_value(partial_download, Options, []),
107    NormalizedMethod = dlhttpc_lib:normalize_method(Method),
108    MaxConnections = proplists:get_value(max_connections, Options, 10),
109    ConnectionTimeout = proplists:get_value(connection_timeout, Options, infinity),
110    {ChunkedUpload, Request} = dlhttpc_lib:format_request(Path, NormalizedMethod,
111        Hdrs, Host, Port, Body, PartialUpload),
112    ConnectOptions = proplists:get_value(connect_options, Options, []),
113    SockOpts = [binary, {packet, http}, {active, false} | ConnectOptions],
114    {SocketRef, Socket} =
115        case MaxConnections of
116            bypass ->
117                {undefined, undefined};
118            Number when is_integer(Number) ->
119                case dlhttpc_disp:checkout(Host, Port, Ssl, MaxConnections, ConnectionTimeout, SockOpts) of
120                    {ok, Ref, S} -> {Ref, S}; % Re-using HTTP/1.1 connections
121                    {error, CheckoutErr} -> throw(CheckoutErr)
122                end
123        end,
124    State = #client_state{
125        req_id = ReqId,
126        host = Host,
127        port = Port,
128        ssl = Ssl,
129        method = NormalizedMethod,
130        request = Request,
131        requester = From,
132        request_headers = Hdrs,
133        socket_ref = SocketRef,
134        socket = Socket,
135        connect_timeout = proplists:get_value(connect_timeout, Options,
136            infinity),
137        connect_options = ConnectOptions,
138        attempts = 1 + proplists:get_value(send_retry, Options, 1),
139        partial_upload = PartialUpload,
140        upload_window = UploadWindowSize,
141        chunked_upload = ChunkedUpload,
142        partial_download = PartialDownload,
143        download_window = proplists:get_value(window_size,
144            PartialDownloadOptions, infinity),
145        part_size = proplists:get_value(part_size,
146            PartialDownloadOptions, infinity)
147    },
148    Response = case {MaxConnections, send_request(State)} of
149        {bypass, {R, NewSocket}} ->
150            dlhttpc_sock:close(NewSocket, Ssl),
151            {ok, R};
152        {_, {R, undefined}} ->
153            {ok, R};
154        {_, {R, NewSocket}} ->
155            % The socket we ended up doing the request over is returned
156            % here, it might be the same as Socket, but we don't know.
157            dlhttpc_disp:checkin(SocketRef, NewSocket),
158            {ok, R}
159    end,
160    {response, ReqId, self(), Response}.
161
162send_request(#client_state{attempts = 0}) ->
163    % Don't try again if the number of allowed attempts is 0.
164    throw(connection_closed);
165send_request(#client_state{socket = undefined} = State) ->
166    Host = State#client_state.host,
167    Port = State#client_state.port,
168    ConnectOptions = State#client_state.connect_options,
169    SockOpts = [binary, {packet, http}, {active, false} | ConnectOptions],
170    Timeout = State#client_state.connect_timeout,
171    Ssl = State#client_state.ssl,
172    case dlhttpc_sock:connect(Host, Port, SockOpts, Timeout, Ssl) of
173        {ok, Socket} ->
174            send_request(State#client_state{socket = Socket});
175        {error, etimedout} ->
176            % TCP stack decided to give up
177            throw(connect_timeout);
178        {error, timeout} ->
179            throw(connect_timeout);
180        {error, Reason} ->
181            erlang:error(Reason)
182    end;
183send_request(State) ->
184    Socket = State#client_state.socket,
185    Ssl = State#client_state.ssl,
186    Request = State#client_state.request,
187    case dlhttpc_sock:send(Socket, Request, Ssl) of
188        ok ->
189            if
190                State#client_state.partial_upload     -> partial_upload(State);
191                not State#client_state.partial_upload -> read_response(State)
192            end;
193        {error, closed} ->
194            dlhttpc_sock:close(Socket, Ssl),
195            NewState = State#client_state{
196                socket = undefined,
197                attempts = State#client_state.attempts - 1
198            },
199            send_request(NewState);
200        {error, Reason} ->
201            dlhttpc_sock:close(Socket, Ssl),
202            erlang:error(Reason)
203    end.
204
205partial_upload(State) ->
206    Response = {ok, {self(), State#client_state.upload_window}},
207    State#client_state.requester ! {response,State#client_state.req_id, self(), Response},
208    partial_upload_loop(State#client_state{attempts = 1, request = undefined}).
209
210partial_upload_loop(State = #client_state{requester = Pid}) ->
211    receive
212        {trailers, Pid, Trailers} ->
213            send_trailers(State, Trailers),
214            read_response(State);
215        {body_part, Pid, http_eob} ->
216            send_body_part(State, http_eob),
217            read_response(State);
218        {body_part, Pid, Data} ->
219            send_body_part(State, Data),
220            Pid ! {ack, self()},
221            partial_upload_loop(State)
222    end.
223
224send_body_part(State = #client_state{socket = Socket, ssl = Ssl}, BodyPart) ->
225    Data = encode_body_part(State, BodyPart),
226    check_send_result(State, dlhttpc_sock:send(Socket, Data, Ssl)).
227
228send_trailers(State = #client_state{chunked_upload = true}, Trailers) ->
229    Socket = State#client_state.socket,
230    Ssl = State#client_state.ssl,
231    Data = [<<"0\r\n">>, dlhttpc_lib:format_hdrs(Trailers)],
232    check_send_result(State, dlhttpc_sock:send(Socket, Data, Ssl));
233send_trailers(#client_state{chunked_upload = false}, _Trailers) ->
234    erlang:error(trailers_not_allowed).
235
236encode_body_part(#client_state{chunked_upload = true}, http_eob) ->
237    <<"0\r\n\r\n">>; % We don't send trailers after http_eob
238encode_body_part(#client_state{chunked_upload = false}, http_eob) ->
239    <<>>;
240encode_body_part(#client_state{chunked_upload = true}, Data) ->
241    Size = list_to_binary(erlang:integer_to_list(iolist_size(Data), 16)),
242    [Size, <<"\r\n">>, Data, <<"\r\n">>];
243encode_body_part(#client_state{chunked_upload = false}, Data) ->
244    Data.
245
246check_send_result(_State, ok) ->
247    ok;
248check_send_result(#client_state{socket = Sock, ssl = Ssl}, {error, Reason}) ->
249    dlhttpc_sock:close(Sock, Ssl),
250    throw(Reason).
251
252read_response(#client_state{socket = Socket, ssl = Ssl} = State) ->
253    dlhttpc_sock:setopts(Socket, [{packet, http}], Ssl),
254    read_response(State, nil, {nil, nil}, []).
255
256read_response(State, Vsn, {StatusCode, _} = Status, Hdrs) ->
257    Socket = State#client_state.socket,
258    Ssl = State#client_state.ssl,
259    case dlhttpc_sock:recv(Socket, Ssl) of
260        {ok, {http_response, NewVsn, NewStatusCode, Reason}} ->
261            NewStatus = {NewStatusCode, Reason},
262            read_response(State, NewVsn, NewStatus, Hdrs);
263        {ok, {http_header, _, Name, _, Value}} ->
264            Header = {dlhttpc_lib:maybe_atom_to_list(Name), Value},
265            read_response(State, Vsn, Status, [Header | Hdrs]);
266        {ok, http_eoh} when StatusCode >= 100, StatusCode =< 199 ->
267            % RFC 2616, section 10.1:
268            % A client MUST be prepared to accept one or more
269            % 1xx status responses prior to a regular
270            % response, even if the client does not expect a
271            % 100 (Continue) status message. Unexpected 1xx
272            % status responses MAY be ignored by a user agent.
273            read_response(State, nil, {nil, nil}, []);
274        {ok, http_eoh} ->
275            dlhttpc_sock:setopts(Socket, [{packet, raw}], Ssl),
276            Response = handle_response_body(State, Vsn, Status, Hdrs),
277            NewHdrs = element(2, Response),
278            ReqHdrs = State#client_state.request_headers,
279            NewSocket = maybe_close_socket(Socket, Ssl, Vsn, ReqHdrs, NewHdrs),
280            {Response, NewSocket};
281        {error, closed} ->
282            % Either we only noticed that the socket was closed after we
283            % sent the request, the server closed it just after we put
284            % the request on the wire or the server has some issues and is
285            % closing connections without sending responses.
286            % If this the first attempt to send the request, we will try again.
287            dlhttpc_sock:close(Socket, Ssl),
288            NewState = State#client_state{
289                socket = undefined,
290                attempts = State#client_state.attempts - 1
291            },
292            send_request(NewState);
293        {error, timeout} ->
294            dlhttpc_sock:close(Socket, Ssl),
295            NewState = State#client_state{
296                socket = undefined,
297                attempts = 0
298            },
299            send_request(NewState)
300    end.
301
302handle_response_body(#client_state{partial_download = false} = State, Vsn,
303        Status, Hdrs) ->
304    Socket = State#client_state.socket,
305    Ssl = State#client_state.ssl,
306    Method = State#client_state.method,
307    {Body, NewHdrs} = case has_body(Method, element(1, Status), Hdrs) of
308        true  -> read_body(Vsn, Hdrs, Ssl, Socket, body_type(Hdrs));
309        false -> {<<>>, Hdrs}
310    end,
311    {Status, NewHdrs, Body};
312handle_response_body(#client_state{partial_download = true} = State, Vsn,
313        Status, Hdrs) ->
314    Method = State#client_state.method,
315    case has_body(Method, element(1, Status), Hdrs) of
316        true ->
317            Response = {ok, {Status, Hdrs, self()}},
318            State#client_state.requester ! {response, State#client_state.req_id, self(), Response},
319            MonRef = erlang:monitor(process, State#client_state.requester),
320            Res = read_partial_body(State, Vsn, Hdrs, body_type(Hdrs)),
321            erlang:demonitor(MonRef, [flush]),
322            Res;
323        false ->
324            {Status, Hdrs, undefined}
325    end.
326
327has_body("HEAD", _, _) ->
328    % HEAD responses aren't allowed to include a body
329    false;
330has_body("OPTIONS", _, Hdrs) ->
331    % OPTIONS can include a body, if Content-Length or Transfer-Encoding
332    % indicates it
333    ContentLength = dlhttpc_lib:header_value("content-length", Hdrs),
334    TransferEncoding = dlhttpc_lib:header_value("transfer-encoding", Hdrs),
335    case {ContentLength, TransferEncoding} of
336        {undefined, undefined} -> false;
337        {_, _}                 -> true
338    end;
339has_body(_, 204, _) ->
340    false; % RFC 2616 10.2.5: 204 No Content
341has_body(_, 304, _) ->
342    false; % RFC 2616 10.3.5: 304 Not Modified
343has_body(_, _, _) ->
344    true. % All other responses are assumed to have a body
345
346body_type(Hdrs) ->
347    % Find out how to read the entity body from the request.
348    % * If we have a Content-Length, just use that and read the complete
349    %   entity.
350    % * If Transfer-Encoding is set to chunked, we should read one chunk at
351    %   the time
352    % * If neither of this is true, we need to read until the socket is
353    %   closed (AFAIK, this was common in versions before 1.1).
354    case dlhttpc_lib:header_value("content-length", Hdrs) of
355        undefined ->
356            TransferEncoding = string:to_lower(
357                dlhttpc_lib:header_value("transfer-encoding", Hdrs, "undefined")
358            ),
359            case TransferEncoding of
360                "chunked" -> chunked;
361                _         -> infinite
362            end;
363        ContentLength ->
364            {fixed_length, list_to_integer(ContentLength)}
365    end.
366
367read_partial_body(State, _Vsn, Hdrs, chunked) ->
368    Window = State#client_state.download_window,
369    read_partial_chunked_body(State, Hdrs, Window, 0, [], 0);
370read_partial_body(State, Vsn, Hdrs, infinite) ->
371    check_infinite_response(Vsn, Hdrs),
372    read_partial_infinite_body(State, Hdrs, State#client_state.download_window);
373read_partial_body(State, _Vsn, Hdrs, {fixed_length, ContentLength}) ->
374    read_partial_finite_body(State, Hdrs, ContentLength,
375        State#client_state.download_window).
376
377read_body(_Vsn, Hdrs, Ssl, Socket, chunked) ->
378    read_chunked_body(Socket, Ssl, Hdrs, []);
379read_body(Vsn, Hdrs, Ssl, Socket, infinite) ->
380    check_infinite_response(Vsn, Hdrs),
381    read_infinite_body(Socket, Hdrs, Ssl);
382read_body(_Vsn, Hdrs, Ssl, Socket, {fixed_length, ContentLength}) ->
383    read_length(Hdrs, Ssl, Socket, ContentLength).
384
385read_partial_finite_body(State = #client_state{}, Hdrs, 0, _Window) ->
386    reply_end_of_body(State, [], Hdrs);
387read_partial_finite_body(State = #client_state{requester = To}, Hdrs,
388        ContentLength, 0) ->
389    receive
390        {ack, To} ->
391            read_partial_finite_body(State, Hdrs, ContentLength, 1);
392        {'DOWN', _, process, To, _} ->
393            exit(normal)
394    end;
395read_partial_finite_body(State, Hdrs, ContentLength, Window) when Window >= 0->
396    Bin = read_body_part(State, ContentLength),
397    State#client_state.requester ! {body_part, self(), Bin},
398    To = State#client_state.requester,
399    receive
400        {ack, To} ->
401            Length = ContentLength - iolist_size(Bin),
402            read_partial_finite_body(State, Hdrs, Length, Window);
403        {'DOWN', _, process, To, _} ->
404            exit(normal)
405    after 0 ->
406            Length = ContentLength - iolist_size(Bin),
407        read_partial_finite_body(State, Hdrs, Length, dlhttpc_lib:dec(Window))
408    end.
409
410read_body_part(#client_state{part_size = infinity} = State, _ContentLength) ->
411    case dlhttpc_sock:recv(State#client_state.socket, State#client_state.ssl) of
412        {ok, Data} ->
413            Data;
414        {error, Reason} ->
415            erlang:error(Reason)
416    end;
417read_body_part(#client_state{part_size = PartSize} = State, ContentLength)
418        when PartSize =< ContentLength ->
419    Socket = State#client_state.socket,
420    Ssl = State#client_state.ssl,
421    PartSize = State#client_state.part_size,
422    case dlhttpc_sock:recv(Socket, PartSize, Ssl) of
423        {ok, Data} ->
424            Data;
425        {error, Reason} ->
426            erlang:error(Reason)
427    end;
428read_body_part(#client_state{part_size = PartSize} = State, ContentLength)
429        when PartSize > ContentLength ->
430    Socket = State#client_state.socket,
431    Ssl = State#client_state.ssl,
432    case dlhttpc_sock:recv(Socket, ContentLength, Ssl) of
433        {ok, Data} ->
434            Data;
435        {error, Reason} ->
436            erlang:error(Reason)
437    end.
438
439read_length(Hdrs, Ssl, Socket, Length) ->
440    case dlhttpc_sock:recv(Socket, Length, Ssl) of
441        {ok, Data} ->
442            {Data, Hdrs};
443        {error, Reason} ->
444            erlang:error(Reason)
445    end.
446
447read_partial_chunked_body(State, Hdrs, Window, BufferSize, Buffer, 0) ->
448    Socket = State#client_state.socket,
449    Ssl = State#client_state.ssl,
450    PartSize = State#client_state.part_size,
451    case read_chunk_size(Socket, Ssl) of
452        0 ->
453            reply_chunked_part(State, Buffer, Window),
454            {Trailers, NewHdrs} = read_trailers(Socket, Ssl, [], Hdrs),
455            reply_end_of_body(State, Trailers, NewHdrs);
456        ChunkSize when PartSize =:= infinity ->
457            Chunk = read_chunk(Socket, Ssl, ChunkSize),
458            NewWindow = reply_chunked_part(State, [Chunk | Buffer], Window),
459            read_partial_chunked_body(State, Hdrs, NewWindow, 0, [], 0);
460        ChunkSize when BufferSize + ChunkSize >= PartSize ->
461            {Chunk, RemSize} = read_partial_chunk(Socket, Ssl,
462                PartSize - BufferSize, ChunkSize),
463            NewWindow = reply_chunked_part(State, [Chunk | Buffer], Window),
464            read_partial_chunked_body(State, Hdrs, NewWindow, 0, [], RemSize);
465        ChunkSize ->
466            Chunk = read_chunk(Socket, Ssl, ChunkSize),
467            read_partial_chunked_body(State, Hdrs, Window,
468                BufferSize + ChunkSize, [Chunk | Buffer], 0)
469    end;
470read_partial_chunked_body(State, Hdrs, Window, BufferSize, Buffer, RemSize) ->
471    Socket = State#client_state.socket,
472    Ssl = State#client_state.ssl,
473    PartSize = State#client_state.part_size,
474    if
475        BufferSize + RemSize >= PartSize ->
476            {Chunk, NewRemSize} =
477                read_partial_chunk(Socket, Ssl, PartSize - BufferSize, RemSize),
478            NewWindow = reply_chunked_part(State, [Chunk | Buffer], Window),
479            read_partial_chunked_body(State, Hdrs, NewWindow, 0, [],
480                NewRemSize);
481        BufferSize + RemSize < PartSize ->
482            Chunk = read_chunk(Socket, Ssl, RemSize),
483            read_partial_chunked_body(State, Hdrs, Window, BufferSize + RemSize,
484                [Chunk | Buffer], 0)
485    end.
486
487read_chunk_size(Socket, Ssl) ->
488    dlhttpc_sock:setopts(Socket, [{packet, line}], Ssl),
489    case dlhttpc_sock:recv(Socket, Ssl) of
490        {ok, ChunkSizeExt} ->
491            chunk_size(ChunkSizeExt);
492        {error, Reason} ->
493            erlang:error(Reason)
494    end.
495
496reply_chunked_part(_State, [], Window) ->
497    Window;
498reply_chunked_part(State = #client_state{requester = Pid}, Buff, 0) ->
499    receive
500        {ack, Pid} ->
501            reply_chunked_part(State, Buff, 1);
502        {'DOWN', _, process, Pid, _} ->
503            exit(normal)
504    end;
505reply_chunked_part(#client_state{requester = Pid}, Buffer, Window) ->
506    Pid ! {body_part, self(), list_to_binary(lists:reverse(Buffer))},
507    receive
508        {ack, Pid} ->  Window;
509        {'DOWN', _, process, Pid, _} -> exit(normal)
510    after 0 ->
511        dlhttpc_lib:dec(Window)
512    end.
513
514read_chunked_body(Socket, Ssl, Hdrs, Chunks) ->
515    case read_chunk_size(Socket, Ssl) of
516        0 ->
517            Body = list_to_binary(lists:reverse(Chunks)),
518            {_, NewHdrs} = read_trailers(Socket, Ssl, [], Hdrs),
519            {Body, NewHdrs};
520        Size ->
521            Chunk = read_chunk(Socket, Ssl, Size),
522            read_chunked_body(Socket, Ssl, Hdrs, [Chunk | Chunks])
523    end.
524
525chunk_size(Bin) ->
526    erlang:list_to_integer(lists:reverse(chunk_size(Bin, [])), 16).
527
528chunk_size(<<$;, _/binary>>, Chars) ->
529    Chars;
530chunk_size(<<"\r\n", _/binary>>, Chars) ->
531    Chars;
532chunk_size(<<$\s, Binary/binary>>, Chars) ->
533    %% Facebook's HTTP server returns a chunk size like "6  \r\n"
534    chunk_size(Binary, Chars);
535chunk_size(<<Char, Binary/binary>>, Chars) ->
536    chunk_size(Binary, [Char | Chars]).
537
538read_partial_chunk(Socket, Ssl, ChunkSize, ChunkSize) ->
539    {read_chunk(Socket, Ssl, ChunkSize), 0};
540read_partial_chunk(Socket, Ssl, Size, ChunkSize) ->
541    dlhttpc_sock:setopts(Socket, [{packet, raw}], Ssl),
542    case dlhttpc_sock:recv(Socket, Size, Ssl) of
543        {ok, Chunk} ->
544            {Chunk, ChunkSize - Size};
545        {error, Reason} ->
546            erlang:error(Reason)
547    end.
548
549read_chunk(Socket, Ssl, Size) ->
550    dlhttpc_sock:setopts(Socket, [{packet, raw}], Ssl),
551    case dlhttpc_sock:recv(Socket, Size + 2, Ssl) of
552        {ok, <<Chunk:Size/binary, "\r\n">>} ->
553            Chunk;
554        {ok, Data} ->
555            erlang:error({invalid_chunk, Data});
556        {error, Reason} ->
557            erlang:error(Reason)
558    end.
559
560read_trailers(Socket, Ssl, Trailers, Hdrs) ->
561    dlhttpc_sock:setopts(Socket, [{packet, httph}], Ssl),
562    case dlhttpc_sock:recv(Socket, Ssl) of
563        {ok, http_eoh} ->
564            {Trailers, Hdrs};
565        {ok, {http_header, _, Name, _, Value}} ->
566            Header = {dlhttpc_lib:maybe_atom_to_list(Name), Value},
567            read_trailers(Socket, Ssl, [Header | Trailers], [Header | Hdrs]);
568        {error, {http_error, Data}} ->
569            erlang:error({bad_trailer, Data})
570    end.
571
572reply_end_of_body(#client_state{requester = Requester}, Trailers, Hdrs) ->
573    Requester ! {http_eob, self(), Trailers},
574    {no_return, Hdrs}.
575
576read_partial_infinite_body(State = #client_state{requester = To}, Hdrs, 0) ->
577    receive
578        {ack, To} ->
579            read_partial_infinite_body(State, Hdrs, 1);
580        {'DOWN', _, process, To, _} ->
581            exit(normal)
582    end;
583read_partial_infinite_body(State = #client_state{requester = To}, Hdrs, Window)
584        when Window >= 0 ->
585    case read_infinite_body_part(State) of
586        http_eob -> reply_end_of_body(State, [], Hdrs);
587        Bin ->
588            State#client_state.requester ! {body_part, self(), Bin},
589            receive
590                {ack, To} ->
591                    read_partial_infinite_body(State, Hdrs, Window);
592                {'DOWN', _, process, To, _} ->
593                    exit(normal)
594            after 0 ->
595                read_partial_infinite_body(State, Hdrs, dlhttpc_lib:dec(Window))
596            end
597    end.
598
599read_infinite_body_part(#client_state{socket = Socket, ssl = Ssl}) ->
600    case dlhttpc_sock:recv(Socket, Ssl) of
601        {ok, Data} ->
602            Data;
603        {error, closed} ->
604            http_eob;
605        {error, Reason} ->
606            erlang:error(Reason)
607    end.
608
609check_infinite_response({1, Minor}, Hdrs) when Minor >= 1 ->
610    HdrValue = dlhttpc_lib:header_value("connection", Hdrs, "keep-alive"),
611    case string:to_lower(HdrValue) of
612        "close" -> ok;
613        _       -> erlang:error(no_content_length)
614    end;
615check_infinite_response(_, Hdrs) ->
616    HdrValue = dlhttpc_lib:header_value("connection", Hdrs, "close"),
617    case string:to_lower(HdrValue) of
618        "keep-alive" -> erlang:error(no_content_length);
619        _            -> ok
620    end.
621
622read_infinite_body(Socket, Hdrs, Ssl) ->
623    read_until_closed(Socket, <<>>, Hdrs, Ssl).
624
625read_until_closed(Socket, Acc, Hdrs, Ssl) ->
626    case dlhttpc_sock:recv(Socket, Ssl) of
627        {ok, Body} ->
628            NewAcc = <<Acc/binary, Body/binary>>,
629            read_until_closed(Socket, NewAcc, Hdrs, Ssl);
630        {error, closed} ->
631            {Acc, Hdrs};
632        {error, Reason} ->
633            erlang:error(Reason)
634    end.
635
636maybe_close_socket(Socket, Ssl, {1, Minor}, ReqHdrs, RespHdrs) when Minor >= 1->
637    ClientConnection = ?CONNECTION_HDR(ReqHdrs, "keep-alive"),
638    ServerConnection = ?CONNECTION_HDR(RespHdrs, "keep-alive"),
639    if
640        ClientConnection =:= "close"; ServerConnection =:= "close" ->
641            dlhttpc_sock:close(Socket, Ssl),
642            undefined;
643        ClientConnection =/= "close", ServerConnection =/= "close" ->
644            Socket
645    end;
646maybe_close_socket(Socket, Ssl, _, ReqHdrs, RespHdrs) ->
647    ClientConnection = ?CONNECTION_HDR(ReqHdrs, "keep-alive"),
648    ServerConnection = ?CONNECTION_HDR(RespHdrs, "close"),
649    if
650        ClientConnection =:= "close"; ServerConnection =/= "keep-alive" ->
651            dlhttpc_sock:close(Socket, Ssl),
652            undefined;
653        ClientConnection =/= "close", ServerConnection =:= "keep-alive" ->
654            Socket
655    end.