Erlang Twitter streaming client - handling of chunked responses - twitter

Question

I'm connecting to the Twitter firehose using the Erlang code at the bottom.
Now I am receiving a stream of data fine but am wondering if the Erlang httpc client is handling the CRLF ('\r\n') chunked response delimiter properly; because I was expecting a series of calls to the three handle_info blocks in turn (stream_start, stream*, stream_end) -
handle_info({http, {_RequestId, stream_start, _Headers}}, State) ->
io:format("start~n"),
{noreply, State};
handle_info({http, {_RequestId, stream, Data}}, State) ->
io:format("~p~n", [Data]),
{noreply, State};
handle_info({http, {_RequestId, stream_end, _Headers}}, State) ->
io:format("end~n"),
{noreply, State};
but instead what happens is that the 'stream_start' block is called once at the outset, and then all subsequent data is handled by the 'stream' block; 'stream_end' is never called.
However when I look at the blocks being handled by the 'stream' block, a very large number have the CRLF delimiter as a suffix.
So am wondering if the httpc client is handling chunked block termination correctly; or maybe I haven't configured it properly ?
TIA
%% https://dev.twitter.com/streaming/reference/post/statuses/filter
-module(twitter_streaming_demo).
-behaviour(gen_server).
%% API.
-export([start_link/5]).
%% gen_server.
-export([init/1]).
-export([handle_call/3]).
-export([handle_cast/2]).
-export([handle_info/2]).
-export([terminate/2]).
-export([code_change/3]).
-define(METHOD, "POST").
-define(URL, "https://stream.twitter.com/1.1/statuses/filter.json").
-define(APPLICATION_FORM_URLENCODED, "application/x-www-form-urlencoded").
-define(TRACK, "track").
-record(state, {consumer,
tokens,
url,
query,
request_id}).
%% API.
%% twitter_streaming_demo:start_link("", "", "", "", "").
start_link(ConsumerKey, ConsumerSecret, Token, TokenSecret, Query) ->
gen_server:start_link(?MODULE, [ConsumerKey, ConsumerSecret, Token, TokenSecret, Query], []).
%% gen_server.
init([ConsumerKey, ConsumerSecret, Token, TokenSecret, Query]) ->
Consumer={ConsumerKey, ConsumerSecret, hmac_sha1},
Tokens={Token, TokenSecret},
{ok, #state{consumer=Consumer,
tokens=Tokens,
url=?URL,
query=Query}, 0}.
handle_call(_Request, _From, State) ->
{reply, ignored, State}.
handle_cast(_Msg, State) ->
{noreply, State}.
handle_info(timeout, #state{consumer=Consumer, tokens=Tokens, url=Url, query=Query}=State) ->
{Token, TokenSecret}=Tokens,
Params=[{?TRACK, Query}],
Signed=oauth:sign("POST", Url, Params, Consumer, Token, TokenSecret),
{AuthorizationParams, _QueryParams}=lists:partition(fun({K, _}) -> lists:prefix("oauth_", K) end, Signed),
Request={oauth:uri(Url, []), %% it's a POST request :-)
[oauth:header(AuthorizationParams)],
?APPLICATION_FORM_URLENCODED,
?TRACK++"="++Query},
{ok, RequestId}=httpc:request(post, Request, [], [{sync, false}, {stream, self}]),
{noreply, State#state{request_id=RequestId}};
handle_info({http, {_RequestId, stream_start, _Headers}}, State) ->
io:format("start~n"),
{noreply, State};
handle_info({http, {_RequestId, stream, Data}}, State) ->
io:format("~p~n", [Data]),
{noreply, State};
handle_info({http, {_RequestId, stream_end, _Headers}}, State) ->
io:format("end~n"),
{noreply, State};
handle_info(Info, State) ->
io:format("~p~n", [Info]),
{noreply, State}.
terminate(_Reason, _State) ->
ok.
code_change(_OldVsn, State, _Extra) ->
{ok, State}.

Answer 1

It looks like the httpc client in OTP should handle chunked encoding (bugs notwithstanding).
CRLF delimiters are valid inside chunks if they are inside the length scope of the chunk, i.e. if the length is after the position of the CRLF. Perhaps those are the line feeds you are seeing? Example from Wikipedia:
e\r\n
in\r\n\r\nchunks.\r\n
Here, the length (e or 14) encompasses the CRLFs (which has a length of 2) between in and chunks. (the trailing line feeds are never counted towards the length).
As for the Twitter API (which I'm not familiar with), it could be possible that it never returns (and just gives you chunks forever)?