Skip to content

Commit

Permalink
HELP-41845: generate CSVs providing a consistent cell ordering (2600h…
Browse files Browse the repository at this point in the history 
…z#5496)

* HELP-41845: generate CSVs providing a consistent cell ordering

Rather than ouputting rows in an ad-hoc manner, apply a consistent
ordering of JSON paths to cells when processing JSON objects that may
vary in the paths present.

* return 204 no content for empty CSVs

* logline updates

* not really an error anymore

* type cleanup

* handle already-encoded CSVs

* log download

* more tests for CSV fetching

* if we have an acc use it

* HELP-41845: handle customized csv chunk response

* maybe not setting headers?

* let not set content-disposition to a file path

* make fmt
  • Loading branch information
@jamesaimonetti @icehess
jamesaimonetti authored and icehess committed Feb 19, 2019
1 parent 16e628b commit 24a01cc
Show file tree
Hide file tree
Showing 21 changed files with 956 additions and 246 deletions.
107 changes: 59 additions & 48 deletions applications/crossbar/src/api_resource.erl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@
,content_types_provided/2
,content_types_accepted/2
,languages_provided/2
,charsets_provided/2
,resource_exists/2
,moved_temporarily/2
,moved_permanently/2
Expand Down Expand Up @@ -288,7 +287,7 @@ pretty_metric(N, 'true') ->
%% @end
%%------------------------------------------------------------------------------
-spec known_methods(cowboy_req:req(), cb_context:context()) ->
{http_methods(), cowboy_req:req(), cb_context:context()}.
{http_methods() | 'stop', cowboy_req:req(), cb_context:context()}.
known_methods(Req, Context) ->
case cb_context:resp_status(Context) of
'stop' ->
Expand Down Expand Up @@ -496,9 +495,10 @@ options(Req0, Context) ->
{'ok', Req0, Context}
end.

-type content_type_callbacks() :: [{{kz_term:ne_binary(), kz_term:ne_binary(), kz_term:proplist()}, atom()} |
-type content_type_callbacks() :: [{{kz_term:ne_binary(), kz_term:ne_binary(), '*' | kz_term:proplist()}, atom()} |
{kz_term:ne_binary(), atom()}
].

-spec content_types_provided(cowboy_req:req(), cb_context:context()) ->
{content_type_callbacks(), cowboy_req:req(), cb_context:context()}.
content_types_provided(Req, Context0) ->
Expand All @@ -524,7 +524,10 @@ content_types_provided(Req, Context, CTPs) ->
[ {EncType, Fun} | Acc1 ];
(CT, Acc1) when is_binary(CT) ->
[{CT, Fun} | Acc1]
end, Acc, L)
end
,Acc
,L
)
end
,[]
,CTPs
Expand Down Expand Up @@ -652,10 +655,6 @@ languages_provided(Req0, Context0) ->
{cb_context:languages_provided(Context1) ++ [A], Req0, Context1}
end.

-spec charsets_provided(cowboy_req:req(), cb_context:context()) -> 'no_call'.
charsets_provided(_Req, _Context) ->
'no_call'.

-spec resource_exists(cowboy_req:req(), cb_context:context()) ->
{boolean(), cowboy_req:req(), cb_context:context()}.
resource_exists(Req, Context) ->
Expand Down Expand Up @@ -803,11 +802,8 @@ create_from_response(Req, Context) ->
create_from_response(Req, Context, 'undefined') ->
create_from_response(Req, Context, <<"*/*">>);
create_from_response(Req, Context, Accept) ->
CTPs = [F || {F, _} <- cb_context:content_types_provided(Context)],
DefaultFun = case CTPs of
[] -> 'to_json';
[F|_] -> F
end,
DefaultFun = content_type_provided_fun(Context),

case to_fun(Context, Accept, DefaultFun) of
'to_json' -> api_util:create_push_response(Req, Context);
'send_file' -> api_util:create_push_response(Req, Context, fun api_util:create_resp_file/2);
Expand All @@ -816,6 +812,12 @@ create_from_response(Req, Context, Accept) ->
api_util:create_push_response(Req, Context)
end.

content_type_provided_fun(Context) ->
case cb_context:content_types_provided(Context) of
[{F, _}|_] -> F;
[] -> 'to_json'
end.

-spec to_json(cowboy_req:req(), cb_context:context()) ->
{iolist() | kz_term:ne_binary() | 'stop', cowboy_req:req(), cb_context:context()}.
to_json(Req, Context) ->
Expand Down Expand Up @@ -877,6 +879,7 @@ to_binary(Req, Context, 'undefined') ->
],
NewContext = cb_context:setters(Context, Setters),
%% Respond, possibly with 206
lager:debug("replying with ~p", [ErrorCode]),
Req1 = cowboy_req:reply(kz_term:to_binary(ErrorCode), cb_context:resp_headers(NewContext), Content, Req),
{'stop', Req1, NewContext}
end;
Expand Down Expand Up @@ -950,12 +953,16 @@ accept_matches_provided(Major, Minor, CTPs) ->
{iolist(), cowboy_req:req(), cb_context:context()}.
to_csv(Req0, Context0) ->
case cb_context:fetch(Context0, 'is_chunked') of
'true' -> to_chunk(<<"to_csv">>, Req0, Context0);
'true' ->
to_chunk(<<"to_csv">>, Req0, cb_context:add_resp_header(Context0, <<"content-type">>, <<"text/csv">>));
_ ->
lager:debug("run: to_csv"),
Event = to_fun_event_name(<<"to_csv">>, Context0),
{Req1, Context1} = crossbar_bindings:fold(Event, {Req0, Context0}),
api_util:create_pull_response(Req1, Context1, fun api_util:create_csv_resp_content/2)
api_util:create_pull_response(Req1
,cb_context:add_resp_header(Context1, <<"content-type">>, <<"text/csv">>)
,fun api_util:create_csv_resp_content/2
)
end.

-spec to_pdf(cowboy_req:req(), cb_context:context()) ->
Expand Down Expand Up @@ -1011,28 +1018,33 @@ next_chunk_fold(#{chunking_started := StartedChunk
,context := Context0
,chunk_response_type := _ToFun
}=ChunkMap0) ->
lager:debug("(chunked) calling next chunk"),
lager:debug("(chunked) calling next chunk ~s", [_ToFun]),
Context1 = cb_context:store(Context0, 'chunking_started', StartedChunk),
ChunkMap1 = #{context := Context2
,cowboy_req := Req0
,event_name := Event
} = crossbar_view:next_chunk(ChunkMap0#{context := Context1}),
ChunkMap1 = #{context := Context2} = crossbar_view:next_chunk(ChunkMap0#{context := Context1}),

case api_util:succeeded(Context2)
andalso crossbar_bindings:fold(Event, {Req0, Context2})
of
next_chunk_results(ChunkMap1, api_util:succeeded(Context2)).

next_chunk_results(#{context := Context
,chunking_started := StartedChunk
}=ChunkMap, 'false') ->
lager:debug("(chunked) getting next chunk was unsuccessful"),
finish_chunked_response(ChunkMap#{context => reset_context_between_chunks(Context, StartedChunk)});
next_chunk_results(#{context := Context0
,cowboy_req := Req0
,chunk_response_type := _ToFun
,chunking_started := StartedChunk
,event_name := Event
}=ChunkMap, 'true') ->
{Req1, Context1} = crossbar_bindings:fold(Event, {Req0, Context0}),
lager:debug("(chunked) ran '~s'", [_ToFun]),
case api_util:succeeded(Context1) of
'true' ->
process_chunk(ChunkMap#{cowboy_req := Req1
,context := Context1
});
'false' ->
lager:debug("(chunked) getting next chunk was unsuccessful"),
finish_chunked_response(ChunkMap1#{context => reset_context_between_chunks(Context2, StartedChunk)});
{Req1, Context3} ->
lager:debug("(chunked) ran '~s'", [_ToFun]),
case api_util:succeeded(Context3) of
'true' ->
process_chunk(ChunkMap1#{cowboy_req := Req1, context := Context3});
'false' ->
lager:debug("(chunked) '~s' was unsuccessful", [_ToFun]),
finish_chunked_response(ChunkMap1#{context => reset_context_between_chunks(Context3, StartedChunk)})
end
lager:debug("(chunked) '~s' was unsuccessful", [_ToFun]),
finish_chunked_response(ChunkMap#{context => reset_context_between_chunks(Context1, StartedChunk)})
end.

%%------------------------------------------------------------------------------
Expand Down Expand Up @@ -1077,28 +1089,33 @@ process_chunk(#{context := Context
}=ChunkMap) ->
case cb_context:resp_data(Context) of
0 ->
lager:debug("(chunked) ~s did not send data", [ToFun]),
next_chunk_fold(ChunkMap#{context => reset_context_between_chunks(Context, IsStarted)
,chunking_started => IsStarted
}
);
SentLength when is_integer(SentLength) ->
lager:debug("(chunked) ~s sent ~p data", [ToFun, SentLength]),
next_chunk_fold(ChunkMap#{context => reset_context_between_chunks(Context, 'true')
,chunking_started => 'true'
,previous_chunk_length => SentLength
}
);
[] ->
lager:debug("(chunked) ~s did not send data", [ToFun]),
next_chunk_fold(ChunkMap#{context => reset_context_between_chunks(Context, IsStarted)
,chunking_started => IsStarted
,previous_chunk_length => 0 %% the module filtered all queried result
}
);
Resp when is_list(Resp) ->
{StartedChunk, Req1} = send_chunk_response(ToFun, Req, Context),
next_chunk_fold(ChunkMap#{context => reset_context_between_chunks(Context, StartedChunk)
PrevLength = length(Resp),
{StartedChunk, Req1, Context1} = send_chunk_response(ToFun, Req, Context),
lager:debug("(chunked) ~s sent ~p chunked resp", [ToFun, PrevLength]),
next_chunk_fold(ChunkMap#{context => reset_context_between_chunks(Context1, StartedChunk)
,cowboy_req => Req1
,chunking_started => StartedChunk
,previous_chunk_length => length(Resp)
,previous_chunk_length => PrevLength
}
);
_Other ->
Expand Down Expand Up @@ -1141,10 +1158,11 @@ reset_context_between_chunks(Context, _StartedChunk, 'false') ->
end.

-spec send_chunk_response(kz_term:ne_binary(), cowboy_req:req(), cb_context:context()) ->
{boolean(), cowboy_req:req()}.
{boolean(), cowboy_req:req(), cb_context:context()}.
send_chunk_response(<<"to_json">>, Req, Context) ->
api_util:create_json_chunk_response(Req, Context);
send_chunk_response(<<"to_csv">>, Req, Context) ->
lager:debug("creating CSV chunk"),
api_util:create_csv_chunk_response(Req, Context).

%%------------------------------------------------------------------------------
Expand All @@ -1161,20 +1179,13 @@ finish_chunked_response(#{chunking_started := 'false'
}) ->
%% chunk is not started, return whatever error's or response data in Context
api_util:create_pull_response(Req, Context);
finish_chunked_response(#{chunking_started := 'false'
,chunk_response_type := <<"to_csv">>
finish_chunked_response(#{chunk_response_type := <<"to_csv">>
,context := Context
,cowboy_req := Req
}) ->
%% chunk is not started, return empty CSV
lager:debug("creating CSV pull response"),
api_util:create_pull_response(Req, Context, fun api_util:create_csv_resp_content/2);
finish_chunked_response(#{chunk_response_type := <<"to_csv">>
,context := Context
,cowboy_req := Req
}) ->
%% Chunk is already started, stopping,
'ok' = cowboy_req:stream_body(<<>>, 'fin', Req),
{'stop', Req, Context};
finish_chunked_response(#{total_queried := TotalQueried
,chunking_started := 'true'
,cowboy_req := Req
Expand Down Expand Up @@ -1209,7 +1220,7 @@ multiple_choices(Req, Context) ->
{'false', Req, Context}.

-spec generate_etag(cowboy_req:req(), cb_context:context()) ->
{kz_term:api_ne_binary(), cowboy_req:req(), cb_context:context()}.
{kz_term:ne_binary(), cowboy_req:req(), cb_context:context()}.
generate_etag(Req0, Context0) ->
Event = api_util:create_event_name(Context0, <<"etag">>),
{Req1, Context1} = crossbar_bindings:fold(Event, {Req0, Context0}),
Expand Down
Loading

0 comments on commit 24a01cc

Please sign in to comment.