Crossbar fixes related to Cowboy upgrades (2600hz#5920)

Includes:

* checks for whether a request envelope is requried for a given API
endpoint request.
* namespace authorize and authenticate bindings (reduces number of
modules checked)
* Adds loopback URI to call-fowarded endpoints
* cleans up sendfile usage

applications/crossbar/src/api_resource.erl

@@ -10,6 +10,7 @@
 -behaviour(cowboy_rest).
 
 -export([init/2, rest_init/2
+        ,service_available/2
         ,terminate/3
         ,known_methods/2
         ,allowed_methods/2
@@ -38,6 +39,7 @@
         ,to_csv/2
         ,to_pdf/2
         ,to_xml/2
+        ,to_custom/2
         ,send_file/2
 
         ,from_json/2, from_binary/2, from_form/2
@@ -57,6 +59,15 @@
 %% @doc Initialize a REST request.
 %% @end
 %%------------------------------------------------------------------------------
+-spec service_available(cowboy_req:req(), kz_term:proplist() | cb_context:context()) ->
+                               {'true', cowboy_req:req(), cb_context:context()}.
+service_available(Req0, Opts)
+  when is_list(Opts) ->
+    {'cowboy_rest', Req, Context} = rest_init(Req0, Opts),
+    {'true', Req, Context};
+service_available(Req, Context) ->
+    {'true', Req, Context}.
+
 -spec init(cowboy_req:req(), kz_term:proplist()) ->
                   {'cowboy_rest', cowboy_req:req(), cb_context:context()}.
 init(Req, Opts) ->
@@ -71,7 +82,7 @@ rest_init(Req, Opts) ->
 
     Setters = [{fun cb_context:set_req_id/2, get_request_id(Req)}
               ,{fun cb_context:set_req_headers/2, cowboy_req:headers(Req)}
-              ,{fun cb_context:set_host_url/2, kz_term:to_binary(cowboy_req:uri(Req))}
+              ,{fun host_url/2, Req}
               ,{fun cb_context:set_port/2, kz_term:to_integer(cowboy_req:port(Req))}
               ,{fun cb_context:set_raw_path/2, kz_term:to_binary(Path)}
               ,{fun cb_context:set_raw_qs/2, kz_term:to_binary(cowboy_req:qs(Req))}
@@ -84,6 +95,7 @@ rest_init(Req, Opts) ->
               ,{fun cb_context:set_api_version/2, find_version(Path, Req)}
               ,{fun cb_context:set_magic_pathed/2, props:is_defined('magic_path', Opts)}
               ,{fun cb_context:store/3, 'metrics', metrics()}
+              ,fun req_nouns/1
               ],
 
     Context0 = cb_context:setters(cb_context:new(), Setters),
@@ -113,6 +125,21 @@ rest_init(Req, Opts) ->
             }
     end.
 
+-spec host_url(cb_context:context(), cowboy_req:req()) -> cb_context:context().
+host_url(Context, Req) ->
+    URI = kz_term:to_binary(cowboy_req:uri(Req)),
+    {Scheme, Location, _Path, _Query, _Frag} = kz_http_util:urlsplit(URI),
+    Value = list_to_binary([Scheme, "://", Location]),
+    cb_context:set_host_url(Context, Value).
+
+-spec req_nouns(cb_context:context()) -> cb_context:context().
+req_nouns(Context) ->
+    Tokens = api_util:path_tokens(Context),
+    case api_util:parse_path_tokens(Context, Tokens) of
+        [_|_] = Nouns -> cb_context:set_req_nouns(Context, Nouns);
+        _Else -> Context
+    end.
+
 -spec get_request_id(cowboy_req:req()) -> kz_term:ne_binary().
 get_request_id(Req) ->
     ReqId = case cowboy_req:header(<<"x-request-id">>, Req) of
@@ -442,8 +469,7 @@ validate_account_resource(Context, AccountArgs) ->
     apply('cb_accounts', 'validate_resource', [Context | AccountArgs]).
 
 -spec is_authorized(cowboy_req:req(), cb_context:context()) ->
-                           {'true' | {'false', <<>>}, cowboy_req:req(), cb_context:context()} |
-                           api_util:stop_return().
+                           {'true' | {'false', <<>>} | 'stop', cowboy_req:req(), cb_context:context()}.
 is_authorized(Req, Context) ->
     api_util:is_authentic(Req, Context).
 
@@ -806,6 +832,7 @@ create_from_response(Req, Context, Accept) ->
         'to_json' -> api_util:create_push_response(Req, Context);
         'send_file' -> api_util:create_push_response(Req, Context, fun api_util:create_resp_file/2);
         'to_binary' -> api_util:create_push_response(Req, Context, fun api_util:create_binary_resp_content/2);
+        'to_custom' -> to_custom(Req, Context);
         'to_xml' -> api_util:create_push_response(Req, Context, fun api_util:create_xml_resp_content/2);
         _Else ->
             %% sending json for now until we implement other types
@@ -847,6 +874,16 @@ to_json(Req, Context, Accept) ->
             (?MODULE):Fun(Req, Context)
     end.
 
+-spec to_custom(cowboy_req:req(), cb_context:context()) -> {'stop', cowboy_req:req(), cb_context:context()}.
+to_custom(Req0, Context0) ->
+    lager:debug("run: to_custom"),
+    [{Mod, Params}|_] = cb_context:req_nouns(Context0),
+    Verb = cb_context:req_verb(Context0),
+    Event = api_util:create_event_name(Context0, [<<"to_custom">>, kz_term:to_lower_binary(Verb), Mod]),
+    Payload = [{Req0, Context0} | Params],
+    {Req, Context} = crossbar_bindings:fold(Event, Payload),
+    {'stop', Req, Context}.
+
 -spec to_binary(cowboy_req:req(), cb_context:context()) ->
                        {binary() | 'stop', cowboy_req:req(), cb_context:context()}.
 to_binary(Req, Context) ->

applications/crossbar/src/api_util.erl

@@ -76,17 +76,9 @@
 -define(DEFAULT_CSV_FILE_NAME, <<"result.csv">>).
 
 -type stop_return() :: {'stop', cowboy_req:req(), cb_context:context()}.
--type resp_file() :: {integer(), send_file_fun()}.
+-type resp_file() :: {'sendfile',  non_neg_integer(),  non_neg_integer(), file:name_all()}.
 -type resp_content_return() :: {kz_term:ne_binary() | iolist() | resp_file(), cowboy_req:req()}.
 -type resp_content_fun() :: fun((cowboy_req:req(), cb_context:context()) ->  resp_content_return()).
--type send_file_fun() :: fun((any(), module()) -> 'ok').
--type pull_file_resp() :: {'stream', integer(), send_file_fun()}.
--type pull_file_response_return() :: {pull_file_resp(), cowboy_req:req(), cb_context:context()} |
-                                     stop_return().
-
--export_type([pull_file_response_return/0
-             ,stop_return/0
-             ]).
 
 %%------------------------------------------------------------------------------
 %% @doc Attempts to determine if this is a cross origin resource preflight request
@@ -262,7 +254,6 @@ handle_url_encoded_body(Context, Req, QS, ReqBody, JObj) ->
             try_json(ReqBody, QS, Context, Req);
         _Vs ->
             lager:debug("was able to parse request body as url-encoded to json: ~p", [JObj]),
-
             set_request_data_in_context(Context, Req, JObj, QS)
     end.
 
@@ -563,8 +554,34 @@ is_valid_request_envelope(Envelope, Context) ->
 
 -spec requires_envelope(cb_context:context()) -> boolean().
 requires_envelope(Context) ->
+    Routines = [fun api_version_requires_envelope/1
+               ,fun content_type_requires_envelope/1
+               ,fun req_noun_requires_envelope/1
+               ],
+    lists:all(fun(F) -> F(Context) end, Routines).
+
+-spec api_version_requires_envelope(cb_context:context()) -> boolean().
+api_version_requires_envelope(Context) ->
     not lists:member(cb_context:api_version(Context), ?NO_ENVELOPE_VERSIONS).
 
+-spec content_type_requires_envelope(cb_context:context()) -> boolean().
+content_type_requires_envelope(Context) ->
+    not lists:member(cb_context:req_header(Context, <<"content-type">>), ?NO_ENVELOPE_CONTENT_TYPES).
+
+-spec req_noun_requires_envelope(cb_context:context()) -> boolean().
+req_noun_requires_envelope(Context) ->
+    req_noun_requires_envelope(Context, cb_context:req_nouns(Context)).
+
+-spec req_noun_requires_envelope(cb_context:context(), req_nouns()) -> boolean().
+req_noun_requires_envelope(_Context, []) -> 'true';
+req_noun_requires_envelope(Context, [{Mod, Params} | _]) ->
+    Event = create_event_name(Context, <<"requires_envelope.", Mod/binary>>),
+    Payload = [Context | Params],
+    case crossbar_bindings:pmap(Event, Payload) of
+        [Value | _] -> not Value;
+        _Else -> 'true'
+    end.
+
 -spec validate_request_envelope(kz_json:object()) -> 'true' | validation_errors().
 validate_request_envelope(Envelope) ->
     case kz_json_schema:validate(?ENVELOPE_SCHEMA, Envelope) of
@@ -741,7 +758,11 @@ is_authentic(Req, Context, ?HTTP_OPTIONS) ->
     {'true', Req, Context};
 is_authentic(Req, Context0, _ReqVerb) ->
     Event = create_event_name(Context0, <<"authenticate">>),
-    case crossbar_bindings:succeeded(crossbar_bindings:pmap(Event, Context0)) of
+    case cb_context:auth_doc(Context0) =:= 'undefined'
+        andalso crossbar_bindings:succeeded(crossbar_bindings:pmap(Event, Context0))
+    of
+        'false' ->
+            {'true', Req, Context0};
         [] ->
             is_authentic(Req, Context0, _ReqVerb, cb_context:req_nouns(Context0));
         ['true'|T] ->
@@ -1178,29 +1199,16 @@ finish_request(_Req, Context) ->
     Verb = cb_context:req_verb(Context),
     Event = create_event_name(Context, [<<"finish_request">>, Verb, Mod]),
     _ = kz_util:spawn(fun crossbar_bindings:pmap/2, [Event, Context]),
-    maybe_cleanup_file(cb_context:fetch(Context, 'csv_acc')),
-    'ok'.
+    maybe_cleanup_file(cb_context:resp_file(Context)).
 
--spec maybe_cleanup_file(kz_csv:file_return() | 'undefined') -> 'ok'.
-maybe_cleanup_file({File, _}) ->
+-spec maybe_cleanup_file(binary()) -> 'ok'.
+maybe_cleanup_file(<<>>) -> 'ok';
+maybe_cleanup_file(File) ->
     _P = spawn(fun() -> cleanup_file(File) end),
-    lager:debug("deleting ~s in ~p", [File, _P]);
-maybe_cleanup_file(_) -> 'ok'.
+    lager:debug("deleting ~s in ~p", [File, _P]).
 
 -spec cleanup_file(file:filename_all()) -> 'ok'.
 cleanup_file(File) ->
-    case file:read_file_info(File) of
-        {'ok', #file_info{size=Bytes}} ->
-            Sleep = round(math:log(Bytes)) * ?MILLISECONDS_IN_SECOND,
-            cleanup_file(File, Sleep);
-        {'error', _Posix} ->
-            lager:debug("failed to read file: ~p", [_Posix]),
-            cleanup_file(File, 5 * ?MILLISECONDS_IN_SECOND)
-    end.
-
--spec cleanup_file(file:filename_all(), pos_integer()) -> 'ok'.
-cleanup_file(File, Sleep) ->
-    timer:sleep(Sleep),
     'ok' = file:delete(File),
     lager:debug("deleted file ~s", [File]).
 
@@ -1347,7 +1355,7 @@ create_csv_resp_content_from_csv_acc(Req, Context, {_File, _}=CSVAcc) ->
                },
     {{'file', File}
     ,maps:fold(fun(H, V, R) -> cowboy_req:set_resp_header(H, V, R) end, Req, Headers)
-    ,Context
+    ,cb_context:set_resp_file(Context, File)
     }.
 
 %%------------------------------------------------------------------------------
@@ -1392,13 +1400,7 @@ create_empty_csv_resp(Req, Context) ->
 create_resp_file(Req, Context) ->
     File = cb_context:resp_file(Context),
     Len = filelib:file_size(File),
-    Fun = fun(Socket, Transport) ->
-                  lager:debug("sending file ~s", [File]),
-                  Res = Transport:sendfile(Socket, kz_term:to_list(File)),
-                  _ = file:delete(File),
-                  Res
-          end,
-    {{Len, Fun}, Req}.
+    {{'sendfile', 0, Len, File}, Req}.
 
 %%------------------------------------------------------------------------------
 %% @doc Encodes the `JObj' and send it as a chunk. Starts chunk response if is
@@ -1587,7 +1589,7 @@ create_pull_response(Req0, Context0, Fun) ->
     end.
 
 -spec maybe_set_pull_response_stream(kz_term:text() | resp_file(), cowboy_req:req(), cb_context:context()) ->
-                                            {kz_term:text() | pull_file_resp()
+                                            {kz_term:text()
                                             ,cowboy_req:req()
                                             ,cb_context:context()
                                             }.
@@ -1596,11 +1598,6 @@ maybe_set_pull_response_stream({'file', File}, Req, Context) ->
     lager:debug("sending file ~s(~p)", [File, Size]),
     Req1 = cowboy_req:reply(200, #{}, {'sendfile', 0, Size, File}, Req),
     {'stop', Req1, cb_context:set_resp_status(Context, 'stop')};
-maybe_set_pull_response_stream({FileLength, TransportFun}, Req, Context)
-  when is_integer(FileLength)
-       andalso is_function(TransportFun, 2) ->
-    lager:debug("streaming file"),
-    {{'stream', FileLength, TransportFun}, Req, Context};
 maybe_set_pull_response_stream(Other, Req, Context) ->
     {Other, Req, Context}.
 

applications/crossbar/src/crossbar.hrl

@@ -23,6 +23,8 @@
 
 -define(CB_APPS_STORE_LIST, <<"apps_store/crossbar_listing">>).
 
+-define(NO_ENVELOPE_CONTENT_TYPES, []).
+
 -define(INBOUND_HOOK, <<"hooks">>).
 
 -define(NO_ENVELOPE_VERSIONS, [?INBOUND_HOOK]).

applications/crossbar/src/crossbar_maintenance.erl

@@ -994,9 +994,16 @@ refresh_app(AppPath, AppUrl) ->
 find_apps(AppsPath) ->
     AccFun =
         fun(AppJSONPath, Acc) ->
-                App = filename:absname(AppJSONPath),
-                %% /.../App/metadata/app.json --> App
-                [filename:dirname(filename:dirname(App)) | Acc]
+                try
+                    lager:debug("find...~p", [AppJSONPath]),
+                    App = AppJSONPath, %%filename:absname(AppJSONPath),
+                    %% /.../App/metadata/app.json --> App
+                    [filename:dirname(filename:dirname(App)) | Acc]
+                catch
+                    _Ex:_Er:_ST ->
+                        kz_util:log_stacktrace(_ST),
+                        Acc
+                end
         end,
     filelib:fold_files(AppsPath, "app\\.json", 'true', AccFun, []).
 

applications/crossbar/src/crossbar_view.erl

@@ -940,6 +940,7 @@ limit_with_last_key('true', PageSize, _ChunkSize, TotalQueried) ->
 %%------------------------------------------------------------------------------
 -spec apply_filter(mapper_fun(), kz_json:objects()) ->
                           kz_json:objects() |
+                          kz_json:object() |
                           {'error', any()}.
 apply_filter(_Mapper, []) ->
     [];

applications/crossbar/src/modules/cb_basic_auth.erl

@@ -35,7 +35,7 @@
 %%------------------------------------------------------------------------------
 -spec init() -> ok.
 init() ->
-    _ = crossbar_bindings:bind(<<"*.authenticate">>, ?MODULE, 'authenticate'),
+    _ = crossbar_bindings:bind(<<"*.early_authenticate">>, ?MODULE, 'authenticate'),
     ok.
 
 %%------------------------------------------------------------------------------
@@ -155,7 +155,20 @@ is_expired(Context, JObj) ->
 -spec set_auth_doc(cb_context:context(), kz_json:object()) ->
                           cb_context:context().
 set_auth_doc(Context, JObj) ->
+    AuthAccountId = kz_doc:account_id(JObj),
+    OwnerId = kz_doc:id(JObj),
     Setters = [{fun cb_context:set_auth_doc/2, JObj}
-              ,{fun cb_context:set_auth_account_id/2 ,kz_doc:account_id(JObj)}
+              ,{fun cb_context:set_auth_account_id/2, AuthAccountId}
+               | maybe_add_is_admins(AuthAccountId, OwnerId)
               ],
     cb_context:setters(Context, Setters).
+
+-spec maybe_add_is_admins(kz_term:api_ne_binary(), kz_term:api_ne_binary()) -> cb_context:setters().
+maybe_add_is_admins(?NE_BINARY = AuthAccountId, ?NE_BINARY = OwnerId) ->
+    [{fun cb_context:set_is_superduper_admin/2, cb_context:is_superduper_admin(AuthAccountId)}
+    ,{fun cb_context:set_is_account_admin/2, cb_context:is_account_admin(AuthAccountId, OwnerId)}
+    ];
+maybe_add_is_admins(?NE_BINARY = AuthAccountId, 'undefined') ->
+    [{fun cb_context:set_is_superduper_admin/2, cb_context:is_superduper_admin(AuthAccountId)}];
+maybe_add_is_admins(_, _) ->
+    [].

applications/crossbar/src/modules/cb_presence.erl

@@ -29,7 +29,7 @@
                                                         ,?PRESENCE_QUERY_DEFAULT_TIMEOUT
                                                         )
        ).
--define(REPORT_CONTENT_TYPE, [{'send_file', [{<<"application">>, <<"json">>}]}]).
+-define(REPORT_CONTENT_TYPE, [{'send_file', [{<<"application">>, <<"json">>, '*'}]}]).
 -define(REPORT_PREFIX, "report-").
 -define(MATCH_REPORT_PREFIX(ReportId), <<?REPORT_PREFIX, ReportId/binary>>).
 -define(MATCH_REPORT_PREFIX, <<?REPORT_PREFIX, _ReportId/binary>>).
@@ -404,7 +404,7 @@ load_report(Context, Report) ->
 -spec set_report(cb_context:context(), kz_term:ne_binary()) -> cb_context:context().
 set_report(Context, File) ->
     Name = kz_term:to_binary(filename:basename(File)),
-    Headers = [{<<"Content-Disposition">>, <<"attachment; filename=", Name/binary>>}],
+    Headers = #{<<"Content-Disposition">> => <<"attachment; filename=", Name/binary>>},
     cb_context:setters(Context,
                        [{fun cb_context:set_resp_file/2, File}
                        ,{fun cb_context:set_resp_etag/2, 'undefined'}

applications/crossbar/src/modules/cb_sms.erl

@@ -239,24 +239,24 @@ get_default_caller_id(Context, OwnerId) ->
 -spec summary(cb_context:context()) -> cb_context:context().
 summary(Context) ->
     {ViewName, Opts} =
-        build_view_name_rane_keys(cb_context:device_id(Context), cb_context:user_id(Context)),
+        build_view_name_range_keys(cb_context:device_id(Context), cb_context:user_id(Context)),
     Options = [{'mapper', fun normalize_view_results/2}
                | Opts
               ],
     crossbar_view:load_modb(Context, ViewName, Options).
 
--spec build_view_name_rane_keys(kz_term:api_binary(), kz_term:api_binary()) -> {kz_term:ne_binary(), crossbar_view:options()}.
-build_view_name_rane_keys('undefined', 'undefined') ->
+-spec build_view_name_range_keys(kz_term:api_binary(), kz_term:api_binary()) -> {kz_term:ne_binary(), crossbar_view:options()}.
+build_view_name_range_keys('undefined', 'undefined') ->
     {?CB_LIST_ALL
     ,[{'range_start_keymap', []}
      ,{'range_end_keymap', crossbar_view:suffix_key_fun([kz_json:new()])}
      ]
     };
-build_view_name_rane_keys('undefined', Id) ->
+build_view_name_range_keys('undefined', Id) ->
     {?CB_LIST_BY_OWNERID
     ,[{'range_keymap', [Id]}]
     };
-build_view_name_rane_keys(Id, _) ->
+build_view_name_range_keys(Id, _) ->
     {?CB_LIST_BY_DEVICE
     ,[{'range_keymap', [Id]}]
     }.

applications/crossbar/src/modules_v2/cb_devices_v2.erl

@@ -56,8 +56,8 @@
 init() ->
     Bindings = [{<<"v2_resource.allowed_methods.devices">>, 'allowed_methods'}
                ,{<<"v2_resource.resource_exists.devices">>, 'resource_exists'}
-               ,{<<"v2_resource.authenticate">>, 'authenticate'}
-               ,{<<"v2_resource.authorize">>, 'authorize'}
+               ,{<<"v2_resource.authenticate.devices">>, 'authenticate'}
+               ,{<<"v2_resource.authorize.devices">>, 'authorize'}
                ,{<<"v2_resource.validate_resource.devices">>, 'validate_resource'}
                ,{<<"v2_resource.validate.devices">>, 'validate'}
                ,{<<"v2_resource.execute.put.devices">>, 'put'}

applications/crossbar/src/modules_v2/cb_phone_numbers_v2.erl

@@ -83,8 +83,8 @@
 %%------------------------------------------------------------------------------
 -spec init() -> ok.
 init() ->
-    _ = crossbar_bindings:bind(<<"v2_resource.authenticate">>, ?MODULE, 'authenticate'),
-    _ = crossbar_bindings:bind(<<"v2_resource.authorize">>, ?MODULE, 'authorize'),
+    _ = crossbar_bindings:bind(<<"v2_resource.authenticate.phone_numbers">>, ?MODULE, 'authenticate'),
+    _ = crossbar_bindings:bind(<<"v2_resource.authorize.phone_numbers">>, ?MODULE, 'authorize'),
     _ = crossbar_bindings:bind(<<"v2_resource.allowed_methods.phone_numbers">>, ?MODULE, 'allowed_methods'),
     _ = crossbar_bindings:bind(<<"v2_resource.resource_exists.phone_numbers">>, ?MODULE, 'resource_exists'),
     _ = crossbar_bindings:bind(<<"v2_resource.validate.phone_numbers">>, ?MODULE, 'validate'),