Skip to content

Commit

Permalink
[master] PROD-30: some enhancements to run custom tasks + template (2…
Browse files Browse the repository at this point in the history
…600hz#6070)

* move triggrable macros to core

If other apps want to start their own tasks module and bind to triggerable
they need access to these macros

* add new type kazoo_month to indicate `{year, month}` term

* adding description to ledgers summary for view

For generating invoice we just only need description, date, account, and amount.
So we can add this to view and instead of reading the whole doc just using view
to retrieve the ledgers

* add render options to kz_template

* better logging in kz_template

Logging template warnings and errors were sometimes
crashes/no_working_properly silently.
Also it seems line calculation was off by 1.

* update summary_by_account to use add period start and doc id to compound key
So we can properly using start/end key in couchdb without worrying about
repeating start keys.

* add from_list_recursive/2 so we don't convert list to binary
from_list_recursive tends to convert a list if all lists' elements are ascii or
printable. Empty list is consider as printable too.

This will cause list values in objects like `[{<<"key">>, []}]` be convert to
binary which should not be!
This commit adds options to hint the from_list_recursive that the object doesn't
have any string (prinatable chars) in list and list should be considered as
list!

* add node name to lager log file
  • Loading branch information
icehess authored and lazedo committed Oct 14, 2019
1 parent e649b5f commit ee7f0a1
Show file tree
Hide file tree
Showing 11 changed files with 282 additions and 159 deletions.
29 changes: 13 additions & 16 deletions applications/crossbar/src/modules/cb_cdrs.erl
Original file line number Diff line number Diff line change
Expand Up @@ -409,28 +409,30 @@ normalize_cdr(Context, <<"json">>, Result) ->
Duration = kzd_cdrs:duration_seconds(JObj, 0),
Timestamp = kzd_cdrs:timestamp(JObj, 0) - Duration,

RowMappers = props:replace_value(<<"datetime">>, fun col_pretty_print/3, csv_rows(Context)),

kz_json:from_list([{K, apply_row_mapper(F, JObj, Timestamp, Context)} || {K, F} <- RowMappers]);
kz_json:from_list([{K, apply_row_mapper(K, F, JObj, Timestamp, Context)} || {K, F} <- csv_rows(Context)]);
normalize_cdr(Context, <<"csv">>, Result) ->
JObj = kz_json:get_json_value(<<"doc">>, Result),
Duration = kzd_cdrs:duration_seconds(JObj, 0),
Timestamp = kzd_cdrs:timestamp(JObj, 0) - Duration,

RowMappers = props:replace_value(<<"datetime">>, fun col_pretty_print/3, csv_rows(Context)),

<<(kz_binary:join([apply_row_mapper(F, JObj, Timestamp, Context)
|| {_, F} <- RowMappers
<<(kz_binary:join([apply_row_mapper(K, F, JObj, Timestamp, Context)
|| {K, F} <- csv_rows(Context)
]
,<<",">>
))/binary
,"\r\n"
>>.

apply_row_mapper(F, JObj, Timestamp, Context) when is_function(F, 3) ->
F(JObj, Timestamp, Context);
apply_row_mapper(F, JObj, Timestamp, _Context) when is_function(F, 2) ->
F(JObj, Timestamp).
-spec apply_row_mapper(kz_term:ne_binary(), fun(), kz_json:object(), kz_time:gregorian_seconds(), cb_context:context()) -> binary().
apply_row_mapper(<<"datetime">>, _F, JObj, Timestamp, Context) ->
col_pretty_print(JObj, Timestamp, Context);
apply_row_mapper(_, F, JObj, Timestamp, _Context) ->
F(JObj, Timestamp, 'undefined').

-spec col_pretty_print(kz_json:object(), kz_time:gregorian_seconds(), cb_context:context()) -> kz_term:ne_binary().
col_pretty_print(_JObj, Timestamp, Context) ->
UTCSecondsOffset = cb_context:req_value(Context, ?KEY_UTC_OFFSET),
kz_time:pretty_print_datetime(handle_utc_time_offset(Timestamp, UTCSecondsOffset)).

-spec maybe_add_csv_header(cb_context:context(), kz_term:ne_binary(), kz_json:objects() | kz_term:binaries()) -> cb_context:context().
maybe_add_csv_header(Context, _, []) ->
Expand Down Expand Up @@ -506,8 +508,3 @@ load_legs(Id, Context) ->
kz_json:objects().
normalize_leg_view_results(JObj, Acc) ->
Acc ++ [kz_json:get_json_value(<<"doc">>, JObj)].

-spec col_pretty_print(kz_json:object(), kz_time:gregorian_seconds(), cb_context:context()) -> kz_term:ne_binary().
col_pretty_print(_JObj, Timestamp, Context) ->
UTCSecondsOffset = cb_context:req_value(Context, ?KEY_UTC_OFFSET),
kz_time:pretty_print_datetime(handle_utc_time_offset(Timestamp, UTCSecondsOffset)).
4 changes: 2 additions & 2 deletions applications/crossbar/src/modules/cb_ledgers.erl
Original file line number Diff line number Diff line change
Expand Up @@ -339,7 +339,7 @@ summary(Context, Options) ->
-spec account_summary(cb_context:context(), kz_term:ne_binary()) ->cb_context:context().
account_summary(Context, MODB) ->
Options = [{'databases', [MODB]}
,{'group_level', 2}
,{'group_level', 1}
,{'mapper', fun normalize_summary_by_account/2}
,{'range_keymap', 'nil'}
],
Expand Down Expand Up @@ -470,7 +470,7 @@ normalize_view_result(LedgerJObj) ->
%%------------------------------------------------------------------------------
-spec normalize_summary_by_account(kz_json:objects(), kz_json:objects()) -> kz_json:objects().
normalize_summary_by_account(JObj, Acc) ->
AccountId = kz_json:get_value(<<"key">>, JObj),
[AccountId, _PeriodStartTS, _DocId] = kz_json:get_value(<<"key">>, JObj),
Ledger = normalize_ledger_jobj(AccountId, kz_json:get_value(<<"value">>, JObj)),
[kz_json:sum_jobjs([Ledger | Acc])].

Expand Down
14 changes: 0 additions & 14 deletions applications/tasks/src/tasks.hrl
Original file line number Diff line number Diff line change
Expand Up @@ -17,19 +17,5 @@

-define(OUTPUT_CSV_HEADER_ERROR, <<"error">>).

-define(TRIGGER_ACCOUNT, <<"tasks.triggers.account">>).
-define(TRIGGER_ACCOUNT_MOD, <<"tasks.triggers.account_mod">>).
-define(TRIGGER_DAILY, <<"tasks.triggers.day">>).
-define(TRIGGER_HOURLY, <<"tasks.triggers.hour">>).
-define(TRIGGER_MINUTELY, <<"tasks.triggers.minute">>).
-define(TRIGGER_OTHER, <<"tasks.triggers.other">>).
-define(TRIGGER_SYSTEM, <<"tasks.triggers.system">>).

-define(TRIGGER_ALL_DBS, [?TRIGGER_ACCOUNT
,?TRIGGER_ACCOUNT_MOD
,?TRIGGER_SYSTEM
,?TRIGGER_OTHER
]).

-define(KZ_TASKS_HRL, 'true').
-endif.
121 changes: 76 additions & 45 deletions core/kazoo_documents/src/kzd_cdrs.erl
Original file line number Diff line number Diff line change
Expand Up @@ -61,8 +61,9 @@
,type/0
]).

-export([to_public_json/1, to_public_json/2
,to_public_csv/1, to_public_csv/2
-export([to_public_json/1, to_public_json/2, to_public_json/3
,to_public_prop/1, to_public_prop/2, to_public_prop/3
,to_public_csv/1, to_public_csv/2, to_public_csv/3
,csv_headers/1
]).

Expand Down Expand Up @@ -656,20 +657,40 @@ to_public_json(Doc) -> to_public_json(Doc, 'false').

-spec to_public_json(doc(), boolean()) -> kz_json:object().
to_public_json(Doc, IsReseller) ->
to_public_json(Doc, IsReseller, 'undefined').

-spec to_public_json(doc(), boolean(), kz_term:api_ne_binary()) -> kz_json:object().
to_public_json(Doc, IsReseller, Timezone) ->
kz_json:from_list(to_public_prop(Doc, IsReseller, Timezone)).

-spec to_public_prop(doc()) -> kz_term:proplist().
to_public_prop(Doc) ->
to_public_prop(Doc, 'false').

-spec to_public_prop(doc(), boolean()) -> kz_term:proplist().
to_public_prop(Doc, IsReseller) ->
to_public_prop(Doc, IsReseller, 'undefined').

-spec to_public_prop(doc(), boolean(), kz_term:api_ne_binary()) -> kz_term:proplist().
to_public_prop(Doc, IsReseller, Timezone) ->
Duration = duration_seconds(Doc, 0),
Timestamp = timestamp(Doc, 0) - Duration,

kz_json:from_list([{K, F(Doc, Timestamp)} || {K, F} <- csv_headers(IsReseller)]).
[{K, F(Doc, Timestamp, Timezone)} || {K, F} <- csv_headers(IsReseller)].

-spec to_public_csv(doc()) -> iodata().
to_public_csv(Doc) -> to_public_csv(Doc, 'false').

-spec to_public_csv(doc(), boolean()) -> [binary()].
-spec to_public_csv(doc(), boolean()) -> iodata().
to_public_csv(Doc, IsReseller) ->
to_public_csv(Doc, IsReseller, 'undefined').

-spec to_public_csv(doc(), boolean(), kz_term:api_ne_binary()) -> iodata().
to_public_csv(Doc, IsReseller, Timezone) ->
Duration = duration_seconds(Doc, 0),
Timestamp = timestamp(Doc, 0) - Duration,

[F(Doc, Timestamp)
[F(Doc, Timestamp, Timezone)
|| {_, F} <- csv_headers(IsReseller)
].

Expand All @@ -680,53 +701,63 @@ csv_headers('false'=_IsReseller) ->
?COLUMNS.

%% see csv_column_fun() for specs for each function here
col_id(JObj, _Timestamp) -> kz_doc:id(JObj, <<>>).
col_call_id(JObj, _Timestamp) -> call_id(JObj, <<>>).
col_caller_id_number(JObj, _Timestamp) -> caller_id_number(JObj, <<>>).
col_caller_id_name(JObj, _Timestamp) -> caller_id_name(JObj, <<>>).
col_callee_id_number(JObj, _Timestamp) -> callee_id_number(JObj, <<>>).
col_callee_id_name(JObj, _Timestamp) -> callee_id_name(JObj, <<>>).
col_duration_seconds(JObj, _Timestamp) -> duration_seconds(JObj, <<>>).
col_billing_seconds(JObj, _Timestamp) -> billing_seconds(JObj, <<>>).
col_timestamp(_JObj, Timestamp) -> kz_term:to_binary(Timestamp).
col_hangup_cause(JObj, _Timestamp) -> hangup_cause(JObj, <<>>).
col_other_leg_call_id(JObj, _Timestamp) -> other_leg_call_id(JObj, <<>>).
col_owner_id(JObj, _Timestamp) -> kz_json:get_value([?KEY_CCV, <<"owner_id">>], JObj, <<>>).
col_to(JObj, _Timestamp) -> to(JObj, <<>>).
col_from(JObj, _Timestamp) -> from(JObj, <<>>).
col_call_direction(JObj, _Timestamp) -> call_direction(JObj, <<>>).
col_request(JObj, _Timestamp) -> request(JObj, <<>>).
col_authorizing_id(JObj, _Timestamp) ->
col_id(JObj, _Timestamp, _Timezone) -> kz_doc:id(JObj, <<>>).
col_call_id(JObj, _Timestamp, _Timezone) -> call_id(JObj, <<>>).
col_caller_id_number(JObj, _Timestamp, _Timezone) -> caller_id_number(JObj, <<>>).
col_caller_id_name(JObj, _Timestamp, _Timezone) -> caller_id_name(JObj, <<>>).
col_callee_id_number(JObj, _Timestamp, _Timezone) -> callee_id_number(JObj, <<>>).
col_callee_id_name(JObj, _Timestamp, _Timezone) -> callee_id_name(JObj, <<>>).
col_duration_seconds(JObj, _Timestamp, _Timezone) -> duration_seconds(JObj, <<>>).
col_billing_seconds(JObj, _Timestamp, _Timezone) -> billing_seconds(JObj, <<>>).

col_timestamp(_JObj, Timestamp, Timezone) -> kz_term:to_binary(
kz_time:adjust_utc_timestamp(Timestamp, Timezone)
).

col_hangup_cause(JObj, _Timestamp, _Timezone) -> hangup_cause(JObj, <<>>).
col_other_leg_call_id(JObj, _Timestamp, _Timezone) -> other_leg_call_id(JObj, <<>>).
col_owner_id(JObj, _Timestamp, _Timezone) -> kz_json:get_value([?KEY_CCV, <<"owner_id">>], JObj, <<>>).
col_to(JObj, _Timestamp, _Timezone) -> to(JObj, <<>>).
col_from(JObj, _Timestamp, _Timezone) -> from(JObj, <<>>).
col_call_direction(JObj, _Timestamp, _Timezone) -> call_direction(JObj, <<>>).
col_request(JObj, _Timestamp, _Timezone) -> request(JObj, <<>>).
col_authorizing_id(JObj, _Timestamp, _Timezone) ->
case {kz_json:get_value([?KEY_CCV, <<"account_id">>], JObj, <<>>)
,kz_json:get_value([?KEY_CCV, <<"authorizing_id">>], JObj, <<>>)
}
of
{A, A} -> <<>>;
{_A, B} -> B
end.
col_customer_cost(JObj, _Timestamp) -> kz_term:to_binary(customer_cost(JObj)).

col_dialed_number(JObj, _Timestamp) -> dialed_number(JObj).
col_calling_from(JObj, _Timestamp) -> calling_from(JObj).
col_pretty_print(_JObj, Timestamp) -> kz_time:pretty_print_datetime(Timestamp).

col_unix_timestamp(_JObj, Timestamp) -> kz_term:to_binary(kz_time:gregorian_seconds_to_unix_seconds(Timestamp)).
col_rfc1036(_JObj, Timestamp) -> kz_time:rfc1036(Timestamp).
col_iso8601(_JObj, Timestamp) -> kz_date:to_iso8601_extended(Timestamp).
col_iso8601_combined(_JObj, Timestamp) -> kz_time:iso8601(Timestamp).
col_account_call_type(JObj, _Timestamp) -> kz_json:get_value([?KEY_CCV, <<"account_billing">>], JObj, <<>>).
col_rate(JObj, _Timestamp) -> kz_term:to_binary(kz_currency:units_to_dollars(kz_json:get_value([?KEY_CCV, <<"rate">>], JObj, 0))).
col_rate_name(JObj, _Timestamp) -> kz_json:get_value([?KEY_CCV, <<"rate_name">>], JObj, <<>>).
col_bridge_id(JObj, _Timestamp) -> kz_json:get_value([?KEY_CCV, <<"bridge_id">>], JObj, <<>>).
col_recording_url(JObj, _Timestamp) -> kz_json:get_value([<<"recording_url">>], JObj, <<>>).
col_media_recordings(JObj, _Timestamp) -> format_recordings(JObj).
col_media_server(JObj, _Timestamp) -> media_server(JObj, <<>>).
col_call_priority(JObj, _Timestamp) -> kz_json:get_value([?KEY_CCV, <<"call_priority">>], JObj, <<>>).

col_reseller_cost(JObj, _Timestamp) -> kz_term:to_binary(reseller_cost(JObj)).
col_reseller_call_type(JObj, _Timestamp) -> kz_json:get_value([?KEY_CCV, <<"reseller_billing">>], JObj, <<>>).

col_interaction_id(JObj, _Timestamp) -> interaction_id(JObj, <<>>).
col_customer_cost(JObj, _Timestamp, _Timezone) -> kz_term:to_binary(customer_cost(JObj)).
col_dialed_number(JObj, _Timestamp, _Timezone) -> dialed_number(JObj).
col_calling_from(JObj, _Timestamp, _Timezone) -> calling_from(JObj).
col_pretty_print(_JObj, Timestamp, Timezone) -> kz_time:pretty_print_datetime(Timestamp, Timezone).
col_unix_timestamp(_JObj, Timestamp, Timezone) -> kz_term:to_binary(
kz_time:gregorian_seconds_to_unix_seconds(
kz_time:adjust_utc_timestamp(Timestamp, Timezone)
)
).
col_rfc1036(_JObj, Timestamp, Timezone) -> kz_time:rfc1036(kz_time:adjust_utc_timestamp(Timestamp, Timezone)
,Timezone
).
col_iso8601(_JObj, Timestamp, Timezone) -> kz_date:to_iso8601_extended(
kz_time:adjust_utc_timestamp(Timestamp, Timezone)
).
col_iso8601_combined(_JObj, Timestamp, Timezone) -> kz_time:iso8601(Timestamp, Timezone).
col_account_call_type(JObj, _Timestamp, _Timezone) -> kz_json:get_value([?KEY_CCV, <<"account_billing">>], JObj, <<>>).
col_rate(JObj, _Timestamp, _Timezone) -> kz_term:to_binary(kz_currency:units_to_dollars(kz_json:get_value([?KEY_CCV, <<"rate">>], JObj, 0))).
col_rate_name(JObj, _Timestamp, _Timezone) -> kz_json:get_value([?KEY_CCV, <<"rate_name">>], JObj, <<>>).
col_bridge_id(JObj, _Timestamp, _Timezone) -> kz_json:get_value([?KEY_CCV, <<"bridge_id">>], JObj, <<>>).
col_recording_url(JObj, _Timestamp, _Timezone) -> kz_json:get_value([<<"recording_url">>], JObj, <<>>).
col_media_recordings(JObj, _Timestamp, _Timezone) -> format_recordings(JObj).
col_media_server(JObj, _Timestamp, _Timezone) -> media_server(JObj, <<>>).
col_call_priority(JObj, _Timestamp, _Timezone) -> kz_json:get_value([?KEY_CCV, <<"call_priority">>], JObj, <<>>).

col_reseller_cost(JObj, _Timestamp, _Timezone) -> kz_term:to_binary(reseller_cost(JObj)).
col_reseller_call_type(JObj, _Timestamp, _Timezone) -> kz_json:get_value([?KEY_CCV, <<"reseller_billing">>], JObj, <<>>).

col_interaction_id(JObj, _Timestamp, _Timezone) -> interaction_id(JObj, <<>>).

-spec format_recordings(kz_json:object()) -> kz_term:binaries().
format_recordings(JObj) ->
Expand Down
74 changes: 37 additions & 37 deletions core/kazoo_documents/src/kzd_cdrs.hrl
Original file line number Diff line number Diff line change
Expand Up @@ -3,52 +3,52 @@
-include("kz_documents.hrl").

-define(COLUMNS
,[{<<"id">>, fun col_id/2}
,{<<"call_id">>, fun col_call_id/2}
,{<<"caller_id_number">>, fun col_caller_id_number/2}
,{<<"caller_id_name">>, fun col_caller_id_name/2}
,{<<"callee_id_number">>, fun col_callee_id_number/2}
,{<<"callee_id_name">>, fun col_callee_id_name/2}
,{<<"duration_seconds">>, fun col_duration_seconds/2}
,{<<"billing_seconds">>, fun col_billing_seconds/2}
,{<<"timestamp">>, fun col_timestamp/2}
,{<<"hangup_cause">>, fun col_hangup_cause/2}
,{<<"other_leg_call_id">>, fun col_other_leg_call_id/2}
,{<<"owner_id">>, fun col_owner_id/2}
,{<<"to">>, fun col_to/2}
,{<<"from">>, fun col_from/2}
,{<<"direction">>, fun col_call_direction/2}
,{<<"request">>, fun col_request/2}
,{<<"authorizing_id">>, fun col_authorizing_id/2}
,{<<"cost">>, fun col_customer_cost/2}
,[{<<"id">>, fun col_id/3}
,{<<"call_id">>, fun col_call_id/3}
,{<<"caller_id_number">>, fun col_caller_id_number/3}
,{<<"caller_id_name">>, fun col_caller_id_name/3}
,{<<"callee_id_number">>, fun col_callee_id_number/3}
,{<<"callee_id_name">>, fun col_callee_id_name/3}
,{<<"duration_seconds">>, fun col_duration_seconds/3}
,{<<"billing_seconds">>, fun col_billing_seconds/3}
,{<<"timestamp">>, fun col_timestamp/3}
,{<<"hangup_cause">>, fun col_hangup_cause/3}
,{<<"other_leg_call_id">>, fun col_other_leg_call_id/3}
,{<<"owner_id">>, fun col_owner_id/3}
,{<<"to">>, fun col_to/3}
,{<<"from">>, fun col_from/3}
,{<<"direction">>, fun col_call_direction/3}
,{<<"request">>, fun col_request/3}
,{<<"authorizing_id">>, fun col_authorizing_id/3}
,{<<"cost">>, fun col_customer_cost/3}
%% New fields
,{<<"dialed_number">>, fun col_dialed_number/2}
,{<<"calling_from">>, fun col_calling_from/2}
,{<<"datetime">>, fun col_pretty_print/2}
,{<<"unix_timestamp">>, fun col_unix_timestamp/2}
,{<<"rfc_1036">>, fun col_rfc1036/2}
,{<<"iso_8601">>, fun col_iso8601/2}
,{<<"iso_8601_combined">>, fun col_iso8601_combined/2}
,{<<"call_type">>, fun col_account_call_type/2}
,{<<"rate">>, fun col_rate/2}
,{<<"rate_name">>, fun col_rate_name/2}
,{<<"bridge_id">>, fun col_bridge_id/2}
,{<<"recording_url">>, fun col_recording_url/2}
,{<<"media_recordings">>, fun col_media_recordings/2}
,{<<"media_server">>, fun col_media_server/2}
,{<<"call_priority">>, fun col_call_priority/2}
,{<<"interaction_id">>, fun col_interaction_id/2}
,{<<"dialed_number">>, fun col_dialed_number/3}
,{<<"calling_from">>, fun col_calling_from/3}
,{<<"datetime">>, fun col_pretty_print/3}
,{<<"unix_timestamp">>, fun col_unix_timestamp/3}
,{<<"rfc_1036">>, fun col_rfc1036/3}
,{<<"iso_8601">>, fun col_iso8601/3}
,{<<"iso_8601_combined">>, fun col_iso8601_combined/3}
,{<<"call_type">>, fun col_account_call_type/3}
,{<<"rate">>, fun col_rate/3}
,{<<"rate_name">>, fun col_rate_name/3}
,{<<"bridge_id">>, fun col_bridge_id/3}
,{<<"recording_url">>, fun col_recording_url/3}
,{<<"media_recordings">>, fun col_media_recordings/3}
,{<<"media_server">>, fun col_media_server/3}
,{<<"call_priority">>, fun col_call_priority/3}
,{<<"interaction_id">>, fun col_interaction_id/3}
]).

-define(COLUMNS_RESELLER
,[{<<"reseller_cost">>, fun col_reseller_cost/2}
,{<<"reseller_call_type">>, fun col_reseller_call_type/2}
,[{<<"reseller_cost">>, fun col_reseller_cost/3}
,{<<"reseller_call_type">>, fun col_reseller_call_type/3}
]).

-define(KEY_CCV, <<"custom_channel_vars">>).
-define(KEY_UTC_OFFSET, <<"utc_offset">>).

-type csv_column_fun() :: fun((kz_json:object(), kz_time:gregorian_seconds()) -> kz_term:ne_binary()).
-type csv_column_fun() :: fun((kz_json:object(), kz_time:gregorian_seconds(), kz_term:api_ne_binary()) -> kz_term:ne_binary() | kz_term:ne_binaries()).

-define(SCHEMA, <<"cdrs">>).
-define(PVT_TYPE, <<"cdr">>).
Expand Down
3 changes: 2 additions & 1 deletion core/kazoo_modb/priv/couchdb/views/ledgers.json
Original file line number Diff line number Diff line change
Expand Up @@ -70,9 +70,10 @@
" var ledgersObj = {};",
" ledgersObj[service] = {",
" 'amount': amount,",
" 'description': doc.description,",
" 'usage': doc.usage || {}",
" };",
" emit(doc.account.id, {",
" emit([doc.account.id, doc.period.start, doc._id], {",
" 'account_name': doc.account.name,",
" 'ledgers': ledgersObj,",
" 'total': amount",
Expand Down
Loading

0 comments on commit ee7f0a1

Please sign in to comment.