Support Erlang/OTP 20.0 (well, 19 for now) (2600hz#4048)

* otp20: try to use Kazoo on OTP 20.0

* otp20: crypto:rand_uniform/2 is deprecated and will be removed in a future release; use rand:uniform/1

* erlang:get_stacktrace/0 used in the wrong part of 'try' expression. (Use it in the block between 'catch' and 'end'.)

* otp20: migrate gen_fsm things to gen_statem

* otp20: m fmt

* otp20: some leftovers

* otp20: fix calls to erlang:start_timer cause OTP's suggestions were wrong

* otp20: same rationale (see github.com/erlang/otp/pull/1527)

* otp20: support ETS tables with Erlang refs as names

* otp20: have circleci run Erlang 20

* otp20: revert FSM changes

* otp20: stick with 19.3 for now, as changes are compatible anyway

* otp20: do not build 20.0

* otp20: still log when failing

* don't rule out otp18 quite yet

* Fix acdc under OTP 20 using gen_statem (2600hz#4084)

* otp20: fix otp18 compilation issue

* otp20: switch to 18.3 from 18.2 in Travis

* otp20: ok gen_statem is >=19.0

* otp20: my bad, lets stick with 19 for now. both 18 and 20 cant be used right now

* otp20: document move to OTP 19 from 18

.travis.yml

@@ -8,8 +8,7 @@ notifications:
       - "irc.freenode.org#2600hz-dev"
 
 otp_release:
-  - 18.2
-  - 19.1
+  - 19.3
 
 sudo: required
 

applications/acdc/doc/architecture.md

@@ -4,7 +4,7 @@ Agents represent the endpoints that a call in a queue will try to connect with.
 
 #### Agents
 
-Agents are comprised of two processes, a `gen_listener` (named process) and a `gen_fsm`. A supervisor ensures that the `gen_listener` and `gen_fsm` are kept alive together. A supervisor above that manages the list of agents.
+Agents are comprised of two processes, a `gen_listener` (named process) and a `gen_statem`. A supervisor ensures that the `gen_listener` and `gen_statem` are kept alive together. A supervisor above that manages the list of agents.
 
 > Agent process tree
 

applications/acdc/src/acdc_queue_listener.erl

@@ -680,20 +680,18 @@ clear_call_state(#state{account_id=AccountId
 -spec publish(api_terms(), kz_amqp_worker:publish_fun()) -> 'ok'.
 -spec publish(ne_binary(), api_terms(), fun((ne_binary(), api_terms()) -> 'ok')) -> 'ok'.
 publish(Req, F) ->
-    case catch F(Req) of
-        'ok' -> 'ok';
-        {'EXIT', _R} ->
+    try F(Req)
+    catch _E:_R ->
             ST = erlang:get_stacktrace(),
-            lager:debug("failed to publish message: ~p", [_R]),
+            lager:debug("failed to publish message: ~p:~p", [_E, _R]),
             kz_util:log_stacktrace(ST),
             'ok'
     end.
 publish(Q, Req, F) ->
-    case catch F(Q, Req) of
-        'ok' -> 'ok';
-        {'EXIT', _R} ->
+    try F(Q, Req)
+    catch _E:_R ->
             ST = erlang:get_stacktrace(),
-            lager:debug("failed to publish message to ~s: ~p", [Q, _R]),
+            lager:debug("failed to publish message to ~s: ~p:~p", [Q, _E, _R]),
             kz_util:log_stacktrace(ST),
             'ok'
     end.

applications/crossbar/doc/about.md

@@ -20,7 +20,7 @@ curl -v -X GET \
 {
     "auth_token": "{AUTH_TOKEN}",
     "data": {
-        "erlang_version": "18",
+        "erlang_version": "19",
         "ports": 21,
         "processes": 1816,
         "used_memory": 89615664,

applications/ecallmgr/src/ecallmgr_call_control.erl

@@ -299,7 +299,7 @@ handle_cast({'fs_nodeup', Node}, #state{node=Node
                                        }=State) ->
     lager:debug("regained connection to media node ~s", [Node]),
     _ = (catch erlang:cancel_timer(TRef)),
-    _ = timer:sleep(crypto:rand_uniform(100, 1500)),
+    _ = timer:sleep(100 + rand:uniform(1400)),
     case freeswitch:api(Node, 'uuid_exists', CallId) of
         {'ok', <<"true">>} ->
             {'noreply', force_queue_advance(State#state{is_node_up='true'})};

applications/fax/src/fax_xmpp.erl

@@ -277,14 +277,12 @@ wait_for_success(Username, Conn) ->
 start_all_printers() ->
     {'ok', Results} = kz_datamgr:get_results(?KZ_FAXES_DB, <<"faxbox/cloud">>),
     List = kz_term:shuffle_list(
-             [ {crypto:rand_uniform(2000, 6000), Id, Jid}
-               || {Id, Jid, <<"claimed">>}
-                      <- [{kz_doc:id(Result)
-                          ,kz_json:get_value([<<"value">>,<<"xmpp_jid">>], Result)
-                          ,kz_json:get_value([<<"value">>,<<"state">>], Result)
-                          }
-                          || Result <- Results
-                         ]
+             [{2000 + rand:uniform(4000)
+              ,kz_doc:id(Result)
+              ,kz_json:get_value([<<"value">>,<<"xmpp_jid">>], Result)
+              }
+              || Result <- Results,
+                 <<"claimed">> =:= kz_json:get_ne_binary_value([<<"value">>,<<"state">>], Result)
              ]),
     [begin
          send_start_printer(Id, Jid),

applications/jonny5/src/j5_channel_destroy.erl

@@ -21,7 +21,7 @@
 handle_req(JObj, _Props) ->
     'true' = kapi_call:event_v(JObj),
     kz_util:put_callid(JObj),
-    timer:sleep(crypto:rand_uniform(1000, 3000)),
+    timer:sleep(1000 + rand:uniform(2000)),
     Request = j5_request:from_jobj(JObj),
     _ = account_reconcile_cdr(Request),
     _ = reseller_reconcile_cdr(Request),

core/kazoo_amqp/src/gen_listener.erl

@@ -472,7 +472,8 @@ handle_cast({'rm_responder', Responder, Keys}, #state{responders=Responders}=Sta
     ,State#state{responders=listener_utils:rm_responder(Responders, Responder, Keys)}
     };
 handle_cast({'add_binding', _, _}=AddBinding, #state{is_consuming='false'}=State) ->
-    Time = ?BIND_WAIT + (crypto:rand_uniform(100, 200)), % wait 100 + [100,200) ms before replaying the binding request
+    %% wait 100 + [100,200) ms before replaying the binding request
+    Time = ?BIND_WAIT + 100 + rand:uniform(100),
     lager:debug("not consuming yet, put binding to end of message queue after ~b ms", [Time]),
     delayed_cast(self(), AddBinding, Time),
     {'noreply', State};

core/kazoo_bindings/src/kazoo_bindings.erl

@@ -666,13 +666,10 @@ fold_bind_results([#kz_responder{module=M
             lager:debug("error: ~p", [_E]),
             E;
         {'EXIT', {'undef', [{_M, _F, _A, _}|_]}} ->
-            ST = erlang:get_stacktrace(),
-            log_undefined(M, F, length(Payload), ST),
+            lager:debug("undefined function ~s:~s/~b", [M, F, length(Payload)]),
             fold_bind_results(Responders, Payload, Route, RespondersLen, ReRunResponders);
         {'EXIT', _E} ->
-            ST = erlang:get_stacktrace(),
             lager:error("~s:~s/~p died unexpectedly: ~p", [M, F, length(Payload), _E]),
-            kz_util:log_stacktrace(ST),
             fold_bind_results(Responders, Payload, Route, RespondersLen, ReRunResponders);
         'ok' ->
             fold_bind_results(Responders, Payload, Route, RespondersLen, ReRunResponders);

core/kazoo_couch/src/kz_couch_util.erl

@@ -50,15 +50,15 @@ retry504s(_Fun, 3) ->
     {'error', 'timeout'};
 retry504s(Fun, Cnt) ->
     kazoo_stats:increment_counter(<<"bigcouch-request">>),
-    case catch Fun() of
+    try Fun() of
         {'error', {'ok', 504, _, _}} ->
             kazoo_stats:increment_counter(<<"bigcouch-504-error">>),
             timer:sleep(100 * (Cnt+1)),
             retry504s(Fun, Cnt+1);
         {'error', {'ok', ErrCode, _Hdrs, _Body}} ->
             kazoo_stats:increment_counter(<<"bigcouch-other-error">>),
             {'error', kz_term:to_integer(ErrCode)};
-%%% couchbeam doesn't pass 202 as acceptable
+        %% couchbeam doesn't pass 202 as acceptable
         {'error', {'bad_response',{202, _Headers, Body}}} ->
             {'ok', kz_json:decode(Body)};
         {'error', {'bad_response',{204, _Headers, _Body}}} ->
@@ -73,14 +73,13 @@ retry504s(Fun, Cnt) ->
         {'error', Other} ->
             kazoo_stats:increment_counter(<<"bigcouch-other-error">>),
             {'error', format_error(Other)};
-        {'ok', _Other}=OK -> OK;
-        {'EXIT', _E} ->
+        OK -> OK
+    catch _E:_R ->
             ST = erlang:get_stacktrace(),
-            lager:debug("exception running fun: ~p", [_E]),
+            lager:debug("exception running fun: ~p:~p", [_E, _R]),
             kz_util:log_stacktrace(ST),
             kazoo_stats:increment_counter(<<"bigcouch-other-error">>),
-            retry504s(Fun, Cnt+1);
-        OK -> OK
+            retry504s(Fun, Cnt+1)
     end.
 
 %%------------------------------------------------------------------------------

core/kazoo_globals/src/kz_globals.erl

@@ -78,7 +78,7 @@
 -define(QUEUE_OPTIONS, []).
 -define(CONSUME_OPTIONS, [{'no_local', 'true'}]).
 
--define(HEARTBEAT, crypto:rand_uniform(5 * ?MILLISECONDS_IN_SECOND, 15 * ?MILLISECONDS_IN_SECOND)).
+-define(HEARTBEAT, rand:uniform(5 * ?MILLISECONDS_IN_SECOND, 15 * ?MILLISECONDS_IN_SECOND)).
 -define(EXPIRE_PERIOD, 1 * ?MILLISECONDS_IN_SECOND).
 -define(FUDGE_FACTOR, 1.25).
 

core/kazoo_globals/src/kz_nodes.erl

@@ -65,7 +65,7 @@
 -define(QUEUE_OPTIONS, []).
 -define(CONSUME_OPTIONS, [{'no_local', 'true'}]).
 
--define(HEARTBEAT, crypto:rand_uniform(5 * ?MILLISECONDS_IN_SECOND, 15 * ?MILLISECONDS_IN_SECOND)).
+-define(HEARTBEAT, 5 * ?MILLISECONDS_IN_SECOND + rand:uniform(10 * ?MILLISECONDS_IN_SECOND)).
 -define(EXPIRE_PERIOD, 1 * ?MILLISECONDS_IN_SECOND).
 -define(FUDGE_FACTOR, 1.25).
 -define(APP_NAME, <<"kz_nodes">>).

core/kazoo_perf/src/kazoo_perf_maintenance.erl

@@ -85,7 +85,7 @@ graphite(Scheme, scheduler_reductions, {TotalReductions, _ReductionsSinceLastCal
 graphite(Scheme, processes_in_run_queue_of_each_schedulers, RunQueue) ->
     print_metric(Scheme, run_queue, RunQueue);
 graphite(Scheme, ets_tables_sizes, Tabs) ->
-    [print_metric(Scheme, "tab_" ++ kz_term:to_list(Tab), Size)
+    [print_metric(Scheme, "tab_" ++ kz_term:to_list(maybe_from_ref(Tab)), Size)
      || {Tab, Size} <- Tabs,
         Size =/= 0
     ];
@@ -122,7 +122,7 @@ to_props(processes_in_run_queue_of_each_schedulers, RunQueue) ->
     [{run_queue, RunQueue}
     ];
 to_props(ets_tables_sizes, Tabs) ->
-    [{kz_term:to_binary(Tab), Size}
+    [{kz_term:to_binary(maybe_from_ref(Tab)), Size}
      || {Tab, Size} <- Tabs,
         Size =/= 0
     ];
@@ -155,3 +155,11 @@ bin_to_integer(Value=?NE_BINARY) ->
     try binary_to_integer(Value)
     catch error:badarg -> not_an_int
     end.
+
+maybe_from_ref(Tab)
+  when is_reference(Tab) ->
+    Bin1 = iolist_to_binary(io_lib:format("~p", [Tab])),
+    Bin2 = binary:replace(Bin1, [<<$<>>,<<$>>>],<<>>, [global]),
+    binary:replace(Bin2, <<$.>>, <<$_>>, [global]);
+maybe_from_ref(Tab) ->
+    Tab.

doc/announcements.md

@@ -17,6 +17,12 @@ We hope that you agree and and are not inconvienced by this change. As always we
 
 ## Versions
 
+### 4.2
+
+1.  Erlang Version Support
+
+    Starting with Kazoo 4.2 Erlang support will target 19+ and will not be backward compatible with prior Erlang versions.
+
 
 ### 4.1
 
@@ -135,4 +141,4 @@ We hope that you agree and and are not inconvienced by this change. As always we
 
     `ibrowse` will be replaced by `core/kazoo_web/kz_http` which is using Erlang `httpc`. `kz_http` is the new HTTP client module now and the previous `kz_http` module is renamed to `kz_http_util`.
 
-    If you maintain code apart from Kazoo that uses `ibrowse`, please either covert to equivalent functionality with `kz_http=/=httpc` or plan how you'll build your custom code with your own dependency of `ibrowse`.
+    If you maintain code apart from Kazoo that uses `ibrowse`, please either covert to equivalent functionality with `kz_http=/=httpc` or plan how you'll build your custom code with your own dependency of `ibrowse`.

doc/installation.md

@@ -33,20 +33,20 @@ Note: `htmldoc` is required only if [you want to be able to download PDFs](./ann
 
 ### Erlang
 
-Kazoo 4 targets Erlang 18+. There are a couple ways to install Erlang:
+Kazoo 4 targets Erlang 19+. There are a couple ways to install Erlang:
 
 1.  From Source
 
-    I prefer to use a tool like [kerl](https://github.com/yrashk/kerl) to manage my installations. If you want to play around with multiple versions of Erlang while hacking on Kazoo, this is probably the best way.
+    I prefer to use a tool like [kerl](https://github.com/kerl/kerl) to manage my installations. If you want to play around with multiple versions of Erlang while hacking on Kazoo, this is probably the best way.
 
     ```shell
-    curl -O https://raw.githubusercontent.com/yrashk/kerl/master/kerl
-    chmod a+x kerl
+    curl -O https://raw.githubusercontent.com/kerl/kerl/master/kerl
+    chmod +x kerl
     mv kerl /usr/bin
     kerl list releases
-    kerl build 18.2 r18.2 # this takes a while
-    kerl install r18.2 /usr/local/erlang
-    . /usr/local/erlang/activate
+    kerl build 19.3 19.3 # this takes a while
+    kerl install 19.3 /usr/local/otp-19.3
+    . /usr/local/otp-19.3/activate
     ```
 
 2.  Erlang Solutions
@@ -128,4 +128,4 @@ alias sup='KAZOO_ROOT=/opt/kazoo sup'
 
     `make sup_completion` creates `sup.bash`: a Bash completion file for the SUP command
 
-    -   Copy or symlink this file to `/etc/bash_completion.d/sup.bash`
+    -   Copy or symlink this file to `/etc/bash_completion.d/sup.bash`