WHISTLE-1441: update media_importer

utils/media_importer/Makefile

@@ -0,0 +1,36 @@
+ROOT = ../..
+DIALYZER = dialyzer
+REBAR = $(ROOT)/bin/rebar
+
+DIRS = . \
+	$(ROOT)/lib/whistle-1.0.0
+
+all: compile
+
+compile:
+	@$(REBAR) compile
+
+deps:
+	@$(REBAR) get-deps
+
+clean:
+	@$(REBAR) clean
+	rm -f test/*.beam
+	rm -f erl_crash.dump
+
+test: clean app eunit
+
+eunit:
+	@$(REBAR) eunit skip_deps=true
+
+build-plt:
+	@$(DIALYZER) --build_plt --output_plt $(ROOT)/.platform_dialyzer.plt \
+		--apps kernel stdlib sasl inets crypto public_key ssl
+
+dialyze: 
+	@$(DIALYZER) $(foreach DIR,$(DIRS),$(DIR)/ebin) \
+                --plt $(ROOT)/.platform_dialyzer.plt --no_native \
+		-Werror_handling -Wrace_conditions -Wunmatched_returns # -Wunderspecs
+
+docs:
+	@$(REBAR) doc skip_deps=true

utils/media_importer/rebar.config

@@ -4,5 +4,5 @@
         ,debug_info
        ]}. % uncomment to get debug messages, remove debug_info for production
 {dialyzer_opts, [{warnings, [unmatched_returns, race_conditions, error_handling]}]}.
-{lib_dirs, ["../../lib"]}.
+{lib_dirs, ["../../whistle_apps/lib", "../../lib"]}.
 

utils/media_importer/src/media_importer.erl

@@ -9,25 +9,18 @@
 -module(media_importer).
 
 -include_lib("couchbeam/include/couchbeam.hrl").
+-include_lib("whistle/include/wh_types.hrl").
 
 -export([main/1]).
 
 -import(proplists, [is_defined/2, get_value/2]).
 
--type proplist() :: list(tuple(binary() | atom(), term())) | [].
--type argv() :: string() | undefined.
--type json_string() :: atom() | binary().
--type json_number() :: integer() | float().
--type json_array() :: [json_term()].
--type json_object() :: {struct, [{json_string(), json_term()}]}.
--type json_iolist() :: {json, iolist()}.
--type json_term() :: json_string() | json_number() | json_array() | json_object() | json_iolist().
-
--spec main/1 :: (Args) -> no_return() when
-      Args :: string().
+-spec main/1 :: (string()) -> none().
 main(Args) ->
     ensure_started(sasl),
     ensure_started(crypto),
+    ensure_started(public_key),
+    ensure_started(ssl),
 %%    ensure_started(riak_err),
     ensure_started(ibrowse),
     ensure_started(couchbeam),
@@ -45,19 +38,26 @@ main(Args) ->
         false ->
             case couchbeam:create_db(Server, DbName) of
                 {ok, _} ->
+                    timer:sleep(100),
                     io:format("CREATED~n", []);
                 Error ->
-                    io:format("FAILED~nerror cause: ~p~n", [Error]),
+                    io:format("FAILED~n~p: error cause: ~p~n", [?LINE, Error]),
                     erlang:halt(1)
             end
     end,
 
-    [to_couch(Dir, Db) || Dir <- Paths],
+    try [to_couch(Dir, Db) || Dir <- Paths] of
+        _ -> ok
+    catch
+        _E:_R ->
+            ST = erlang:get_stacktrace(),
+            io:format("~nERROR: ~p:~p~n", [_E, _R]),
+            [io:format("~p~n", [S]) || S <- ST]
+    end,
 
     erlang:halt(0).
 
--spec conn_to_server/1 :: (Options) -> #server{} when
-      Options :: proplist().
+-spec conn_to_server/1 :: (wh_proplist()) -> #server{}.
 conn_to_server(Options) ->
     Host = get_value(host, Options),
     Port = get_value(port, Options),
@@ -81,60 +81,84 @@ conn_to_server(Options) ->
             is_binary(BigCouchVersion) andalso io:format("BigCouch version ~75s~s~n", [" ", BigCouchVersion]),
             Server;
         Error ->
-            io:format("FAILED~nerror cause: ~p~n", [Error]),
+            io:format("FAILED~n~p: error cause: ~p~n", [?LINE, Error]),
             erlang:halt(1)
     end.
 
--spec to_couch/2 :: (Dir, Db) -> no_return() when
-      Dir :: string(),
-      Db :: #db{}.
+-spec to_couch/2 :: (string(), #db{}) -> any().
 to_couch(Dir, Db) ->
     [import_file(F, Db) || F <- filelib:wildcard(Dir)].
 
--spec import_file/2 :: (Path, Db) -> no_return() when
-      Path :: string(),
-      Db :: #db{}.
+-spec import_file/2 :: (string(), #db{}) -> any().
 import_file(Path, Db) ->
     io:format("Import file ~-80s", [Path]),
+    erlang:garbage_collect(),
+    timer:sleep(100),
+
     {File, Id} = parse_path(wh_util:to_binary(Path)),
     case couchbeam:lookup_doc_rev(Db, binary_to_list(Id)) of
-        {ok, Rev} ->
-            {ok, Content} = file:read_file(Path),
-            couchbeam:put_attachment(Db, Id, File, Content, [{'content_type', "audio/x-wav"}, {'rev', Rev}]),
-            io:format("UPDATED~n", []);
-        _ ->
+        {error, not_found} ->
             case couchbeam:save_doc(Db, create_doc(File, Id)) of
-                {ok, _} ->
-                    {ok, Rev} = couchbeam:lookup_doc_rev(Db, Id),
+                {ok, Saved} ->
                     {ok, Content} = file:read_file(Path),
-                    couchbeam:put_attachment(Db, Id, File, Content, [{'content_type', "audio/x-wav"}, {'rev', Rev}]),
-                    io:format("SUCCESS~n", []);
+                    Opts = [{'content_type', "audio/x-wav"}
+                            ,{'content_length', byte_size(Content)}
+                            ,{'rev', wh_json:get_string_value(<<"_rev">>, Saved)}
+                           ],
+                    save_attachment(Db, Id, File, Content, Opts, "CREATED");
+                {error, conflict} ->
+                    io:format("EXISTS~n", []);
+                {error, retry_later} ->
+                    timer:sleep(100),
+                    import_file(Path, Db);
                 _Else ->
-                    io:format("FAILED~nerror cause: ~p~n", [_Else]),
+                    io:format("FAILED~n~p: error cause: ~p~n", [?LINE, _Else]),
                     erlang:halt(1)
-            end
+            end;
+        {error, retry_later} ->
+            timer:sleep(250),
+            import_file(Path, Db);
+        {error, _E} ->
+            io:format("REV ERROR: ~p~n", [_E]);
+        Rev when is_binary(Rev) ->
+            {ok, Content} = file:read_file(Path),
+            Opts = [{'content_type', "audio/x-wav"}
+                    ,{'content_length', byte_size(Content)}
+                    ,{'rev', wh_util:to_list(Rev)}
+                   ],
+            save_attachment(Db, Id, File, Content, Opts, "UPDATED")
+    end.
+
+save_attachment(Db, Id, File, Content, Opts, Success) ->
+    case couchbeam:put_attachment(Db, Id, File, Content, Opts) of
+        {error, {ok, Err, _, _Body}} ->
+            io:format("DB ERROR ~s: ~s~n", [Err, _Body]);
+        {error, retry_later} ->
+            io:format("DB ERROR: retry later~n", []);
+        {error, conflict} ->
+            io:format("DB ERROR: conflict ~p~n", [props:get_value(rev, Opts)]);
+        {error, _E} ->
+            io:format("DB ERROR: ~p~n", [_E]);
+        _OK ->
+            io:format("~s~n", [Success])
     end.
 
--spec create_doc/2 :: (File, Id) -> json_object() when
-      File :: binary(),
-      Id :: binary().
+-spec create_doc/2 :: (ne_binary(), ne_binary()) -> wh_json:object().
 create_doc(File, Id) ->
-    {struct, [
-               {<<"_id">>, Id}
-              ,{<<"pvt_type">>, <<"media">>}
-              ,{<<"display_name">>, File}
-              ,{<<"description">>, <<"">>}
-             ]}.
-
--spec parse_path/1 :: (Path) -> tuple(binary(), binary()) when
-      Path :: string().
+    wh_json:from_list([{<<"_id">>, Id}
+                       ,{<<"pvt_type">>, <<"media">>}
+                       ,{<<"display_name">>, File}
+                       ,{<<"description">>, <<"">>}
+                      ]).
+
+-spec parse_path/1 :: (ne_binary()) -> {ne_binary(), ne_binary()}.
 parse_path(Path) ->
     PTokens = binary:split(Path, <<"/">>, [global, trim]),
     File = lists:last(PTokens),
     {File, hd(binary:split(File, <<".">>))}.
 
--spec parse_args/1 :: (Args) -> tuple(ok, proplist(), list()) | tuple(error, atom()) when
-      Args :: string().
+-spec parse_args/1 :: (string()) -> {'ok', wh_proplist(), list()} |
+                                    {'error', atom()}.
 parse_args(Args) ->
     OptSpecList = option_spec_list(),
     case getopt:parse(OptSpecList, Args) of
@@ -152,13 +176,13 @@ parse_args(Args) ->
             {error, Reason}
     end.
 
--spec display_help/0 :: () -> no_return().
+-spec display_help/0 :: () -> none().
 display_help() ->
     OptSpecList = option_spec_list(),
     getopt:usage(OptSpecList, "media_importer", "<path_to_fs_sounds> ..."),
     erlang:halt().
 
--spec option_spec_list/0 :: () -> proplist().
+-spec option_spec_list/0 :: () -> [{atom(), byte(), string(), 'undefined' | tuple(), string()},...].
 option_spec_list() ->
     [
      {help,    $?,    "help",    undefined,                     "Show the program options"},
@@ -171,8 +195,8 @@ option_spec_list() ->
 
 ensure_started(App) ->
     case application:start(App) of
-	ok ->
-	    ok;
-	{error, {already_started, App}} ->
-	    ok
+        ok ->
+            ok;
+        {error, {already_started, App}} ->
+            ok
     end.