Merge pull request #31 from cloudant/46268-fix-match-arrays

Replace element position with brackets

src/mango_doc.erl

@@ -22,7 +22,6 @@
 
     get_field/2,
     get_field/3,
-    parse_field/1,
     rem_field/2,
     set_field/3
 ]).
@@ -373,7 +372,7 @@ get_field(Props, Field) ->
 
 
 get_field(Props, Field, Validator) when is_binary(Field) ->
-    {ok, Path} = parse_field(Field),
+    {ok, Path} = mango_util:parse_field(Field),
     get_field(Props, Path, Validator);
 get_field(Props, [], no_validation) ->
     Props;
@@ -411,7 +410,7 @@ get_field(_, [_|_], _) ->
 
 
 rem_field(Props, Field) when is_binary(Field) ->
-    {ok, Path} = parse_field(Field),
+    {ok, Path} = mango_util:parse_field(Field),
     rem_field(Props, Path);
 rem_field({Props}, [Name]) ->
     case lists:keytake(Name, 1, Props) of
@@ -472,7 +471,7 @@ rem_field(_, [_|_]) ->
 
 
 set_field(Props, Field, Value) when is_binary(Field) ->
-    {ok, Path} = parse_field(Field),
+    {ok, Path} = mango_util:parse_field(Field),
     set_field(Props, Path, Value);
 set_field({Props}, [Name], Value) ->
     {lists:keystore(Name, 1, Props, {Name, Value})};
@@ -536,43 +535,3 @@ set_elem(1, [_ | Rest], Value) ->
     [Value | Rest];
 set_elem(I, [Item | Rest], Value) when I > 1 ->
     [Item | set_elem(I-1, Rest, Value)].
-
-parse_field(Field) ->
-    case binary:match(Field, <<"\\">>, []) of
-        nomatch ->
-            % Fast path, no regex required
-            {ok, check_non_empty(Field, binary:split(Field, <<".">>, [global]))};
-        _ ->
-            parse_field_slow(Field)
-    end.
-
-parse_field_slow(Field) ->
-    Path = lists:map(fun
-        (P) when P =:= <<>> ->
-            ?MANGO_ERROR({invalid_field_name, Field});
-        (P) ->
-            re:replace(P, <<"\\\\">>, <<>>, [global, {return, binary}])
-    end, re:split(Field, <<"(?<!\\\\)\\.">>)),
-    {ok, Path}.
-
-check_non_empty(Field, Parts) ->
-    case lists:member(<<>>, Parts) of
-        true ->
-            ?MANGO_ERROR({invalid_field_name, Field});
-        false ->
-            Parts
-    end.
-
--ifdef(TEST).
--include_lib("eunit/include/eunit.hrl").
-
-parse_field_test() ->
-    ?assertEqual({ok, [<<"ab">>]}, parse_field(<<"ab">>)),
-    ?assertEqual({ok, [<<"a">>, <<"b">>]}, parse_field(<<"a.b">>)),
-    ?assertEqual({ok, [<<"a.b">>]}, parse_field(<<"a\\.b">>)),
-    ?assertEqual({ok, [<<"a">>, <<"b">>, <<"c">>]}, parse_field(<<"a.b.c">>)),
-    ?assertEqual({ok, [<<"a">>, <<"b.c">>]}, parse_field(<<"a.b\\.c">>)),
-    Exception = {mango_error, ?MODULE, {invalid_field_name, <<"a..b">>}},
-    ?assertThrow(Exception, parse_field(<<"a..b">>)).
-
--endif.

src/mango_fields.erl

@@ -35,7 +35,7 @@ extract(Doc, all_fields) ->
     Doc;
 extract(Doc, Fields) ->
     lists:foldl(fun(F, NewDoc) ->
-        {ok, Path} = mango_doc:parse_field(F),
+        {ok, Path} = mango_util:parse_field(F),
         case mango_doc:get_field(Doc, Path) of
             not_found ->
                 NewDoc;

src/mango_selector_text.erl

@@ -25,6 +25,12 @@
 -include("mango.hrl").
 
 
+%% Regex for <<"\\.">>
+-define(PERIOD, {re_pattern,0,0,<<69,82,67,80,57,0,0,0,0,0,0,0,2,0,0,0,
+0,0,0,0,46,0,0,0,48,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,93,0,
+5,27,46,84,0,5,0>>}).
+
+
 convert(Object) ->
     TupleTree = convert([], Object),
     iolist_to_binary(to_query(TupleTree)).
@@ -159,12 +165,35 @@ convert(Path, {[{<<"$size">>, Arg}]}) ->
 convert(_Path, {[{<<"$", _/binary>>=Op, _}]}) ->
     ?MANGO_ERROR({invalid_operator, Op});
 
-% We've hit a field name specifier. We need to break the name
-% into path parts and continue our conversion.
-convert(Path, {[{Field, Cond}]}) ->
-    NewPathParts = re:split(Field, <<"\\.">>),
-    NewPath = lists:reverse(NewPathParts) ++ Path,
-    convert(NewPath, Cond);
+% We've hit a field name specifier. Check if the field name is accessing
+% arrays. Convert occurrences of element position references to .[]. Then we
+% need to break the name into path parts and continue our conversion.
+convert(Path, {[{Field0, Cond}]}) ->
+    {ok, PP0} = case Field0 of
+        <<>> ->
+            {ok, []};
+        _ ->
+            mango_util:parse_field(Field0)
+    end,
+    % Later on, we perform a lucene_escape_user call on the
+    % final Path, which calls parse_field again. Calling the function
+    % twice converts <<"a\\.b">> to [<<"a">>,<<"b">>]. This leads to
+    % an incorrect query since we need [<<"a.b">>]. Without breaking
+    % our escaping mechanism, we simply revert this first parse_field
+    % effect and replace instances of "." to "\\.".
+    PP1 = [re:replace(P, ?PERIOD, <<"\\\\.">>,
+        [global,{return,binary}]) || P <- PP0],
+    {PP2, HasInteger} = replace_array_indexes(PP1, [], false),
+    NewPath = PP2 ++ Path,
+    case HasInteger of
+        true ->
+            OldPath = lists:reverse(PP1, Path),
+            OldParts = convert(OldPath, Cond),
+            NewParts = convert(NewPath, Cond),
+            {op_or, [OldParts, NewParts]};
+        false ->
+            convert(NewPath, Cond)
+    end;
 
 %% For $in
 convert(Path, Val) when is_binary(Val); is_number(Val); is_boolean(Val) ->
@@ -362,3 +391,16 @@ get_sort_types(Field, {[{_, Cond}]}, Acc)  when is_tuple(Cond)->
 
 get_sort_types(_Field, _, Acc)  ->
     Acc.
+
+
+replace_array_indexes([], NewPartsAcc, HasIntAcc) ->
+    {NewPartsAcc, HasIntAcc};
+replace_array_indexes([Part | Rest], NewPartsAcc, HasIntAcc) ->
+    {NewPart, HasInt} = try
+        _ = list_to_integer(binary_to_list(Part)),
+        {<<"[]">>, true}
+    catch _:_ ->
+        {Part, false}
+    end,
+    replace_array_indexes(Rest, [NewPart | NewPartsAcc],
+         HasInt or HasIntAcc).

src/mango_util.erl

@@ -40,7 +40,9 @@
 
     has_suffix/2,
 
-    join/2
+    join/2,
+
+    parse_field/1
 ]).
 
 
@@ -343,7 +345,7 @@ lucene_escape_qv(<<C, Rest/binary>>) ->
 
 
 lucene_escape_user(Field) ->
-    {ok, Path} = mango_doc:parse_field(Field),
+    {ok, Path} = parse_field(Field),
     Escaped = [mango_util:lucene_escape_field(P) || P <- Path],
     iolist_to_binary(join(".", Escaped)).
 
@@ -377,3 +379,46 @@ is_number_string(Value) when is_list(Value)->
         _ ->
             true
     end.
+
+
+parse_field(Field) ->
+    case binary:match(Field, <<"\\">>, []) of
+        nomatch ->
+            % Fast path, no regex required
+            {ok, check_non_empty(Field, binary:split(Field, <<".">>, [global]))};
+        _ ->
+            parse_field_slow(Field)
+    end.
+
+parse_field_slow(Field) ->
+    Path = lists:map(fun
+        (P) when P =:= <<>> ->
+            ?MANGO_ERROR({invalid_field_name, Field});
+        (P) ->
+            re:replace(P, <<"\\\\">>, <<>>, [global, {return, binary}])
+    end, re:split(Field, <<"(?<!\\\\)\\.">>)),
+    {ok, Path}.
+
+
+check_non_empty(Field, Parts) ->
+    case lists:member(<<>>, Parts) of
+        true ->
+            ?MANGO_ERROR({invalid_field_name, Field});
+        false ->
+            Parts
+    end.
+
+
+-ifdef(TEST).
+-include_lib("eunit/include/eunit.hrl").
+
+parse_field_test() ->
+    ?assertEqual({ok, [<<"ab">>]}, parse_field(<<"ab">>)),
+    ?assertEqual({ok, [<<"a">>, <<"b">>]}, parse_field(<<"a.b">>)),
+    ?assertEqual({ok, [<<"a.b">>]}, parse_field(<<"a\\.b">>)),
+    ?assertEqual({ok, [<<"a">>, <<"b">>, <<"c">>]}, parse_field(<<"a.b.c">>)),
+    ?assertEqual({ok, [<<"a">>, <<"b.c">>]}, parse_field(<<"a.b\\.c">>)),
+    Exception = {mango_error, ?MODULE, {invalid_field_name, <<"a..b">>}},
+    ?assertThrow(Exception, parse_field(<<"a..b">>)).
+
+-endif.

test/06-basic-text-test.py

@@ -39,6 +39,28 @@ def test_with_array(self):
         assert docs[0]["name"]["first"] == "Stephanie"
         assert docs[0]["favorites"] == faves
 
+    def test_array_ref(self):
+        docs = self.db.find({"favorites.1": "Python"})
+        assert len(docs) == 4
+        for d in docs:
+            assert "Python" in d["favorites"]
+
+        # Nested Level
+        docs = self.db.find({"favorites.0.2": "Python"})
+        print len(docs)
+        assert len(docs) == 1
+        for d in docs:
+            assert "Python" in d["favorites"][0][2]
+
+    def test_number_ref(self):
+        docs = self.db.find({"11111": "number_field"})
+        assert len(docs) == 1
+        assert docs[0]["11111"] == "number_field"
+
+        docs = self.db.find({"22222.33333": "nested_number_field"})
+        assert len(docs) == 1
+        assert docs[0]["22222"]["33333"] == "nested_number_field"
+
     def test_lt(self):
         docs = self.db.find({"age": {"$lt": 22}})
         assert len(docs) == 0

test/user_docs.py

@@ -197,7 +197,9 @@ def add_text_indexes(db, kwargs):
             "Erlang",
             "C",
             "Erlang"
-        ]
+        ],
+        "11111": "number_field",
+        "22222": {"33333" : "nested_number_field"}
     },
     {
         "_id": "8e1c90c0-ac18-4832-8081-40d14325bde0",