Merge pull request #21 from cloudant/43621-mango-escape-period

43621 mango escape period

src/mango_native_proc.erl

@@ -168,8 +168,7 @@ get_text_field_values(Values, TAcc) when is_list(Values) ->
     % We bypass make_text_field and directly call make_text_field_name
     % because the length field name is not part of the path.
     LengthFieldName = make_text_field_name(NewTAcc#tacc.path, <<"length">>),
-    EncLFN = mango_util:lucene_escape_field(LengthFieldName),
-    LengthField = [{EncLFN, <<"length">>, length(Values)}],
+    LengthField = [{LengthFieldName, <<"length">>, length(Values)}],
     get_text_field_values_arr(Values, NewTAcc, LengthField);
 
 get_text_field_values(Bin, TAcc) when is_binary(Bin) ->
@@ -258,7 +257,8 @@ should_index(Selector, Doc) ->
 get_text_field_list(IdxProps) ->
     case couch_util:get_value(<<"fields">>, IdxProps) of
         Fields when is_list(Fields) ->
-            lists:flatmap(fun get_text_field_info/1, Fields);
+            RawList = lists:flatmap(fun get_text_field_info/1, Fields),
+            [mango_util:lucene_escape_user(Field) || Field <- RawList];
         _ ->
             all_fields
     end.
@@ -286,17 +286,13 @@ make_text_field(TAcc, Type, Value) ->
     Fields = TAcc#tacc.fields,
     case Fields == all_fields orelse lists:member(FieldName, Fields) of
         true ->
-            [{mango_util:lucene_escape_field(FieldName), Type,
-            Value}];
+            [{FieldName, Type, Value}];
         false ->
             []
     end.
 
 
 make_text_field_name([P | Rest], Type) ->
-    make_text_field_name0(Rest, [P, ":", Type]).
-
-make_text_field_name0([], Name) ->
-    iolist_to_binary(Name);
-make_text_field_name0([P | Rest], Name) ->
-    make_text_field_name0(Rest, [P, "." | Name]).
+    Parts = lists:reverse(Rest, [iolist_to_binary([P, ":", Type])]),
+    Escaped = [mango_util:lucene_escape_field(N) || N <- Parts],
+    iolist_to_binary(mango_util:join(".", Escaped)).

src/mango_selector_text.erl

@@ -171,11 +171,11 @@ convert(_Path, {Props} = Sel) when length(Props) > 1 ->
 
 
 to_query({op_and, Args}) when is_list(Args) ->
-    Res = ["(", join(<<" AND ">>, lists:map(fun to_query/1, Args)), ")"],
-    Res;
+    QueryArgs = lists:map(fun to_query/1, Args),
+    ["(", mango_util:join(<<" AND ">>, QueryArgs), ")"];
 
 to_query({op_or, Args}) when is_list(Args) ->
-    ["(", join(" OR ", lists:map(fun to_query/1, Args)), ")"];
+    ["(", mango_util:join(" OR ", lists:map(fun to_query/1, Args)), ")"];
 
 to_query({op_not, {ExistsQuery, Arg}}) when is_tuple(Arg) ->
     ["(", to_query(ExistsQuery), " AND NOT (", to_query(Arg), "))"];
@@ -191,27 +191,21 @@ to_query({op_insert, Arg}) when is_binary(Arg) ->
 %% This needs to be resolved.
 to_query({op_field, {Name, Value}}) ->
     NameBin = iolist_to_binary(Name),
-    ["(", mango_util:lucene_escape_field(NameBin), ":", Value, ")"];
+    ["(", mango_util:lucene_escape_user(NameBin), ":", Value, ")"];
 
 %% This is for indexable_fields
 to_query({op_null, {Name, Value}}) ->
     NameBin = iolist_to_binary(Name),
-    ["(", mango_util:lucene_escape_field(NameBin), ":", Value, ")"];
+    ["(", mango_util:lucene_escape_user(NameBin), ":", Value, ")"];
 
 to_query({op_fieldname, {Name, Wildcard}}) ->
     NameBin = iolist_to_binary(Name),
-    ["($fieldnames:", mango_util:lucene_escape_field(NameBin), Wildcard, ")"];
+    ["($fieldnames:", mango_util:lucene_escape_user(NameBin), Wildcard, ")"];
 
 to_query({op_default, Value}) ->
     ["($default:", Value, ")"].
 
 
-join(_Sep, [Item]) ->
-    [Item];
-join(Sep, [Item | Rest]) ->
-    [Item, Sep | join(Sep, Rest)].
-
-
 %% We match on fieldname and fieldname.[]
 convert_in(Path, Args) ->
     Path0 = [<<"[]">> | Path],
@@ -255,8 +249,11 @@ field_exists_query(Path) ->
     % match a path foo.name against foo.name_first (if were to just
     % appened * isntead).
     Parts = [
+        % We need to remove the period from the path list to indicate that it is
+        % a path separator. We escape the colon because it is not used as a
+        % separator and we escape colons in field names.
         {op_fieldname, {[path_str(Path), ":"], "*"}},
-        {op_fieldname, {[path_str(Path), "."], "*"}}
+        {op_fieldname, {[path_str(Path)], ".*"}}
     ],
     {op_or, Parts}.
 
@@ -304,7 +301,7 @@ value_str(null) ->
 
 
 append_sort_type(RawSortField, Selector) ->
-    EncodeField = mango_util:lucene_escape_field(RawSortField),
+    EncodeField = mango_util:lucene_escape_user(RawSortField),
     String = mango_util:has_suffix(EncodeField, <<"_3astring">>),
     Number = mango_util:has_suffix(EncodeField, <<"_3anumber">>),
     case {String, Number} of

src/mango_util.erl

@@ -33,8 +33,11 @@
 
     lucene_escape_field/1,
     lucene_escape_query_value/1,
+    lucene_escape_user/1,
 
-    has_suffix/2
+    has_suffix/2,
+
+    join/2
 ]).
 
 
@@ -281,6 +284,12 @@ lucene_escape_qv(<<C, Rest/binary>>) ->
     Out ++ lucene_escape_qv(Rest).
 
 
+lucene_escape_user(Field) ->
+    {ok, Path} = mango_doc:parse_field(Field),
+    Escaped = [mango_util:lucene_escape_field(P) || P <- Path],
+    iolist_to_binary(join(".", Escaped)).
+
+
 has_suffix(Bin, Suffix) when is_binary(Bin), is_binary(Suffix) ->
     SBin = size(Bin),
     SSuffix = size(Suffix),
@@ -293,3 +302,9 @@ has_suffix(Bin, Suffix) when is_binary(Bin), is_binary(Suffix) ->
                 false
         end
     end.
+
+
+join(_Sep, [Item]) ->
+    [Item];
+join(Sep, [Item | Rest]) ->
+    [Item, Sep | join(Sep, Rest)].

test/04-key-tests.py

@@ -26,15 +26,17 @@
         "dot.key": "dot's value",
         "none": {
             "dot": "none dot's value"
-        }
+        },
+        "name.first" : "Kvothe"
     },
     {
         "type": "complex_key",
         "title": "key with peso",
         "$key": "peso",
         "deep": {
             "$key": "deep peso"
-        }
+        },
+        "name": {"first" : "Master Elodin"}
     },
     {
         "type": "complex_key",
@@ -121,3 +123,27 @@ def check(docs):
             assert docs[0]["title"] == "internal_fields_format"
         for query in queries:
             self.run_check(query, check, indexes=["text"])
+
+    def test_escape_period(self):
+        query = {"name\\.first" : "Kvothe"}
+        def check(docs):
+            assert len(docs) == 1
+            assert docs[0]["name.first"] == "Kvothe"
+        self.run_check(query, check, indexes=["text"])
+
+        query = {"name.first" : "Kvothe"}
+        def check_empty(docs):
+            assert len(docs) == 0
+        self.run_check(query, check_empty, indexes=["text"])
+
+    def test_object_period(self):
+        query = {"name.first" : "Master Elodin"}
+        def check(docs):
+            assert len(docs) == 1
+            assert docs[0]["title"] == "key with peso"
+        self.run_check(query, check, indexes=["text"])
+
+        query = {"name\\.first" : "Master Elodin"}
+        def check_empty(docs):
+            assert len(docs) == 0
+        self.run_check(query, check_empty, indexes=["text"])

test/07-text-custom-field-list-test.py

@@ -26,7 +26,8 @@ class CustomFieldsTest(mango.UserDocsTextTests):
         {
             "name": "location.address.street",
             "type": "string"
-        }
+        },
+        {"name": "name\\.first", "type": "string"}
     ]
 
     def test_basic(self):
@@ -60,3 +61,17 @@ def test_field_analyzer_is_keyword(self):
         docs = self.db.find({"location.state": "New Hampshire"})
         assert len(docs) == 1
         assert docs[0]["user_id"] == 10
+
+    # Since our FIELDS list only includes "name\\.first", we should
+    # get an error when we try to search for "name.first", since the index
+    # for that field does not exist.
+    def test_escaped_field(self):
+        docs = self.db.find({"name\\.first": "name dot first"})
+        assert len(docs) == 1
+        assert docs[0]["name.first"] == "name dot first"
+
+        try:
+            self.db.find({"name.first": "name dot first"})
+            raise Exception("Should have thrown an HTTPError")
+        except:
+            return

test/user_docs.py

@@ -224,7 +224,8 @@ def add_text_indexes(db, kwargs):
             "C",
             "Ruby",
             "Ruby"
-        ]
+        ],
+        "name.first" : "name dot first"
     },
     {
         "_id": "a33d5457-741a-4dce-a217-3eab28b24e3e",