Search DSL improvements for filtering bool/or clauses, multiple filte…

…r clauses

search/filter.go

@@ -1,35 +1,131 @@
 package search
 
-import ()
+import (
+	"encoding/json"
+	"fmt"
 
-/*
-"filter": {
-	"range": {
-	  "@timestamp": {
-	    "from": "2012-12-29T16:52:48+00:00",
-	    "to": "2012-12-29T17:52:48+00:00"
-	  }
+	. "github.com/araddon/gou"
+)
+
+var (
+	_ = DEBUG
+)
+
+// A bool (and/or) clause
+type BoolClause string
+
+// Filter clause is either a boolClause or FilterOp
+type FilterClause interface {
+	String() string
+}
+
+// A wrapper to allow for custom serialization
+type FilterWrap struct {
+	boolClause string
+	filters    []interface{}
+}
+
+func NewFilterWrap() *FilterWrap {
+	return &FilterWrap{filters: make([]interface{}, 0), boolClause: "and"}
+}
+
+func (f *FilterWrap) String() string {
+	return fmt.Sprintf(`fopv: %d:%v`, len(f.filters), f.filters)
+}
+
+// Custom marshalling to support the query dsl 
+func (f *FilterWrap) addFilters(fl []interface{}) {
+	if len(fl) > 1 {
+		fc := fl[0]
+		switch fc.(type) {
+		case BoolClause, string:
+			f.boolClause = fc.(string)
+			fl = fl[1:]
+		}
 	}
+	f.filters = append(f.filters, fl...)
 }
-"filter": {
-    "missing": {
-        "field": "repository.name"
-    }
+
+// Custom marshalling to support the query dsl 
+func (f *FilterWrap) MarshalJSON() ([]byte, error) {
+	var root interface{}
+	if len(f.filters) > 1 {
+		root = map[string]interface{}{f.boolClause: f.filters}
+	} else if len(f.filters) == 1 {
+		root = f.filters[0]
+	}
+	return json.Marshal(root)
 }
 
+/*
+	"filter": {
+		"range": {
+		  "@timestamp": {
+		    "from": "2012-12-29T16:52:48+00:00",
+		    "to": "2012-12-29T17:52:48+00:00"
+		  }
+		}
+	}
+	"filter": {
+	    "missing": {
+	        "field": "repository.name"
+	    }
+	}
+
+	"filter" : {
+	    "terms" : {
+	        "user" : ["kimchy", "elasticsearch"],
+	        "execution" : "bool",
+	        "_cache": true
+	    }
+	}
+
+	"filter" : {
+	    "term" : { "user" : "kimchy"}
+	}
+
+	"filter" : {
+	    "and" : [
+	        {
+	            "range" : { 
+	                "postDate" : { 
+	                    "from" : "2010-03-01",
+	                    "to" : "2010-04-01"
+	                }
+	            }
+	        },
+	        {
+	            "prefix" : { "name.second" : "ba" }
+	        }
+	    ]
+	}
+
 */
 
+// Filter Operation
+//
+//   Filter().Term("user","kimchy")
+//    
+//   // we use variadics to allow n arguments, first is the "field" rest are values
+//   Filter().Terms("user", "kimchy", "elasticsearch")
+// 
+//   Filter().Exists("repository.name")
+//
 func Filter() *FilterOp {
 	return &FilterOp{}
 }
 
 type FilterOp struct {
 	curField    string
+	TermsMap    map[string][]interface{}     `json:"terms,omitempty"`
 	Range       map[string]map[string]string `json:"range,omitempty"`
 	Exist       map[string]string            `json:"exists,omitempty"`
 	MisssingVal map[string]string            `json:"missing,omitempty"`
 }
 
+// A range is a special type of Filter operation
+//
+//    Filter().Exists("repository.name")
 func Range() *FilterOp {
 	return &FilterOp{Range: make(map[string]map[string]string)}
 }
@@ -42,6 +138,23 @@ func (f *FilterOp) Field(fld string) *FilterOp {
 	}
 	return f
 }
+
+// Filter Terms
+//
+//   Filter().Terms("user","kimchy")
+//    
+//   // we use variadics to allow n arguments, first is the "field" rest are values
+//   Filter().Terms("user", "kimchy", "elasticsearch")
+func (f *FilterOp) Terms(field string, values ...interface{}) *FilterOp {
+	if len(f.TermsMap) == 0 {
+		f.TermsMap = make(map[string][]interface{})
+	}
+	for _, val := range values {
+		f.TermsMap[field] = append(f.TermsMap[field], val)
+	}
+
+	return f
+}
 func (f *FilterOp) From(from string) *FilterOp {
 	f.Range[f.curField]["from"] = from
 	return f
@@ -61,6 +174,7 @@ func (f *FilterOp) Missing(name string) *FilterOp {
 
 // Add another Filterop, "combines" two filter ops into one
 func (f *FilterOp) Add(fop *FilterOp) *FilterOp {
+	// TODO, this is invalid, refactor
 	if len(fop.Exist) > 0 {
 		f.Exist = fop.Exist
 	}

search/filter_test.go

@@ -0,0 +1,70 @@
+package search
+
+import (
+	//"encoding/json"
+	. "github.com/araddon/gou"
+	"testing"
+)
+
+func TestFilters(t *testing.T) {
+	// search for docs that are missing repository.name
+	qry := Search("github").Filter(
+		Filter().Exists("repository.name"),
+	)
+	out, err := qry.Result()
+	Assert(err == nil, t, "should not have error")
+	Assert(out.Hits.Len() == 10, t, "Should have 10 docs %v", out.Hits.Len())
+	Assert(out.Hits.Total == 7241, t, "Should have 7241 total= %v", out.Hits.Total)
+
+	qry = Search("github").Filter(
+		Filter().Missing("repository.name"),
+	)
+	out, _ = qry.Result()
+	Assert(out.Hits.Len() == 10, t, "Should have 10 docs %v", out.Hits.Len())
+	Assert(out.Hits.Total == 304, t, "Should have 304 total= %v", out.Hits.Total)
+
+	//actor_attributes: {type: "User",
+	qry = Search("github").Filter(
+		Filter().Terms("actor_attributes.location", "portland"),
+	)
+	out, _ = qry.Result()
+	Debug(out)
+	Assert(out.Hits.Len() == 10, t, "Should have 10 docs %v", out.Hits.Len())
+	Assert(out.Hits.Total == 65, t, "Should have 65 total= %v", out.Hits.Total)
+
+	/*
+		Should this be an AND by default?
+	*/
+	qry = Search("github").Filter(
+		Filter().Terms("actor_attributes.location", "portland"),
+		Filter().Terms("repository.has_wiki", true),
+	)
+	out, err = qry.Result()
+	Debug(out)
+	Assert(err == nil, t, "should not have error")
+	Assert(out.Hits.Len() == 10, t, "Should have 10 docs %v", out.Hits.Len())
+	Assert(out.Hits.Total == 43, t, "Should have 43 total= %v", out.Hits.Total)
+
+	// NOW, lets try with two query calls instead of one
+	qry = Search("github").Filter(
+		Filter().Terms("actor_attributes.location", "portland"),
+	)
+	qry.Filter(
+		Filter().Terms("repository.has_wiki", true),
+	)
+	out, err = qry.Result()
+	Debug(out)
+	Assert(err == nil, t, "should not have error")
+	Assert(out.Hits.Len() == 10, t, "Should have 10 docs %v", out.Hits.Len())
+	Assert(out.Hits.Total == 43, t, "Should have 43 total= %v", out.Hits.Total)
+
+	qry = Search("github").Filter(
+		"or",
+		Filter().Terms("actor_attributes.location", "portland"),
+		Filter().Terms("repository.has_wiki", true),
+	)
+	out, err = qry.Result()
+	Assert(err == nil, t, "should not have error")
+	Assert(out.Hits.Len() == 10, t, "Should have 10 docs %v", out.Hits.Len())
+	Assert(out.Hits.Total == 6290, t, "Should have 6290 total= %v", out.Hits.Total)
+}

search/query.go

@@ -91,6 +91,7 @@ func (q *QueryDsl) Range(fop *FilterOp) *QueryDsl {
 		q.FilterVal = fop
 		return q
 	}
+	// TODO:  this is not valid, refactor
 	q.FilterVal.Add(fop)
 	return q
 }

search/search.go

@@ -8,6 +8,12 @@ import (
 	"log"
 	"net/url"
 	"strings"
+
+	. "github.com/araddon/gou"
+)
+
+var (
+	_ = DEBUG
 )
 
 // This is the entry point to the SearchDsl, it is a chainable set of utilities
@@ -28,17 +34,16 @@ func Search(index string) *SearchDsl {
 type SearchDsl struct {
 	args      url.Values
 	types     []string
-	FromVal   int        `json:"from,omitempty"`
-	SizeVal   int        `json:"size,omitempty"`
-	Index     string     `json:"-"`
-	FacetVal  *FacetDsl  `json:"facets,omitempty"`
-	QueryVal  *QueryDsl  `json:"query,omitempty"`
-	SortBody  []*SortDsl `json:"sort,omitempty"`
-	FilterVal *FilterOp  `json:"filter,omitempty"`
+	FromVal   int         `json:"from,omitempty"`
+	SizeVal   int         `json:"size,omitempty"`
+	Index     string      `json:"-"`
+	FacetVal  *FacetDsl   `json:"facets,omitempty"`
+	QueryVal  *QueryDsl   `json:"query,omitempty"`
+	SortBody  []*SortDsl  `json:"sort,omitempty"`
+	FilterVal *FilterWrap `json:"filter,omitempty"`
 }
 
 func (s *SearchDsl) Bytes() ([]byte, error) {
-
 	return api.DoCommand("POST", s.url(), s)
 }
 
@@ -51,6 +56,7 @@ func (s *SearchDsl) Result() (*core.SearchResult, error) {
 	}
 	body, err := s.Bytes()
 	if err != nil {
+		Logf(ERROR, "%v", err)
 		return nil, err
 	}
 	jsonErr := json.Unmarshal([]byte(body), &retval)
@@ -109,8 +115,30 @@ func (s *SearchDsl) Query(q *QueryDsl) *SearchDsl {
 	s.QueryVal = q
 	return s
 }
-func (s *SearchDsl) Filter(f *FilterOp) *SearchDsl {
-	s.FilterVal = f
+
+// Add Filter Clause with optional Boolean Clause.  This accepts n number of
+// filter clauses.  If more than one, and missing Boolean Clause it assumes "and"
+//
+//     qry := Search("github").Filter(
+//         Filter().Exists("repository.name"),
+//     )
+//
+//     qry := Search("github").Filter(
+//         "or",
+//         Filter().Exists("repository.name"),
+//         Filter().Terms("actor_attributes.location", "portland"),
+//     )
+//
+//     qry := Search("github").Filter(
+//         Filter().Exists("repository.name"),
+//         Filter().Terms("repository.has_wiki", true)
+//     )
+func (s *SearchDsl) Filter(fl ...interface{}) *SearchDsl {
+	if s.FilterVal == nil {
+		s.FilterVal = NewFilterWrap()
+	}
+
+	s.FilterVal.addFilters(fl)
 	return s
 }
 
@@ -121,42 +149,3 @@ func (s *SearchDsl) Sort(sort ...*SortDsl) *SearchDsl {
 	s.SortBody = append(s.SortBody, sort...)
 	return s
 }
-
-/* 
-	Sorting accepts any number of Sort commands
-
-	Query().Sort(
-		Sort("last_name").Desc(),
-		Sort("age"),
-	)
-*/
-func Sort(field string) *SortDsl {
-	return &SortDsl{Name: field}
-}
-
-type SortBody []interface{}
-type SortDsl struct {
-	Name   string
-	IsDesc bool
-}
-
-func (s *SortDsl) Desc() *SortDsl {
-	s.IsDesc = true
-	return s
-}
-func (s *SortDsl) Asc() *SortDsl {
-	s.IsDesc = false
-	return s
-}
-func (s *SortDsl) MarshalJSON() ([]byte, error) {
-	if s.IsDesc {
-		return json.Marshal(map[string]string{s.Name: "desc"})
-	}
-	if s.Name == "_score" {
-		return []byte(`"_score"`), nil
-	}
-	return []byte(fmt.Sprintf(`"%s"`, s.Name)), nil // "user"  assuming default = asc?
-	// TODO
-	//    { "price" : {"missing" : "_last"} },
-	//    { "price" : {"ignore_unmapped" : true} },
-}