From d68401eb409636883eb50469b18d5d98ea46d7d0 Mon Sep 17 00:00:00 2001
From: Mike Yoon <mike@yaki.org>
Date: Sat, 21 Mar 2015 23:02:46 -0700
Subject: [PATCH 01/81] Add term search filter and modify search query json

Modified QueryDsl to add a query parent node to filtered queries when
serializing
---
 lib/searchfilter.go | 18 +++++++++++++++---
 lib/searchquery.go  |  2 +-
 2 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/lib/searchfilter.go b/lib/searchfilter.go
index 82830e75..c9b2eeea 100644
--- a/lib/searchfilter.go
+++ b/lib/searchfilter.go
@@ -135,6 +135,7 @@ func CompoundFilter(fl ...interface{}) *FilterWrap {
 type FilterOp struct {
 	curField    string
 	TermsMap    map[string][]interface{}          `json:"terms,omitempty"`
+	TermMap     map[string]interface{}            `json:"term,omitempty"`
 	Range       map[string]map[string]interface{} `json:"range,omitempty"`
 	Exist       map[string]string                 `json:"exists,omitempty"`
 	MisssingVal map[string]string                 `json:"missing,omitempty"`
@@ -156,6 +157,17 @@ func (f *FilterOp) Field(fld string) *FilterOp {
 	return f
 }
 
+func (f *FilterOp) Term(field string, value interface{}) *FilterOp {
+	//Multiple terms in a filter may not be compatible with older versions of
+	//ElasticSearch
+	if len(f.TermMap) == 0 {
+		f.TermMap = make(map[string]interface{})
+	}
+
+	f.TermMap[field] = value
+	return f
+}
+
 // Filter Terms
 //
 //   Filter().Terms("user","kimchy")
@@ -164,9 +176,9 @@ func (f *FilterOp) Field(fld string) *FilterOp {
 //   Filter().Terms("user", "kimchy", "elasticsearch")
 //
 func (f *FilterOp) Terms(field string, values ...interface{}) *FilterOp {
-	if len(f.TermsMap) == 0 {
-		f.TermsMap = make(map[string][]interface{})
-	}
+	//You can only have one terms in a filter
+	f.TermsMap = make(map[string][]interface{})
+
 	for _, val := range values {
 		f.TermsMap[field] = append(f.TermsMap[field], val)
 	}
diff --git a/lib/searchquery.go b/lib/searchquery.go
index 3c2ebb8d..a3ab6408 100644
--- a/lib/searchquery.go
+++ b/lib/searchquery.go
@@ -85,7 +85,7 @@ func (qd *QueryDsl) MarshalJSON() ([]byte, error) {
 		if err != nil {
 			return filterB, err
 		}
-		return []byte(fmt.Sprintf(`{"filtered":{"query":%s,"filter":%s}}`, queryB, filterB)), nil
+		return []byte(fmt.Sprintf(`{"query":{"filtered":{"query":%s,"filter":%s}}}`, queryB, filterB)), nil
 	}
 	return json.Marshal(q)
 }

From cf98c144ec0ad34171f23257ef38e1cddc2876f7 Mon Sep 17 00:00:00 2001
From: Mike Yoon <mike@yaki.org>
Date: Wed, 25 Mar 2015 00:12:20 -0700
Subject: [PATCH 02/81] Add and filter

---
 lib/searchfilter.go | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/lib/searchfilter.go b/lib/searchfilter.go
index c9b2eeea..387916f9 100644
--- a/lib/searchfilter.go
+++ b/lib/searchfilter.go
@@ -139,6 +139,7 @@ type FilterOp struct {
 	Range       map[string]map[string]interface{} `json:"range,omitempty"`
 	Exist       map[string]string                 `json:"exists,omitempty"`
 	MisssingVal map[string]string                 `json:"missing,omitempty"`
+	AndFilters  []FilterOp                        `json:"and,omitempty"`
 }
 
 // A range is a special type of Filter operation
@@ -168,6 +169,13 @@ func (f *FilterOp) Term(field string, value interface{}) *FilterOp {
 	return f
 }
 
+func (f *FilterOp) And(filter *FilterOp) *FilterOp {
+	if len(f.AndFilters) == 0 {
+		f.AndFilters = []FilterOp{*filter}
+
+	return f
+}
+
 // Filter Terms
 //
 //   Filter().Terms("user","kimchy")

From dc4e366cfa1112e3dd791261aa0cbf7bca880946 Mon Sep 17 00:00:00 2001
From: Mike Yoon <mike@yaki.org>
Date: Wed, 25 Mar 2015 00:12:34 -0700
Subject: [PATCH 03/81] Fix and filter

---
 lib/searchfilter.go | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/lib/searchfilter.go b/lib/searchfilter.go
index 387916f9..2bd8573a 100644
--- a/lib/searchfilter.go
+++ b/lib/searchfilter.go
@@ -172,6 +172,9 @@ func (f *FilterOp) Term(field string, value interface{}) *FilterOp {
 func (f *FilterOp) And(filter *FilterOp) *FilterOp {
 	if len(f.AndFilters) == 0 {
 		f.AndFilters = []FilterOp{*filter}
+	} else {
+        f.AndFilters = append(f.AndFilters, *filter)
+    }
 
 	return f
 }

From d6236b0c252981a35c08a1c702d7c201b7b4fb88 Mon Sep 17 00:00:00 2001
From: Mike Yoon <mike@yaki.org>
Date: Wed, 25 Mar 2015 00:19:40 -0700
Subject: [PATCH 04/81] Add or filter

---
 lib/searchfilter.go | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/lib/searchfilter.go b/lib/searchfilter.go
index 2bd8573a..364e6ecf 100644
--- a/lib/searchfilter.go
+++ b/lib/searchfilter.go
@@ -140,6 +140,7 @@ type FilterOp struct {
 	Exist       map[string]string                 `json:"exists,omitempty"`
 	MisssingVal map[string]string                 `json:"missing,omitempty"`
 	AndFilters  []FilterOp                        `json:"and,omitempty"`
+	OrFilters   []FilterOp                        `json:"or,omitempty"`
 }
 
 // A range is a special type of Filter operation
@@ -173,8 +174,18 @@ func (f *FilterOp) And(filter *FilterOp) *FilterOp {
 	if len(f.AndFilters) == 0 {
 		f.AndFilters = []FilterOp{*filter}
 	} else {
-        f.AndFilters = append(f.AndFilters, *filter)
-    }
+		f.AndFilters = append(f.AndFilters, *filter)
+	}
+
+	return f
+}
+
+func (f *FilterOp) Or(filter *FilterOp) *FilterOp {
+	if len(f.OrFilters) == 0 {
+		f.OrFilters = []FilterOp{*filter}
+	} else {
+		f.OrFilters = append(f.OrFilters, *filter)
+	}
 
 	return f
 }

From 736af2bf71bffe82b9c02c2813e8b6ac623ec639 Mon Sep 17 00:00:00 2001
From: Mike Yoon <mike@yaki.org>
Date: Thu, 26 Mar 2015 00:27:57 -0700
Subject: [PATCH 05/81] Cleanup filter dsl a bit

* Added an AddRange function for creating a range filter
* Change QueryDsl to return a query root node when there's no filter
---
 lib/searchfilter.go | 98 ++++++++++++++++++---------------------------
 lib/searchquery.go  |  6 ++-
 2 files changed, 43 insertions(+), 61 deletions(-)

diff --git a/lib/searchfilter.go b/lib/searchfilter.go
index 364e6ecf..3bcbb0ec 100644
--- a/lib/searchfilter.go
+++ b/lib/searchfilter.go
@@ -134,34 +134,38 @@ func CompoundFilter(fl ...interface{}) *FilterWrap {
 
 type FilterOp struct {
 	curField    string
-	TermsMap    map[string][]interface{}          `json:"terms,omitempty"`
-	TermMap     map[string]interface{}            `json:"term,omitempty"`
-	Range       map[string]map[string]interface{} `json:"range,omitempty"`
-	Exist       map[string]string                 `json:"exists,omitempty"`
-	MisssingVal map[string]string                 `json:"missing,omitempty"`
-	AndFilters  []FilterOp                        `json:"and,omitempty"`
-	OrFilters   []FilterOp                        `json:"or,omitempty"`
+	TermsMap    map[string][]interface{} `json:"terms,omitempty"`
+	TermMap     map[string]interface{}   `json:"term,omitempty"`
+	RangeMap    map[string]RangeFilter   `json:"range,omitempty"`
+	Exist       map[string]string        `json:"exists,omitempty"`
+	MisssingVal map[string]string        `json:"missing,omitempty"`
+	AndFilters  []FilterOp               `json:"and,omitempty"`
+	OrFilters   []FilterOp               `json:"or,omitempty"`
+	Limit       *LimitFilter             `json:"limit,omitempty"`
+}
+
+type LimitFilter struct {
+	Value int `json:"value,omitempty"`
+}
+
+type RangeFilter struct {
+	Gte      interface{} `json:"gte,omitempty"`
+	Lte      interface{} `json:"lte,omitempty"`
+	Gt       interface{} `json:"gt,omitempty"`
+	Lt       interface{} `json:"lt,omitempty"`
+	TimeZone string      `json:"time_zone,omitempty"` //Ideally this would be an int
 }
 
 // A range is a special type of Filter operation
 //
 //    Range().Exists("repository.name")
 func Range() *FilterOp {
-	return &FilterOp{Range: make(map[string]map[string]interface{})}
-}
-
-func (f *FilterOp) Field(fld string) *FilterOp {
-	f.curField = fld
-	if _, ok := f.Range[fld]; !ok {
-		m := make(map[string]interface{})
-		f.Range[fld] = m
-	}
-	return f
+	return &FilterOp{RangeMap: make(map[string]RangeFilter)}
 }
 
+// Term will add a term to the filter.
+// Multiple Term filters can be added, and ES will OR them.
 func (f *FilterOp) Term(field string, value interface{}) *FilterOp {
-	//Multiple terms in a filter may not be compatible with older versions of
-	//ElasticSearch
 	if len(f.TermMap) == 0 {
 		f.TermMap = make(map[string]interface{})
 	}
@@ -192,11 +196,8 @@ func (f *FilterOp) Or(filter *FilterOp) *FilterOp {
 
 // Filter Terms
 //
-//   Filter().Terms("user","kimchy")
-//
-//   // we use variadics to allow n arguments, first is the "field" rest are values
-//   Filter().Terms("user", "kimchy", "elasticsearch")
-//
+//   Filter().Terms("user","kimchy","stuff")
+//	 Note: you can only have one terms clause in a filter. Use a bool filter to combine
 func (f *FilterOp) Terms(field string, values ...interface{}) *FilterOp {
 	//You can only have one terms in a filter
 	f.TermsMap = make(map[string][]interface{})
@@ -207,42 +208,21 @@ func (f *FilterOp) Terms(field string, values ...interface{}) *FilterOp {
 
 	return f
 }
-func (f *FilterOp) From(from string) *FilterOp {
-	f.Range[f.curField]["from"] = from
-	return f
-}
-func (f *FilterOp) To(to string) *FilterOp {
-	f.Range[f.curField]["to"] = to
-	return f
-}
-func (f *FilterOp) Gt(gt interface{}) *FilterOp {
-	f.Range[f.curField]["gt"] = gt
-	return f
-}
-func (f *FilterOp) Lt(lt interface{}) *FilterOp {
-	f.Range[f.curField]["lt"] = lt
-	return f
-}
-func (f *FilterOp) Exists(name string) *FilterOp {
-	f.Exist = map[string]string{"field": name}
-	return f
-}
-func (f *FilterOp) Missing(name string) *FilterOp {
-	f.MisssingVal = map[string]string{"field": name}
-	return f
-}
 
-// Add another Filterop, "combines" two filter ops into one
-func (f *FilterOp) Add(fop *FilterOp) *FilterOp {
-	// TODO, this is invalid, refactor
-	if len(fop.Exist) > 0 {
-		f.Exist = fop.Exist
-	}
-	if len(fop.MisssingVal) > 0 {
-		f.MisssingVal = fop.MisssingVal
-	}
-	if len(fop.Range) > 0 {
-		f.Range = fop.Range
+// AddRange adds a range filter for the given field.
+func (f *FilterOp) AddRange(field string, gte interface{},
+	gt interface{}, lte interface{}, lt interface{}, timeZone string) *FilterOp {
+
+	if f.RangeMap == nil {
+		f.RangeMap = make(map[string]RangeFilter)
 	}
+
+	f.RangeMap[field] = RangeFilter{
+		Gte:      gte,
+		Gt:       gt,
+		Lte:      lte,
+		Lt:       lt,
+		TimeZone: timeZone}
+
 	return f
 }
diff --git a/lib/searchquery.go b/lib/searchquery.go
index a3ab6408..489d6cd4 100644
--- a/lib/searchquery.go
+++ b/lib/searchquery.go
@@ -87,7 +87,9 @@ func (qd *QueryDsl) MarshalJSON() ([]byte, error) {
 		}
 		return []byte(fmt.Sprintf(`{"query":{"filtered":{"query":%s,"filter":%s}}}`, queryB, filterB)), nil
 	}
-	return json.Marshal(q)
+
+	retval, err := json.Marshal(q)
+	return []byte(fmt.Sprintf(`{"query": %s}`, retval)), err
 }
 
 // get all
@@ -103,7 +105,7 @@ func (q *QueryDsl) Range(fop *FilterOp) *QueryDsl {
 		return q
 	}
 	// TODO:  this is not valid, refactor
-	q.FilterVal.Add(fop)
+	//q.FilterVal.Add(fop)
 	return q
 }
 

From 64df646c43f42f750bd81cc7304d750c5f044344 Mon Sep 17 00:00:00 2001
From: Mike Yoon <mike@yaki.org>
Date: Thu, 26 Mar 2015 00:49:25 -0700
Subject: [PATCH 06/81] Remove the extra root query node

---
 lib/searchquery.go | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/lib/searchquery.go b/lib/searchquery.go
index 489d6cd4..1c7028a0 100644
--- a/lib/searchquery.go
+++ b/lib/searchquery.go
@@ -85,11 +85,9 @@ func (qd *QueryDsl) MarshalJSON() ([]byte, error) {
 		if err != nil {
 			return filterB, err
 		}
-		return []byte(fmt.Sprintf(`{"query":{"filtered":{"query":%s,"filter":%s}}}`, queryB, filterB)), nil
+		return []byte(fmt.Sprintf(`{"filtered":{"query":%s,"filter":%s}}`, queryB, filterB)), nil
 	}
-
-	retval, err := json.Marshal(q)
-	return []byte(fmt.Sprintf(`{"query": %s}`, retval)), err
+	return json.Marshal(q)
 }
 
 // get all

From ce50bb17b90c25e9f0308575902569d550f81416 Mon Sep 17 00:00:00 2001
From: Mike Yoon <mike@yaki.org>
Date: Thu, 26 Mar 2015 23:23:14 -0700
Subject: [PATCH 07/81] Add limit to filterop and lenient to querydsl

---
 lib/searchfilter.go | 7 ++++++-
 lib/searchquery.go  | 8 +++++++-
 2 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/lib/searchfilter.go b/lib/searchfilter.go
index 3bcbb0ec..0d85bbb2 100644
--- a/lib/searchfilter.go
+++ b/lib/searchfilter.go
@@ -145,7 +145,7 @@ type FilterOp struct {
 }
 
 type LimitFilter struct {
-	Value int `json:"value,omitempty"`
+	Value int `json:"value"`
 }
 
 type RangeFilter struct {
@@ -226,3 +226,8 @@ func (f *FilterOp) AddRange(field string, gte interface{},
 
 	return f
 }
+
+func (f *FilterOp) SetLimit(maxResults int) *FilterOp {
+	f.Limit = &LimitFilter{Value: maxResults}
+	return f
+}
diff --git a/lib/searchquery.go b/lib/searchquery.go
index 1c7028a0..555c464f 100644
--- a/lib/searchquery.go
+++ b/lib/searchquery.go
@@ -132,6 +132,11 @@ func (q *QueryDsl) Qs(qs *QueryString) *QueryDsl {
 	return q
 }
 
+func (q *QueryDsl) SetLenient(lenient bool) *QueryDsl {
+	q.QueryEmbed.Qs.Lenient = lenient
+	return q
+}
+
 // Fields in query_string search
 //     Fields("fieldname","search_for","","")
 //
@@ -170,7 +175,7 @@ type QueryWrap struct {
 
 // QueryString based search
 func NewQueryString(field, query string) QueryString {
-	return QueryString{"", field, query, "", "", nil}
+	return QueryString{"", field, query, "", "", nil, false}
 }
 
 type QueryString struct {
@@ -180,6 +185,7 @@ type QueryString struct {
 	Exists          string   `json:"_exists_,omitempty"`
 	Missing         string   `json:"_missing_,omitempty"`
 	Fields          []string `json:"fields,omitempty"`
+	Lenient         bool     `json:"lenient,omitempty"`
 	//_exists_:field1,
 	//_missing_:field1,
 }

From 963428c868bd3b92a6eaa4462ad9ee6c66568a32 Mon Sep 17 00:00:00 2001
From: Mike Yoon <myoon@airware.com>
Date: Tue, 31 Mar 2015 19:02:49 -0700
Subject: [PATCH 08/81] Add more filters to the filter struct

---
 lib/searchfilter.go | 49 +++++++++++++++++++++++++++++----------------
 1 file changed, 32 insertions(+), 17 deletions(-)

diff --git a/lib/searchfilter.go b/lib/searchfilter.go
index 0d85bbb2..b48eabd5 100644
--- a/lib/searchfilter.go
+++ b/lib/searchfilter.go
@@ -14,7 +14,6 @@ package elastigo
 import (
 	"encoding/json"
 	"fmt"
-
 	. "github.com/araddon/gou"
 )
 
@@ -133,21 +132,40 @@ func CompoundFilter(fl ...interface{}) *FilterWrap {
 }
 
 type FilterOp struct {
-	curField    string
-	TermsMap    map[string][]interface{} `json:"terms,omitempty"`
-	TermMap     map[string]interface{}   `json:"term,omitempty"`
-	RangeMap    map[string]RangeFilter   `json:"range,omitempty"`
-	Exist       map[string]string        `json:"exists,omitempty"`
-	MisssingVal map[string]string        `json:"missing,omitempty"`
-	AndFilters  []FilterOp               `json:"and,omitempty"`
-	OrFilters   []FilterOp               `json:"or,omitempty"`
-	Limit       *LimitFilter             `json:"limit,omitempty"`
+	TermsMap   map[string][]interface{} `json:"terms,omitempty"`
+	TermMap    map[string]interface{}   `json:"term,omitempty"`
+	RangeMap   map[string]RangeFilter   `json:"range,omitempty"`
+	ExistMap   map[string]string        `json:"exists,omitempty"`
+	MissingMap map[string]string        `json:"missing,omitempty"`
+	AndFilters []FilterOp               `json:"and,omitempty"`
+	OrFilters  []FilterOp               `json:"or,omitempty"`
+	NotFilters []FilterOp               `json:"not,omitempty"`
+	Limit      *LimitFilter             `json:"limit,omitempty"`
+	Type       *TypeFilter              `json:"type,omitempty"`
+	Ids        *IdFilter                `json:"ids,omitempty"`
+	Script     *ScriptFilter            `json:"script,omitempty"`
+	GeoDistance *GeoDistanceFilter		`json"geo_distance,omitempty"`
 }
 
 type LimitFilter struct {
 	Value int `json:"value"`
 }
 
+type TypeFilter struct {
+	Value string `json:"value"`
+}
+
+type IdFilter struct {
+	Type   string   `json:"type,omitempty"`
+	Values []string `json:"values,omitempty"`
+}
+
+type ScriptFilter struct {
+	Script   string                 `json:"script"`
+	Params   map[string]interface{} `json:"params,omitempty"`
+	IsCached bool                   `json:"_cache,omitempty"`
+}
+
 type RangeFilter struct {
 	Gte      interface{} `json:"gte,omitempty"`
 	Lte      interface{} `json:"lte,omitempty"`
@@ -156,11 +174,8 @@ type RangeFilter struct {
 	TimeZone string      `json:"time_zone,omitempty"` //Ideally this would be an int
 }
 
-// A range is a special type of Filter operation
-//
-//    Range().Exists("repository.name")
-func Range() *FilterOp {
-	return &FilterOp{RangeMap: make(map[string]RangeFilter)}
+type GeoDistanceFilter struct {
+	Distance string `json:"distance"`
 }
 
 // Term will add a term to the filter.
@@ -209,8 +224,8 @@ func (f *FilterOp) Terms(field string, values ...interface{}) *FilterOp {
 	return f
 }
 
-// AddRange adds a range filter for the given field.
-func (f *FilterOp) AddRange(field string, gte interface{},
+// Range adds a range filter for the given field.
+func (f *FilterOp) Range(field string, gte interface{},
 	gt interface{}, lte interface{}, lt interface{}, timeZone string) *FilterOp {
 
 	if f.RangeMap == nil {

From 967d7c756418344521104f96903d4fc7a776a8b9 Mon Sep 17 00:00:00 2001
From: Mike Yoon <myoon@airware.com>
Date: Wed, 1 Apr 2015 19:53:27 -0700
Subject: [PATCH 09/81] Add geodistance filters

---
 lib/searchfilter.go | 66 ++++++++++++++++++++++++++++++++++-----------
 1 file changed, 51 insertions(+), 15 deletions(-)

diff --git a/lib/searchfilter.go b/lib/searchfilter.go
index b48eabd5..61fb65ae 100644
--- a/lib/searchfilter.go
+++ b/lib/searchfilter.go
@@ -132,19 +132,20 @@ func CompoundFilter(fl ...interface{}) *FilterWrap {
 }
 
 type FilterOp struct {
-	TermsMap   map[string][]interface{} `json:"terms,omitempty"`
-	TermMap    map[string]interface{}   `json:"term,omitempty"`
-	RangeMap   map[string]RangeFilter   `json:"range,omitempty"`
-	ExistMap   map[string]string        `json:"exists,omitempty"`
-	MissingMap map[string]string        `json:"missing,omitempty"`
-	AndFilters []FilterOp               `json:"and,omitempty"`
-	OrFilters  []FilterOp               `json:"or,omitempty"`
-	NotFilters []FilterOp               `json:"not,omitempty"`
-	Limit      *LimitFilter             `json:"limit,omitempty"`
-	Type       *TypeFilter              `json:"type,omitempty"`
-	Ids        *IdFilter                `json:"ids,omitempty"`
-	Script     *ScriptFilter            `json:"script,omitempty"`
-	GeoDistance *GeoDistanceFilter		`json"geo_distance,omitempty"`
+	TermsMap     map[string][]interface{} `json:"terms,omitempty"`
+	TermMap      map[string]interface{}   `json:"term,omitempty"`
+	RangeMap     map[string]RangeFilter   `json:"range,omitempty"`
+	ExistMap     map[string]string        `json:"exists,omitempty"`
+	MissingMap   map[string]string        `json:"missing,omitempty"`
+	AndFilters   []FilterOp               `json:"and,omitempty"`
+	OrFilters    []FilterOp               `json:"or,omitempty"`
+	NotFilters   []FilterOp               `json:"not,omitempty"`
+	Limit        *LimitFilter             `json:"limit,omitempty"`
+	Type         *TypeFilter              `json:"type,omitempty"`
+	Ids          *IdFilter                `json:"ids,omitempty"`
+	Script       *ScriptFilter            `json:"script,omitempty"`
+	GeoDist      map[string]interface{}   `json:"geo_distance,omitempty"`
+	GeoDistRange map[string]interface{}   `json:"geo_distance_range,omitempty"`
 }
 
 type LimitFilter struct {
@@ -174,8 +175,14 @@ type RangeFilter struct {
 	TimeZone string      `json:"time_zone,omitempty"` //Ideally this would be an int
 }
 
-type GeoDistanceFilter struct {
-	Distance string `json:"distance"`
+type GeoLocation struct {
+	Latitude  float32 `json:"lat"`
+	Longitude float32 `json:"lon"`
+}
+
+type GeoField struct {
+	GeoLocation
+	Field string
 }
 
 // Term will add a term to the filter.
@@ -209,6 +216,35 @@ func (f *FilterOp) Or(filter *FilterOp) *FilterOp {
 	return f
 }
 
+func (f *FilterOp) GeoDistance(distance string, fields ...GeoField) *FilterOp {
+	f.GeoDist = make(map[string]interface{})
+	f.GeoDist["distance"] = distance
+	for _, val := range fields {
+		f.GeoDist[val.Field] = val.GeoLocation
+	}
+
+	return f
+}
+
+func (f *FilterOp) GeoDistanceRange(from string, to string, fields ...GeoField) *FilterOp {
+	f.GeoDist = make(map[string]interface{})
+	f.GeoDist["from"] = from
+	f.GeoDist["to"] = to
+
+	for _, val := range fields {
+		f.GeoDist[val.Field] = val.GeoLocation
+	}
+
+	return f
+}
+
+// Helper to create values for the GeoDistance filters
+func NewGeoField(field string, latitude float32, longitude float32) GeoField {
+	return GeoField{
+		GeoLocation: GeoLocation{Latitude: latitude, Longitude: longitude},
+		Field:       field}
+}
+
 // Filter Terms
 //
 //   Filter().Terms("user","kimchy","stuff")

From 61a34658f8e2a2847f8ec6f10d8de527d842f505 Mon Sep 17 00:00:00 2001
From: Mike Yoon <mike@yaki.org>
Date: Thu, 2 Apr 2015 01:09:27 -0700
Subject: [PATCH 10/81] Add exists and missing filter create functions

Also add execution mode to terms filter create
Update searchfilter_test to match new Dsl
---
 lib/searchfilter.go      | 35 ++++++++++++++++++++++++++++++++---
 lib/searchfilter_test.go | 20 ++++++++++----------
 lib/searchsearch.go      |  6 +++---
 3 files changed, 45 insertions(+), 16 deletions(-)

diff --git a/lib/searchfilter.go b/lib/searchfilter.go
index 61fb65ae..3b4adbd7 100644
--- a/lib/searchfilter.go
+++ b/lib/searchfilter.go
@@ -24,6 +24,17 @@ var (
 // A bool (and/or) clause
 type BoolClause string
 
+type TermExecutionMode string
+
+const (
+	TEM_DEFAULT TermExecutionMode = ""
+	TEM_PLAIN                     = "plain"
+	TEM_FIELD                     = "field_data"
+	TEM_BOOL                      = "bool"
+	TEM_AND                       = "and"
+	TEM_OR                        = "or"
+)
+
 // Filter clause is either a boolClause or FilterOp
 type FilterClause interface {
 	String() string
@@ -135,8 +146,8 @@ type FilterOp struct {
 	TermsMap     map[string][]interface{} `json:"terms,omitempty"`
 	TermMap      map[string]interface{}   `json:"term,omitempty"`
 	RangeMap     map[string]RangeFilter   `json:"range,omitempty"`
-	ExistMap     map[string]string        `json:"exists,omitempty"`
-	MissingMap   map[string]string        `json:"missing,omitempty"`
+	ExistsProp   *PropertyPathMarker      `json:"exists,omitempty"`
+	MissingProp  *PropertyPathMarker      `json:"missing,omitempty"`
 	AndFilters   []FilterOp               `json:"and,omitempty"`
 	OrFilters    []FilterOp               `json:"or,omitempty"`
 	NotFilters   []FilterOp               `json:"not,omitempty"`
@@ -148,6 +159,10 @@ type FilterOp struct {
 	GeoDistRange map[string]interface{}   `json:"geo_distance_range,omitempty"`
 }
 
+type PropertyPathMarker struct {
+	Field string `json:"field"`
+}
+
 type LimitFilter struct {
 	Value int `json:"value"`
 }
@@ -249,10 +264,14 @@ func NewGeoField(field string, latitude float32, longitude float32) GeoField {
 //
 //   Filter().Terms("user","kimchy","stuff")
 //	 Note: you can only have one terms clause in a filter. Use a bool filter to combine
-func (f *FilterOp) Terms(field string, values ...interface{}) *FilterOp {
+func (f *FilterOp) Terms(field string, executionMode TermExecutionMode, values ...interface{}) *FilterOp {
 	//You can only have one terms in a filter
 	f.TermsMap = make(map[string][]interface{})
 
+	if executionMode != "" {
+		f.TermsMap["execution"] = executionMode
+	}
+
 	for _, val := range values {
 		f.TermsMap[field] = append(f.TermsMap[field], val)
 	}
@@ -278,6 +297,16 @@ func (f *FilterOp) Range(field string, gte interface{},
 	return f
 }
 
+func (f *FilterOp) Exists(field string) *FilterOp {
+	f.ExistsProp = &PropertyPathMarker{Field: field}
+	return f
+}
+
+func (f *FilterOp) Missing(field string) *FilterOp {
+	f.MissingProp = &PropertyPathMarker{Field: field}
+	return f
+}
+
 func (f *FilterOp) SetLimit(maxResults int) *FilterOp {
 	f.Limit = &LimitFilter{Value: maxResults}
 	return f
diff --git a/lib/searchfilter_test.go b/lib/searchfilter_test.go
index 7f533891..384d04a2 100644
--- a/lib/searchfilter_test.go
+++ b/lib/searchfilter_test.go
@@ -41,7 +41,7 @@ func TestFilters(t *testing.T) {
 
 	//actor_attributes: {type: "User",
 	qry = Search("github").Filter(
-		Filter().Terms("actor_attributes.location", "portland"),
+		Filter().Terms("actor_attributes.location", TEM_DEFAULT, "portland"),
 	)
 	out, _ = qry.Result(c)
 	expectedDocs = 10
@@ -53,8 +53,8 @@ func TestFilters(t *testing.T) {
 		Should this be an AND by default?
 	*/
 	qry = Search("github").Filter(
-		Filter().Terms("actor_attributes.location", "portland"),
-		Filter().Terms("repository.has_wiki", true),
+		Filter().Terms("actor_attributes.location", TEM_DEFAULT, "portland"),
+		Filter().Terms("repository.has_wiki", TEM_DEFAULT, true),
 	)
 	out, err = qry.Result(c)
 	expectedDocs = 10
@@ -65,10 +65,10 @@ func TestFilters(t *testing.T) {
 
 	// NOW, lets try with two query calls instead of one
 	qry = Search("github").Filter(
-		Filter().Terms("actor_attributes.location", "portland"),
+		Filter().Terms("actor_attributes.location", TEM_DEFAULT, "portland"),
 	)
 	qry.Filter(
-		Filter().Terms("repository.has_wiki", true),
+		Filter().Terms("repository.has_wiki", TEM_DEFAULT, true),
 	)
 	out, err = qry.Result(c)
 	//gou.Debug(out)
@@ -78,8 +78,8 @@ func TestFilters(t *testing.T) {
 
 	qry = Search("github").Filter(
 		"or",
-		Filter().Terms("actor_attributes.location", "portland"),
-		Filter().Terms("repository.has_wiki", true),
+		Filter().Terms("actor_attributes.location", TEM_DEFAULT, "portland"),
+		Filter().Terms("repository.has_wiki", TEM_DEFAULT, true),
 	)
 	out, err = qry.Result(c)
 	expectedHits = 6676
@@ -92,9 +92,9 @@ func TestFilterRange(t *testing.T) {
 	c := NewTestConn()
 
 	// now lets filter range for repositories with more than 100 forks
-	out, _ := Search("github").Size("25").Filter(
-		Range().Field("repository.forks").From("100"),
-	).Result(c)
+	out, _ := Search("github").Size("25").Filter(Filter().
+		Range("repository.forks", 100, nil, nil, nil, "")).Result(c)
+
 	if out == nil || &out.Hits == nil {
 		t.Fail()
 		return
diff --git a/lib/searchsearch.go b/lib/searchsearch.go
index 820f23d9..bbab5d14 100644
--- a/lib/searchsearch.go
+++ b/lib/searchsearch.go
@@ -47,7 +47,7 @@ type SearchDsl struct {
 	FacetVal      *FacetDsl                `json:"facets,omitempty"`
 	QueryVal      *QueryDsl                `json:"query,omitempty"`
 	SortBody      []*SortDsl               `json:"sort,omitempty"`
-	FilterVal     *FilterWrap              `json:"filter,omitempty"`
+	FilterVal     *FilterOp                `json:"filter,omitempty"`
 	AggregatesVal map[string]*AggregateDsl `json:"aggregations,omitempty"`
 }
 
@@ -173,12 +173,12 @@ func (s *SearchDsl) Query(q *QueryDsl) *SearchDsl {
 //         Filter().Exists("repository.name"),
 //         Filter().Terms("repository.has_wiki", true)
 //     )
-func (s *SearchDsl) Filter(fl ...interface{}) *SearchDsl {
+func (s *SearchDsl) Filter(fl FilterOp) *SearchDsl {
 	if s.FilterVal == nil {
 		s.FilterVal = NewFilterWrap()
 	}
 
-	s.FilterVal.addFilters(fl)
+	s.FilterVal = fl
 	return s
 }
 

From aa538426af99b59ccc68a79d4fdff4f66cdaadec Mon Sep 17 00:00:00 2001
From: Mike Yoon <mike@yaki.org>
Date: Thu, 2 Apr 2015 01:18:18 -0700
Subject: [PATCH 11/81] Fix Terms filter and compile error

---
 lib/searchsearch.go | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/lib/searchsearch.go b/lib/searchsearch.go
index bbab5d14..731d1e6a 100644
--- a/lib/searchsearch.go
+++ b/lib/searchsearch.go
@@ -173,11 +173,7 @@ func (s *SearchDsl) Query(q *QueryDsl) *SearchDsl {
 //         Filter().Exists("repository.name"),
 //         Filter().Terms("repository.has_wiki", true)
 //     )
-func (s *SearchDsl) Filter(fl FilterOp) *SearchDsl {
-	if s.FilterVal == nil {
-		s.FilterVal = NewFilterWrap()
-	}
-
+func (s *SearchDsl) Filter(fl *FilterOp) *SearchDsl {
 	s.FilterVal = fl
 	return s
 }

From c394b750b43b433550e760ae194601096cf0c30f Mon Sep 17 00:00:00 2001
From: Mike Yoon <mike@yaki.org>
Date: Thu, 2 Apr 2015 01:19:16 -0700
Subject: [PATCH 12/81] Forgot this file change for the last commit

---
 lib/searchfilter.go | 34 ++++++++++++++++------------------
 1 file changed, 16 insertions(+), 18 deletions(-)

diff --git a/lib/searchfilter.go b/lib/searchfilter.go
index 3b4adbd7..d8d07082 100644
--- a/lib/searchfilter.go
+++ b/lib/searchfilter.go
@@ -143,20 +143,20 @@ func CompoundFilter(fl ...interface{}) *FilterWrap {
 }
 
 type FilterOp struct {
-	TermsMap     map[string][]interface{} `json:"terms,omitempty"`
-	TermMap      map[string]interface{}   `json:"term,omitempty"`
-	RangeMap     map[string]RangeFilter   `json:"range,omitempty"`
-	ExistsProp   *PropertyPathMarker      `json:"exists,omitempty"`
-	MissingProp  *PropertyPathMarker      `json:"missing,omitempty"`
-	AndFilters   []FilterOp               `json:"and,omitempty"`
-	OrFilters    []FilterOp               `json:"or,omitempty"`
-	NotFilters   []FilterOp               `json:"not,omitempty"`
-	Limit        *LimitFilter             `json:"limit,omitempty"`
-	Type         *TypeFilter              `json:"type,omitempty"`
-	Ids          *IdFilter                `json:"ids,omitempty"`
-	Script       *ScriptFilter            `json:"script,omitempty"`
-	GeoDist      map[string]interface{}   `json:"geo_distance,omitempty"`
-	GeoDistRange map[string]interface{}   `json:"geo_distance_range,omitempty"`
+	TermsMap     map[string]interface{} `json:"terms,omitempty"`
+	TermMap      map[string]interface{} `json:"term,omitempty"`
+	RangeMap     map[string]RangeFilter `json:"range,omitempty"`
+	ExistsProp   *PropertyPathMarker    `json:"exists,omitempty"`
+	MissingProp  *PropertyPathMarker    `json:"missing,omitempty"`
+	AndFilters   []FilterOp             `json:"and,omitempty"`
+	OrFilters    []FilterOp             `json:"or,omitempty"`
+	NotFilters   []FilterOp             `json:"not,omitempty"`
+	Limit        *LimitFilter           `json:"limit,omitempty"`
+	Type         *TypeFilter            `json:"type,omitempty"`
+	Ids          *IdFilter              `json:"ids,omitempty"`
+	Script       *ScriptFilter          `json:"script,omitempty"`
+	GeoDist      map[string]interface{} `json:"geo_distance,omitempty"`
+	GeoDistRange map[string]interface{} `json:"geo_distance_range,omitempty"`
 }
 
 type PropertyPathMarker struct {
@@ -266,15 +266,13 @@ func NewGeoField(field string, latitude float32, longitude float32) GeoField {
 //	 Note: you can only have one terms clause in a filter. Use a bool filter to combine
 func (f *FilterOp) Terms(field string, executionMode TermExecutionMode, values ...interface{}) *FilterOp {
 	//You can only have one terms in a filter
-	f.TermsMap = make(map[string][]interface{})
+	f.TermsMap = make(map[string]interface{})
 
 	if executionMode != "" {
 		f.TermsMap["execution"] = executionMode
 	}
 
-	for _, val := range values {
-		f.TermsMap[field] = append(f.TermsMap[field], val)
-	}
+	f.TermsMap[field] = values
 
 	return f
 }

From 7d9074ce2849ad5df1af6184cbf46c99cdb085da Mon Sep 17 00:00:00 2001
From: Mike Yoon <myoon@airware.com>
Date: Fri, 3 Apr 2015 19:56:30 -0700
Subject: [PATCH 13/81] Update tests to match new filter API

---
 lib/searchaggregate_test.go |  2 +-
 lib/searchfilter_test.go    | 10 ++++------
 lib/searchsearch_test.go    |  6 +++---
 3 files changed, 8 insertions(+), 10 deletions(-)

diff --git a/lib/searchaggregate_test.go b/lib/searchaggregate_test.go
index 984b5d57..5fbf1f32 100644
--- a/lib/searchaggregate_test.go
+++ b/lib/searchaggregate_test.go
@@ -116,7 +116,7 @@ func TestAggregateFilter(t *testing.T) {
 	avg := Aggregate("avg_price").Avg("price")
 
 	dateAgg := Aggregate("in_stock_products").Filter(
-		Range().Field("stock").Gt(0),
+		Filter().Range("stock", nil, 0, nil, nil, ""),
 	)
 
 	dateAgg.Aggregates(
diff --git a/lib/searchfilter_test.go b/lib/searchfilter_test.go
index 384d04a2..5b265844 100644
--- a/lib/searchfilter_test.go
+++ b/lib/searchfilter_test.go
@@ -53,9 +53,8 @@ func TestFilters(t *testing.T) {
 		Should this be an AND by default?
 	*/
 	qry = Search("github").Filter(
-		Filter().Terms("actor_attributes.location", TEM_DEFAULT, "portland"),
-		Filter().Terms("repository.has_wiki", TEM_DEFAULT, true),
-	)
+		Filter().And(Filter().Terms("actor_attributes.location", TEM_DEFAULT, "portland")).
+		And(Filter().Terms("repository.has_wiki", TEM_DEFAULT, true)))
 	out, err = qry.Result(c)
 	expectedDocs = 10
 	expectedHits = 44
@@ -77,9 +76,8 @@ func TestFilters(t *testing.T) {
 	assert.T(t, out.Hits.Total == expectedHits, fmt.Sprintf("Should have %v total got %v", expectedHits, out.Hits.Total))
 
 	qry = Search("github").Filter(
-		"or",
-		Filter().Terms("actor_attributes.location", TEM_DEFAULT, "portland"),
-		Filter().Terms("repository.has_wiki", TEM_DEFAULT, true),
+		Filter().Or(Filter().Terms("actor_attributes.location", TEM_DEFAULT, "portland")).
+		Or(Filter().Terms("repository.has_wiki", TEM_DEFAULT, true)),
 	)
 	out, err = qry.Result(c)
 	expectedHits = 6676
diff --git a/lib/searchsearch_test.go b/lib/searchsearch_test.go
index 4c44fd9d..9613119d 100644
--- a/lib/searchsearch_test.go
+++ b/lib/searchsearch_test.go
@@ -194,7 +194,7 @@ func TestSearchFacetRange(t *testing.T) {
 		Facet().Fields("actor").Size("500"),
 	).Query(
 		Query().Range(
-			Range().Field("created_at").From("2012-12-10T15:00:00-08:00").To("2012-12-10T15:10:00-08:00"),
+			Filter().Range("created_at", "2012-12-10T15:00:00-08:00", nil, "2012-12-10T15:10:00-08:00", nil, ""),
 		).Search("add"),
 	)
 	out, err = qry.Result(c)
@@ -276,7 +276,7 @@ func TestSearchFilterQuery(t *testing.T) {
 	out, _ := Search("github").Size("25").Query(
 		Query().Fields("repository.name", "jas*", "", ""),
 	).Filter(
-		Filter().Terms("repository.has_wiki", true),
+		Filter().Terms("repository.has_wiki", TEM_DEFAULT, true),
 	).Result(c)
 	if out == nil || &out.Hits == nil {
 		t.Fail()
@@ -295,7 +295,7 @@ func TestSearchRange(t *testing.T) {
 	// now lets filter by a subset of the total time
 	out, _ := Search("github").Size("25").Query(
 		Query().Range(
-			Range().Field("created_at").From("2012-12-10T15:00:00-08:00").To("2012-12-10T15:10:00-08:00"),
+			Filter().Range("created_at", "2012-12-10T15:00:00-08:00", nil, "2012-12-10T15:10:00-08:00", nil, ""),
 		).Search("add"),
 	).Result(c)
 	assert.T(t, out != nil && &out.Hits != nil, "Must not have nil results, or hits")

From 99c2cb87f867e5112986a913dede76c8a0d9fc9a Mon Sep 17 00:00:00 2001
From: Mike Yoon <myoon@airware.com>
Date: Fri, 3 Apr 2015 19:56:54 -0700
Subject: [PATCH 14/81] Update vagrant image and use apt-get to install es

---
 Vagrantfile | 35 ++++++++++++++++++++++-------------
 1 file changed, 22 insertions(+), 13 deletions(-)

diff --git a/Vagrantfile b/Vagrantfile
index 860e02e0..ad8fc4cb 100644
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -2,10 +2,19 @@
 # vi: set ft=ruby :
 
 Vagrant.configure("2") do |config|
-  config.vm.box = "lucid64"
-  config.vm.box_url = "http://files.vagrantup.com/lucid64.box"
+  config.vm.box = "ubuntu/trusty64"
+  config.vm.network :forwarded_port, guest: 9200, host: 9200, auto_correct: true
   config.vm.network :forwarded_port, guest: 9300, host: 9300, auto_correct: true
-  config.vm.provision :shell, :inline => "gem install chef --version 10.26.0 --no-rdoc --no-ri --conservative"
+  #config.vm.provision :shell, :inline => "curl -L https://www.chef.io/chef/install.sh | sudo bash"
+
+  config.vm.provision "shell", inline: <<-SHELL
+    wget -qO - https://packages.elasticsearch.org/GPG-KEY-elasticsearch | sudo apt-key add -
+    sudo apt-add-repository 'deb http://packages.elasticsearch.org/elasticsearch/1.4/debian stable main'
+    sudo apt-get update
+    sudo apt-get install -y openjdk-7-jre
+    sudo apt-get install -y elasticsearch
+    sudo /etc/init.d/elasticsearch start
+  SHELL
 
   config.vm.provider :virtualbox do |vb|
     vb.gui = false
@@ -16,14 +25,14 @@ Vagrant.configure("2") do |config|
     # be enabled by default depending on what version of VirtualBox is used.
     vb.customize ["setextradata", :id, "VBoxInternal2/SharedFoldersEnableSymlinksCreate/v-root", "1"]
   end
-  config.vm.provision :chef_solo do |chef|
-    chef.cookbooks_path = "cookbooks"
-    chef.add_recipe("apt")
-    chef.add_recipe("java")
-    chef.add_recipe("elasticsearch")
-    chef.add_recipe("git")
-    chef.add_recipe("mercurial")
-    chef.add_recipe("build-essential")
-    chef.add_recipe("golang")
-  end
+  #config.vm.provision :chef_solo do |chef|
+  #  chef.cookbooks_path = "cookbooks"
+  #  chef.add_recipe("apt")
+  #  chef.add_recipe("java")
+  #  chef.add_recipe("elasticsearch")
+  #  chef.add_recipe("git")
+  #  chef.add_recipe("mercurial")
+  #  chef.add_recipe("build-essential")
+  #  chef.add_recipe("golang")
+  #end
 end
\ No newline at end of file

From 628e61a666441bb4eee1ac01e2bee2f48b49c080 Mon Sep 17 00:00:00 2001
From: Mike Yoon <myoon@airware.com>
Date: Fri, 3 Apr 2015 20:06:45 -0700
Subject: [PATCH 15/81] Fix a terms query test

---
 lib/searchfilter_test.go | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/lib/searchfilter_test.go b/lib/searchfilter_test.go
index 5b265844..b4e99371 100644
--- a/lib/searchfilter_test.go
+++ b/lib/searchfilter_test.go
@@ -64,11 +64,11 @@ func TestFilters(t *testing.T) {
 
 	// NOW, lets try with two query calls instead of one
 	qry = Search("github").Filter(
-		Filter().Terms("actor_attributes.location", TEM_DEFAULT, "portland"),
-	)
-	qry.Filter(
-		Filter().Terms("repository.has_wiki", TEM_DEFAULT, true),
+		Filter().
+			And(Filter().Terms("actor_attributes.location", TEM_DEFAULT, "portland")).
+			And(Filter().Terms("repository.has_wiki", TEM_DEFAULT, true)),
 	)
+
 	out, err = qry.Result(c)
 	//gou.Debug(out)
 	assert.T(t, err == nil, t, "should not have error")

From 5e43987e65c08e50b3460da63b0930c468a01d74 Mon Sep 17 00:00:00 2001
From: Mike Yoon <mike@yaki.org>
Date: Sat, 4 Apr 2015 02:32:45 -0700
Subject: [PATCH 16/81] Start adding filter dsl tests

---
 lib/searchfilter_test.go | 50 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 50 insertions(+)

diff --git a/lib/searchfilter_test.go b/lib/searchfilter_test.go
index b4e99371..20a7db67 100644
--- a/lib/searchfilter_test.go
+++ b/lib/searchfilter_test.go
@@ -16,8 +16,58 @@ import (
 	//"github.com/araddon/gou"
 	"github.com/bmizerany/assert"
 	"testing"
+	"encoding/json"
 )
 
+func GetJson(input interface{}) map[string]interface{} {
+	var result map[string]interface{}
+	bytes, _ := json.Marshal(input)
+
+	json.Unmarshal(bytes, &result)
+	return result
+}
+
+func TestTermsDsl(t *testing.T) {
+	filter := Filter().Terms("Sample", TEM_AND, "asdf", 123, true)
+	actual := GetJson(filter)
+
+	actualTerms := actual["terms"].(map[string]interface{})
+	actualValues := actualTerms["Sample"].([]interface{})
+
+	assert.Equal(t, 1, len(actual), "JSON should only have one key")
+	assert.Equal(t, 3, len(actualValues), "Should have 3 term values")
+	assert.Equal(t, actualValues[0], "asdf")
+	assert.Equal(t, actualValues[1], float64(123))
+	assert.Equal(t, actualValues[2], true)
+	assert.Equal(t, "and", actualTerms["execution"])
+}
+
+func TestTermDsl(t *testing.T) {
+	filter := Filter().Term("Sample", "asdf").Term("field2", 341.4)
+	actual := GetJson(filter)
+
+	actualTerm := actual["term"].(map[string]interface{})
+
+	assert.Equal(t, 1, len(actual), "JSON should only have one key")
+	assert.Equal(t, "asdf", actualTerm["Sample"])
+	assert.Equal(t, float64(341.4), actualTerm["field2"])
+}
+
+func TestRangeDsl(t *testing.T) {
+	filter := Filter().Range("rangefield", 1, 2, 3, 4, "+08:00")
+	actual := GetJson(filter)
+	//A bit lazy, probably should assert keys exist
+	actualRange := actual["range"].(map[string]interface{})["rangefield"].(map[string]interface{})
+
+
+	assert.Equal(t, 1, len(actual), "JSON should only have one key")
+	assert.Equal(t, float64(1), actualRange["gte"])
+	assert.Equal(t, float64(2), actualRange["gt"])
+	assert.Equal(t, float64(3), actualRange["lte"])
+	assert.Equal(t, float64(4), actualRange["lt"])
+	assert.Equal(t, "+08:00", actualRange["time_zone"])
+}
+
 func TestFilters(t *testing.T) {
 	c := NewTestConn()
 

From eb0ff28844562cf0a63769a25f49d248aa39fff9 Mon Sep 17 00:00:00 2001
From: Mike Yoon <mike@yaki.org>
Date: Sat, 4 Apr 2015 02:44:10 -0700
Subject: [PATCH 17/81] Merge properly

---
 lib/searchfilter.go | 23 +----------------------
 1 file changed, 1 insertion(+), 22 deletions(-)

diff --git a/lib/searchfilter.go b/lib/searchfilter.go
index cb58c109..e8a28c0e 100644
--- a/lib/searchfilter.go
+++ b/lib/searchfilter.go
@@ -299,7 +299,6 @@ func (f *FilterOp) Exists(field string) *FilterOp {
 	f.ExistsProp = &PropertyPathMarker{Field: field}
 	return f
 }
-<<<<<<< HEAD
 
 func (f *FilterOp) Missing(field string) *FilterOp {
 	f.MissingProp = &PropertyPathMarker{Field: field}
@@ -308,24 +307,4 @@ func (f *FilterOp) Missing(field string) *FilterOp {
 
 func (f *FilterOp) SetLimit(maxResults int) *FilterOp {
 	f.Limit = &LimitFilter{Value: maxResults}
-=======
-func (f *FilterOp) Missing(name string) *FilterOp {
-	f.MissingVal = map[string]string{"field": name}
-	return f
-}
-
-// Add another Filterop, "combines" two filter ops into one
-func (f *FilterOp) Add(fop *FilterOp) *FilterOp {
-	// TODO, this is invalid, refactor
-	if len(fop.Exist) > 0 {
-		f.Exist = fop.Exist
-	}
-	if len(fop.MissingVal) > 0 {
-		f.MissingVal = fop.MissingVal
-	}
-	if len(fop.Range) > 0 {
-		f.Range = fop.Range
-	}
->>>>>>> mattbaird/master
-	return f
-}
+}
\ No newline at end of file

From de5a0aa32f88386a413c8a2aa699cfebb3f4dfcf Mon Sep 17 00:00:00 2001
From: Mike Yoon <myoon@airware.com>
Date: Wed, 8 Apr 2015 13:50:43 -0700
Subject: [PATCH 18/81] Finish new filter dsl tests

Make new filter op naming more consistent
Add Type, Ids, and Not filters functions
---
 lib/searchfilter.go      |  54 +++++++--------
 lib/searchfilter_test.go | 145 +++++++++++++++++++++++++++++++++++++--
 2 files changed, 168 insertions(+), 31 deletions(-)

diff --git a/lib/searchfilter.go b/lib/searchfilter.go
index e8a28c0e..1c0411dd 100644
--- a/lib/searchfilter.go
+++ b/lib/searchfilter.go
@@ -143,20 +143,6 @@ func CompoundFilter(fl ...interface{}) *FilterWrap {
 }
 
 type FilterOp struct {
-	TermsMap     map[string]interface{} `json:"terms,omitempty"`
-	TermMap      map[string]interface{} `json:"term,omitempty"`
-	RangeMap     map[string]RangeFilter `json:"range,omitempty"`
-	ExistsProp   *PropertyPathMarker    `json:"exists,omitempty"`
-	MissingProp  *PropertyPathMarker    `json:"missing,omitempty"`
-	AndFilters   []FilterOp             `json:"and,omitempty"`
-	OrFilters    []FilterOp             `json:"or,omitempty"`
-	NotFilters   []FilterOp             `json:"not,omitempty"`
-	Limit        *LimitFilter           `json:"limit,omitempty"`
-	Type         *TypeFilter            `json:"type,omitempty"`
-	Ids          *IdFilter              `json:"ids,omitempty"`
-	Script       *ScriptFilter          `json:"script,omitempty"`
-	GeoDist      map[string]interface{} `json:"geo_distance,omitempty"`
-	GeoDistRange map[string]interface{} `json:"geo_distance_range,omitempty"`
 }
 
 type PropertyPathMarker struct {
@@ -171,9 +157,8 @@ type TypeFilter struct {
 	Value string `json:"value"`
 }
 
-type IdFilter struct {
-	Type   string   `json:"type,omitempty"`
-	Values []string `json:"values,omitempty"`
+type IdsFilter struct {
+	Values []interface{} `json:"values,omitempty"`
 }
 
 type ScriptFilter struct {
@@ -231,23 +216,26 @@ func (f *FilterOp) Or(filter *FilterOp) *FilterOp {
 	return f
 }
 
+func (f *FilterOp) Not(filter *FilterOp) *FilterOp {
+	if len(f.NotFilters) == 0 {
+		f.NotFilters = []FilterOp{*filter}
+	} else {
+		f.NotFilters = append(f.NotFilters, *filter)
+	}
+
+	return f
+}
+
 func (f *FilterOp) GeoDistance(distance string, fields ...GeoField) *FilterOp {
-	f.GeoDist = make(map[string]interface{})
-	f.GeoDist["distance"] = distance
 	for _, val := range fields {
-		f.GeoDist[val.Field] = val.GeoLocation
 	}
 
 	return f
 }
 
 func (f *FilterOp) GeoDistanceRange(from string, to string, fields ...GeoField) *FilterOp {
-	f.GeoDist = make(map[string]interface{})
-	f.GeoDist["from"] = from
-	f.GeoDist["to"] = to
 
 	for _, val := range fields {
-		f.GeoDist[val.Field] = val.GeoLocation
 	}
 
 	return f
@@ -295,6 +283,18 @@ func (f *FilterOp) Range(field string, gte interface{},
 	return f
 }
 
+func (f *FilterOp) Type(fieldType string) *FilterOp {
+	return f
+}
+
+func (f *FilterOp) Ids(ids ...interface{}) *FilterOp {
+	return f
+}
+
+func (f *FilterOp) IdsByTypes(types []string, ids ...interface{}) *FilterOp {
+	return f
+}
+
 func (f *FilterOp) Exists(field string) *FilterOp {
 	f.ExistsProp = &PropertyPathMarker{Field: field}
 	return f
@@ -305,6 +305,6 @@ func (f *FilterOp) Missing(field string) *FilterOp {
 	return f
 }
 
-func (f *FilterOp) SetLimit(maxResults int) *FilterOp {
-	f.Limit = &LimitFilter{Value: maxResults}
-}
\ No newline at end of file
+func (f *FilterOp) Limit(maxResults int) *FilterOp {
+	return f
+}
diff --git a/lib/searchfilter_test.go b/lib/searchfilter_test.go
index 20a7db67..34781616 100644
--- a/lib/searchfilter_test.go
+++ b/lib/searchfilter_test.go
@@ -14,9 +14,9 @@ package elastigo
 import (
 	"fmt"
 	//"github.com/araddon/gou"
+	"encoding/json"
 	"github.com/bmizerany/assert"
 	"testing"
-	"encoding/json"
 )
 
 func GetJson(input interface{}) map[string]interface{} {
@@ -27,6 +27,53 @@ func GetJson(input interface{}) map[string]interface{} {
 	return result
 }
 
+func HasKey(input map[string]interface{}, key string) bool {
+	if _, ok := input[key]; ok {
+		return true
+	}
+
+	return false
+}
+
+func TestAndDsl(t *testing.T) {
+	filter := Filter().And(Filter().Term("test", "asdf")).
+		And(Filter().Range("rangefield", 1, 2, 3, 4, "+08:00"))
+	actual := GetJson(filter)
+
+	actualFilters := actual["and"].([]interface{})
+
+	assert.Equal(t, 1, len(actual), "JSON should only have one key")
+	assert.Equal(t, 2, len(actualFilters), "Should have 2 filters")
+	assert.Equal(t, true, HasKey(actualFilters[0].(map[string]interface{}), "term"), "first filter is term")
+	assert.Equal(t, true, HasKey(actualFilters[1].(map[string]interface{}), "range"), "second filter is range")
+}
+
+func TestOrDsl(t *testing.T) {
+	filter := Filter().Or(Filter().Term("test", "asdf")).
+		Or(Filter().Range("rangefield", 1, 2, 3, 4, "+08:00"))
+	actual := GetJson(filter)
+
+	actualFilters := actual["or"].([]interface{})
+
+	assert.Equal(t, 1, len(actual), "JSON should only have one key")
+	assert.Equal(t, 2, len(actualFilters), "Should have 2 filters")
+	assert.Equal(t, true, HasKey(actualFilters[0].(map[string]interface{}), "term"), "first filter is term")
+	assert.Equal(t, true, HasKey(actualFilters[1].(map[string]interface{}), "range"), "second filter is range")
+}
+
+func TestNotDsl(t *testing.T) {
+	filter := Filter().Not(Filter().Term("test", "asdf")).
+		Not(Filter().Range("rangefield", 1, 2, 3, 4, "+08:00"))
+	actual := GetJson(filter)
+
+	actualFilters := actual["not"].([]interface{})
+
+	assert.Equal(t, 1, len(actual), "JSON should only have one key")
+	assert.Equal(t, 2, len(actualFilters), "Should have 2 filters")
+	assert.Equal(t, true, HasKey(actualFilters[0].(map[string]interface{}), "term"), "first filter is term")
+	assert.Equal(t, true, HasKey(actualFilters[1].(map[string]interface{}), "range"), "second filter is range")
+}
+
 func TestTermsDsl(t *testing.T) {
 	filter := Filter().Terms("Sample", TEM_AND, "asdf", 123, true)
 	actual := GetJson(filter)
@@ -59,7 +106,6 @@ func TestRangeDsl(t *testing.T) {
 	//A bit lazy, probably should assert keys exist
 	actualRange := actual["range"].(map[string]interface{})["rangefield"].(map[string]interface{})
 
-
 	assert.Equal(t, 1, len(actual), "JSON should only have one key")
 	assert.Equal(t, float64(1), actualRange["gte"])
 	assert.Equal(t, float64(2), actualRange["gt"])
@@ -68,6 +114,97 @@ func TestRangeDsl(t *testing.T) {
 	assert.Equal(t, "+08:00", actualRange["time_zone"])
 }
 
+func TestExistsDsl(t *testing.T) {
+	filter := Filter().Exists("field1")
+	actual := GetJson(filter)
+
+	actualValue := actual["exists"].(map[string]interface{})
+
+	assert.Equal(t, 1, len(actual), "JSON should only have one key")
+	assert.Equal(t, "field1", actualValue["field"], "exist field should match")
+}
+
+func TestMissingDsl(t *testing.T) {
+	filter := Filter().Missing("field1")
+	actual := GetJson(filter)
+
+	actualValue := actual["missing"].(map[string]interface{})
+
+	assert.Equal(t, 1, len(actual), "JSON should only have one key")
+	assert.Equal(t, "field1", actualValue["field"], "missing field should match")
+}
+
+func TestLimitDsl(t *testing.T) {
+	filter := Filter().Limit(100)
+	actual := GetJson(filter)
+
+	actualValue := actual["limit"].(map[string]interface{})
+	assert.Equal(t, 1, len(actual), "JSON should only have one key")
+	assert.Equal(t, float64(100), actualValue["value"], "limit value should match")
+}
+
+func TestTypeDsl(t *testing.T) {
+	filter := Filter().Type("my_type")
+	actual := GetJson(filter)
+
+	actualValue := actual["type"].(map[string]interface{})
+	assert.Equal(t, 1, len(actual), "JSON should only have one key")
+	assert.Equal(t, "my_type", actualValue["value"], "type value should match")
+}
+
+func TestIdsDsl(t *testing.T) {
+	filter := Filter().Ids("test", "asdf", "fdsa")
+	actual := GetJson(filter)
+
+	actualValue := actual["ids"].(map[string]interface{})
+	actualValues := actualValue["values"].([]interface{})
+	assert.Equal(t, 1, len(actual), "JSON should only have one key")
+	assert.Equal(t, nil, actualValue["type"], "Should have no type specified")
+	assert.Equal(t, 3, len(actualValues), "Should have 3 values specified")
+	assert.Equal(t, "test", actualValues[0], "Should have same value")
+	assert.Equal(t, "asdf", actualValues[1], "Should have same value")
+	assert.Equal(t, "fdsa", actualValues[2], "Should have same value")
+}
+
+func TestIdsTypeDsl(t *testing.T) {
+	filter := Filter().IdsByTypes([]string{"my_type"}, "test", "asdf", "fdsa")
+	actual := GetJson(filter)
+
+	actualValue := actual["ids"].(map[string]interface{})
+	actualTypes := actualValue["type"].([]interface{})
+	actualValues := actualValue["values"].([]interface{})
+	assert.Equal(t, 1, len(actual), "JSON should only have one key")
+	assert.Equal(t, 1, len(actualTypes), "Should have one type specified")
+	assert.Equal(t, "my_type", actualTypes[0], "Should have correct type specified")
+	assert.Equal(t, 3, len(actualValues), "Should have 3 values specified")
+	assert.Equal(t, "test", actualValues[0], "Should have same value")
+	assert.Equal(t, "asdf", actualValues[1], "Should have same value")
+	assert.Equal(t, "fdsa", actualValues[2], "Should have same value")
+}
+
+func TestGeoDistDsl(t *testing.T) {
+	filter := Filter().GeoDistance("100km", NewGeoField("pin.location", 32.3, 23.4))
+	actual := GetJson(filter)
+
+	actualValue := actual["geo_distance"].(map[string]interface{})
+	actualLocation := actualValue["pin.location"].(map[string]interface{})
+	assert.Equal(t, "100km", actualValue["distance"], "Distance should be equal")
+	assert.Equal(t, float64(32.3), actualLocation["lat"], "Latitude should be equal")
+	assert.Equal(t, float64(23.4), actualLocation["lon"], "Longitude should be equal")
+}
+
+func TestGeoDistRangeDsl(t *testing.T) {
+	filter := Filter().GeoDistanceRange("100km", "200km", NewGeoField("pin.location", 32.3, 23.4))
+	actual := GetJson(filter)
+
+	actualValue := actual["geo_distance_range"].(map[string]interface{})
+	actualLocation := actualValue["pin.location"].(map[string]interface{})
+	assert.Equal(t, "100km", actualValue["from"], "From should be equal")
+	assert.Equal(t, "200km", actualValue["to"], "To should be equal")
+	assert.Equal(t, float64(32.3), actualLocation["lat"], "Latitude should be equal")
+	assert.Equal(t, float64(23.4), actualLocation["lon"], "Longitude should be equal")
+}
+
 func TestFilters(t *testing.T) {
 	c := NewTestConn()
 
@@ -104,7 +241,7 @@ func TestFilters(t *testing.T) {
 	*/
 	qry = Search("github").Filter(
 		Filter().And(Filter().Terms("actor_attributes.location", TEM_DEFAULT, "portland")).
-		And(Filter().Terms("repository.has_wiki", TEM_DEFAULT, true)))
+			And(Filter().Terms("repository.has_wiki", TEM_DEFAULT, true)))
 	out, err = qry.Result(c)
 	expectedDocs = 10
 	expectedHits = 44
@@ -127,7 +264,7 @@ func TestFilters(t *testing.T) {
 
 	qry = Search("github").Filter(
 		Filter().Or(Filter().Terms("actor_attributes.location", TEM_DEFAULT, "portland")).
-		Or(Filter().Terms("repository.has_wiki", TEM_DEFAULT, true)),
+			Or(Filter().Terms("repository.has_wiki", TEM_DEFAULT, true)),
 	)
 	out, err = qry.Result(c)
 	expectedHits = 6676

From 6649d461dfe3c1be24808276158aa3ced32495c5 Mon Sep 17 00:00:00 2001
From: Mike Yoon <myoon@airware.com>
Date: Wed, 8 Apr 2015 13:51:26 -0700
Subject: [PATCH 19/81] Make filter name more consistent

---
 lib/searchfilter.go | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/lib/searchfilter.go b/lib/searchfilter.go
index 1c0411dd..79d630b8 100644
--- a/lib/searchfilter.go
+++ b/lib/searchfilter.go
@@ -143,6 +143,20 @@ func CompoundFilter(fl ...interface{}) *FilterWrap {
 }
 
 type FilterOp struct {
+	TermsMap        map[string]interface{} `json:"terms,omitempty"`
+	TermMap         map[string]interface{} `json:"term,omitempty"`
+	RangeMap        map[string]RangeFilter `json:"range,omitempty"`
+	ExistsProp      *PropertyPathMarker    `json:"exists,omitempty"`
+	MissingProp     *PropertyPathMarker    `json:"missing,omitempty"`
+	AndFilters      []FilterOp             `json:"and,omitempty"`
+	OrFilters       []FilterOp             `json:"or,omitempty"`
+	NotFilters      []FilterOp             `json:"not,omitempty"`
+	LimitProp       *LimitFilter           `json:"limit,omitempty"`
+	TypeProp        *TypeFilter            `json:"type,omitempty"`
+	IdsProp         *IdsFilter             `json:"ids,omitempty"`
+	ScriptProp      *ScriptFilter          `json:"script,omitempty"`
+	GeoDistMap      map[string]interface{} `json:"geo_distance,omitempty"`
+	GeoDistRangeMap map[string]interface{} `json:"geo_distance_range,omitempty"`
 }
 
 type PropertyPathMarker struct {
@@ -158,6 +172,7 @@ type TypeFilter struct {
 }
 
 type IdsFilter struct {
+	Type   []string      `json:"type,omitempty"`
 	Values []interface{} `json:"values,omitempty"`
 }
 
@@ -227,15 +242,22 @@ func (f *FilterOp) Not(filter *FilterOp) *FilterOp {
 }
 
 func (f *FilterOp) GeoDistance(distance string, fields ...GeoField) *FilterOp {
+	f.GeoDistMap = make(map[string]interface{})
+	f.GeoDistMap["distance"] = distance
 	for _, val := range fields {
+		f.GeoDistMap[val.Field] = val.GeoLocation
 	}
 
 	return f
 }
 
 func (f *FilterOp) GeoDistanceRange(from string, to string, fields ...GeoField) *FilterOp {
+	f.GeoDistRangeMap = make(map[string]interface{})
+	f.GeoDistRangeMap["from"] = from
+	f.GeoDistRangeMap["to"] = to
 
 	for _, val := range fields {
+		f.GeoDistRangeMap[val.Field] = val.GeoLocation
 	}
 
 	return f
@@ -284,14 +306,17 @@ func (f *FilterOp) Range(field string, gte interface{},
 }
 
 func (f *FilterOp) Type(fieldType string) *FilterOp {
+	f.TypeProp = &TypeFilter{Value: fieldType}
 	return f
 }
 
 func (f *FilterOp) Ids(ids ...interface{}) *FilterOp {
+	f.IdsProp = &IdsFilter{Values: ids}
 	return f
 }
 
 func (f *FilterOp) IdsByTypes(types []string, ids ...interface{}) *FilterOp {
+	f.IdsProp = &IdsFilter{Type: types, Values: ids}
 	return f
 }
 
@@ -306,5 +331,6 @@ func (f *FilterOp) Missing(field string) *FilterOp {
 }
 
 func (f *FilterOp) Limit(maxResults int) *FilterOp {
+	f.LimitProp = &LimitFilter{Value: maxResults}
 	return f
 }

From 98aa2032e7877c1b486e1f7aab806de33a8bffff Mon Sep 17 00:00:00 2001
From: Nicolas Donna <donnanicolas@gmail.com>
Date: Thu, 30 Apr 2015 19:07:12 -0300
Subject: [PATCH 20/81] Added parent parameter to bulk indexer

---
 lib/corebulk.go      | 20 +++++++++++++-------
 lib/corebulk_test.go | 18 +++++++++---------
 lib/coretest_test.go |  2 +-
 3 files changed, 23 insertions(+), 17 deletions(-)

diff --git a/lib/corebulk.go b/lib/corebulk.go
index dbf8e70d..039850c5 100644
--- a/lib/corebulk.go
+++ b/lib/corebulk.go
@@ -307,9 +307,9 @@ func (b *BulkIndexer) shutdown() {
 // The index bulk API adds or updates a typed JSON document to a specific index, making it searchable.
 // it operates by buffering requests, and ocassionally flushing to elasticsearch
 // http://www.elasticsearch.org/guide/reference/api/bulk.html
-func (b *BulkIndexer) Index(index string, _type string, id, ttl string, date *time.Time, data interface{}, refresh bool) error {
+func (b *BulkIndexer) Index(index string, _type string, id, parent, ttl string, date *time.Time, data interface{}, refresh bool) error {
 	//{ "index" : { "_index" : "test", "_type" : "type1", "_id" : "1" } }
-	by, err := WriteBulkBytes("index", index, _type, id, ttl, date, data, refresh)
+	by, err := WriteBulkBytes("index", index, _type, id, parent, ttl, date, data, refresh)
 	if err != nil {
 		return err
 	}
@@ -317,9 +317,9 @@ func (b *BulkIndexer) Index(index string, _type string, id, ttl string, date *ti
 	return nil
 }
 
-func (b *BulkIndexer) Update(index string, _type string, id, ttl string, date *time.Time, data interface{}, refresh bool) error {
+func (b *BulkIndexer) Update(index string, _type string, id, parent, ttl string, date *time.Time, data interface{}, refresh bool) error {
 	//{ "index" : { "_index" : "test", "_type" : "type1", "_id" : "1" } }
-	by, err := WriteBulkBytes("update", index, _type, id, ttl, date, data, refresh)
+	by, err := WriteBulkBytes("update", index, _type, id, parent, ttl, date, data, refresh)
 	if err != nil {
 		return err
 	}
@@ -333,7 +333,7 @@ func (b *BulkIndexer) Delete(index, _type, id string, refresh bool) {
 	return
 }
 
-func (b *BulkIndexer) UpdateWithPartialDoc(index string, _type string, id, ttl string, date *time.Time, partialDoc interface{}, upsert bool, refresh bool) error {
+func (b *BulkIndexer) UpdateWithPartialDoc(index string, _type string, id, parent, ttl string, date *time.Time, partialDoc interface{}, upsert bool, refresh bool) error {
 
 	var data map[string]interface{} = make(map[string]interface{})
 
@@ -341,7 +341,7 @@ func (b *BulkIndexer) UpdateWithPartialDoc(index string, _type string, id, ttl s
 	if upsert {
 		data["doc_as_upsert"] = true
 	}
-	return b.Update(index, _type, id, ttl, date, data, refresh)
+	return b.Update(index, _type, id, parent, ttl, date, data, refresh)
 }
 
 // This does the actual send of a buffer, which has already been formatted
@@ -374,7 +374,7 @@ func (b *BulkIndexer) Send(buf *bytes.Buffer) error {
 
 // Given a set of arguments for index, type, id, data create a set of bytes that is formatted for bulkd index
 // http://www.elasticsearch.org/guide/reference/api/bulk.html
-func WriteBulkBytes(op string, index string, _type string, id, ttl string, date *time.Time, data interface{}, refresh bool) ([]byte, error) {
+func WriteBulkBytes(op string, index string, _type string, id, parent, ttl string, date *time.Time, data interface{}, refresh bool) ([]byte, error) {
 	// only index and update are currently supported
 	if op != "index" && op != "update" {
 		return nil, errors.New(fmt.Sprintf("Operation '%s' is not yet supported", op))
@@ -393,6 +393,12 @@ func WriteBulkBytes(op string, index string, _type string, id, ttl string, date
 		buf.WriteString(`"`)
 	}
 
+	if len(parent) > 0 {
+		buf.WriteString(`,"_parent":"`)
+		buf.WriteString(parent)
+		buf.WriteString(`"`)
+	}
+
 	if op == "update" {
 		buf.WriteString(`,"retry_on_conflict":3`)
 	}
diff --git a/lib/corebulk_test.go b/lib/corebulk_test.go
index 3a6b615e..1acc5eab 100644
--- a/lib/corebulk_test.go
+++ b/lib/corebulk_test.go
@@ -75,7 +75,7 @@ func TestBulkIndexerBasic(t *testing.T) {
 		"date": "yesterday",
 	}
 
-	err := indexer.Index(testIndex, "user", "1", "", &date, data, true)
+	err := indexer.Index(testIndex, "user", "1", "", "", &date, data, true)
 
 	waitFor(func() bool {
 		return len(buffers) > 0
@@ -87,7 +87,7 @@ func TestBulkIndexerBasic(t *testing.T) {
 	expectedBytes := 144
 	assert.T(t, totalBytesSent == expectedBytes, fmt.Sprintf("Should have sent %v bytes but was %v", expectedBytes, totalBytesSent))
 
-	err = indexer.Index(testIndex, "user", "2", "", nil, data, true)
+	err = indexer.Index(testIndex, "user", "2", "", "", nil, data, true)
 	<-time.After(time.Millisecond * 10) // we need to wait for doc to hit send channel
 	// this will test to ensure that Flush actually catches a doc
 	indexer.Flush()
@@ -134,7 +134,7 @@ func XXXTestBulkUpdate(t *testing.T) {
 	data := map[string]interface{}{
 		"script": "ctx._source.count += 2",
 	}
-	err = indexer.Update("users", "user", "5", "", &date, data, true)
+	err = indexer.Update("users", "user", "5", "", "", &date, data, true)
 	// So here's the deal. Flushing does seem to work, you just have to give the
 	// channel a moment to recieve the message ...
 	//	<- time.After(time.Millisecond * 20)
@@ -180,9 +180,9 @@ func TestBulkSmallBatch(t *testing.T) {
 	indexer.Start()
 	<-time.After(time.Millisecond * 20)
 
-	indexer.Index("users", "user", "2", "", &date, data, true)
-	indexer.Index("users", "user", "3", "", &date, data, true)
-	indexer.Index("users", "user", "4", "", &date, data, true)
+	indexer.Index("users", "user", "2", "", "", &date, data, true)
+	indexer.Index("users", "user", "3", "", "", &date, data, true)
+	indexer.Index("users", "user", "4", "", "", &date, data, true)
 	<-time.After(time.Millisecond * 200)
 	//	indexer.Flush()
 	indexer.Stop()
@@ -231,7 +231,7 @@ func XXXTestBulkErrors(t *testing.T) {
 		for i := 0; i < 20; i++ {
 			date := time.Unix(1257894000, 0)
 			data := map[string]interface{}{"name": "smurfs", "age": 22, "date": date}
-			indexer.Index("users", "user", strconv.Itoa(i), "", &date, data, true)
+			indexer.Index("users", "user", strconv.Itoa(i), "", "", &date, data, true)
 		}
 	}()
 	var errBuf *ErrorBuffer
@@ -271,7 +271,7 @@ func BenchmarkSend(b *testing.B) {
 		about := make([]byte, 1000)
 		rand.Read(about)
 		data := map[string]interface{}{"name": "smurfs", "age": 22, "date": time.Unix(1257894000, 0), "about": about}
-		indexer.Index("users", "user", strconv.Itoa(i), "", nil, data, true)
+		indexer.Index("users", "user", strconv.Itoa(i), "", "", nil, data, true)
 	}
 	log.Printf("Sent %d messages in %d sets totaling %d bytes \n", b.N, sets, totalBytes)
 	if indexer.NumErrors() != 0 {
@@ -305,7 +305,7 @@ func BenchmarkSendBytes(b *testing.B) {
 		return indexer.Send(buf)
 	}
 	for i := 0; i < b.N; i++ {
-		indexer.Index("users", "user", strconv.Itoa(i), "", nil, body, true)
+		indexer.Index("users", "user", strconv.Itoa(i), "", "", nil, body, true)
 	}
 	log.Printf("Sent %d messages in %d sets totaling %d bytes \n", b.N, sets, totalBytes)
 	if indexer.NumErrors() != 0 {
diff --git a/lib/coretest_test.go b/lib/coretest_test.go
index 9af08bce..6d2acf72 100644
--- a/lib/coretest_test.go
+++ b/lib/coretest_test.go
@@ -179,7 +179,7 @@ func LoadTestData() {
 				log.Println("HM, already exists? ", ge.Url)
 			}
 			docsm[id] = true
-			indexer.Index(testIndex, ge.Type, id, "", &ge.Created, line, true)
+			indexer.Index(testIndex, ge.Type, id, "", "", &ge.Created, line, true)
 			docCt++
 		} else {
 			log.Println("ERROR? ", string(line))

From e36ea99d4939a43aeb8dc03e3b14543117efb957 Mon Sep 17 00:00:00 2001
From: Matt Keller <m@cognusion.com>
Date: Wed, 20 May 2015 18:15:08 -0400
Subject: [PATCH 21/81] Insert a milli-sleep after Stop() has been called to
 let other goros set the buffer before flushing. Without this, a quick, small
 bulk write will never actually be sent.

---
 lib/corebulk.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/corebulk.go b/lib/corebulk.go
index 660078ed..5010f843 100644
--- a/lib/corebulk.go
+++ b/lib/corebulk.go
@@ -136,6 +136,7 @@ func (b *BulkIndexer) Start() {
 		b.startDocChannel()
 		b.startTimer()
 		ch := <-b.shutdownChan
+		time.Sleep(2 * time.Millisecond)
 		b.Flush()
 		b.shutdown()
 		ch <- struct{}{}

From 340027e56b1040624dfa5cf651b2174308c8db3a Mon Sep 17 00:00:00 2001
From: Nicolas Donna <donnanicolas@gmail.com>
Date: Thu, 30 Apr 2015 19:07:12 -0300
Subject: [PATCH 22/81] Added parent parameter to bulk indexer

---
 lib/corebulk.go      | 20 +++++++++++++-------
 lib/corebulk_test.go | 18 +++++++++---------
 lib/coretest_test.go |  2 +-
 3 files changed, 23 insertions(+), 17 deletions(-)

diff --git a/lib/corebulk.go b/lib/corebulk.go
index 660078ed..35386314 100644
--- a/lib/corebulk.go
+++ b/lib/corebulk.go
@@ -280,9 +280,9 @@ func (b *BulkIndexer) shutdown() {
 // The index bulk API adds or updates a typed JSON document to a specific index, making it searchable.
 // it operates by buffering requests, and ocassionally flushing to elasticsearch
 // http://www.elasticsearch.org/guide/reference/api/bulk.html
-func (b *BulkIndexer) Index(index string, _type string, id, ttl string, date *time.Time, data interface{}, refresh bool) error {
+func (b *BulkIndexer) Index(index string, _type string, id, parent, ttl string, date *time.Time, data interface{}, refresh bool) error {
 	//{ "index" : { "_index" : "test", "_type" : "type1", "_id" : "1" } }
-	by, err := WriteBulkBytes("index", index, _type, id, ttl, date, data, refresh)
+	by, err := WriteBulkBytes("index", index, _type, id, parent, ttl, date, data, refresh)
 	if err != nil {
 		return err
 	}
@@ -290,9 +290,9 @@ func (b *BulkIndexer) Index(index string, _type string, id, ttl string, date *ti
 	return nil
 }
 
-func (b *BulkIndexer) Update(index string, _type string, id, ttl string, date *time.Time, data interface{}, refresh bool) error {
+func (b *BulkIndexer) Update(index string, _type string, id, parent, ttl string, date *time.Time, data interface{}, refresh bool) error {
 	//{ "index" : { "_index" : "test", "_type" : "type1", "_id" : "1" } }
-	by, err := WriteBulkBytes("update", index, _type, id, ttl, date, data, refresh)
+	by, err := WriteBulkBytes("update", index, _type, id, parent, ttl, date, data, refresh)
 	if err != nil {
 		return err
 	}
@@ -313,7 +313,7 @@ func (b *BulkIndexer) UpdateWithWithScript(index string, _type string, id, ttl s
 	return b.Update(index, _type, id, ttl, date, data, refresh)
 }
 
-func (b *BulkIndexer) UpdateWithPartialDoc(index string, _type string, id, ttl string, date *time.Time, partialDoc interface{}, upsert bool, refresh bool) error {
+func (b *BulkIndexer) UpdateWithPartialDoc(index string, _type string, id, parent, ttl string, date *time.Time, partialDoc interface{}, upsert bool, refresh bool) error {
 
 	var data map[string]interface{} = make(map[string]interface{})
 
@@ -321,7 +321,7 @@ func (b *BulkIndexer) UpdateWithPartialDoc(index string, _type string, id, ttl s
 	if upsert {
 		data["doc_as_upsert"] = true
 	}
-	return b.Update(index, _type, id, ttl, date, data, refresh)
+	return b.Update(index, _type, id, parent, ttl, date, data, refresh)
 }
 
 // This does the actual send of a buffer, which has already been formatted
@@ -354,7 +354,7 @@ func (b *BulkIndexer) Send(buf *bytes.Buffer) error {
 
 // Given a set of arguments for index, type, id, data create a set of bytes that is formatted for bulkd index
 // http://www.elasticsearch.org/guide/reference/api/bulk.html
-func WriteBulkBytes(op string, index string, _type string, id, ttl string, date *time.Time, data interface{}, refresh bool) ([]byte, error) {
+func WriteBulkBytes(op string, index string, _type string, id, parent, ttl string, date *time.Time, data interface{}, refresh bool) ([]byte, error) {
 	// only index and update are currently supported
 	if op != "index" && op != "update" {
 		return nil, errors.New(fmt.Sprintf("Operation '%s' is not yet supported", op))
@@ -373,6 +373,12 @@ func WriteBulkBytes(op string, index string, _type string, id, ttl string, date
 		buf.WriteString(`"`)
 	}
 
+	if len(parent) > 0 {
+		buf.WriteString(`,"_parent":"`)
+		buf.WriteString(parent)
+		buf.WriteString(`"`)
+	}
+
 	if op == "update" {
 		buf.WriteString(`,"retry_on_conflict":3`)
 	}
diff --git a/lib/corebulk_test.go b/lib/corebulk_test.go
index 3a6b615e..1acc5eab 100644
--- a/lib/corebulk_test.go
+++ b/lib/corebulk_test.go
@@ -75,7 +75,7 @@ func TestBulkIndexerBasic(t *testing.T) {
 		"date": "yesterday",
 	}
 
-	err := indexer.Index(testIndex, "user", "1", "", &date, data, true)
+	err := indexer.Index(testIndex, "user", "1", "", "", &date, data, true)
 
 	waitFor(func() bool {
 		return len(buffers) > 0
@@ -87,7 +87,7 @@ func TestBulkIndexerBasic(t *testing.T) {
 	expectedBytes := 144
 	assert.T(t, totalBytesSent == expectedBytes, fmt.Sprintf("Should have sent %v bytes but was %v", expectedBytes, totalBytesSent))
 
-	err = indexer.Index(testIndex, "user", "2", "", nil, data, true)
+	err = indexer.Index(testIndex, "user", "2", "", "", nil, data, true)
 	<-time.After(time.Millisecond * 10) // we need to wait for doc to hit send channel
 	// this will test to ensure that Flush actually catches a doc
 	indexer.Flush()
@@ -134,7 +134,7 @@ func XXXTestBulkUpdate(t *testing.T) {
 	data := map[string]interface{}{
 		"script": "ctx._source.count += 2",
 	}
-	err = indexer.Update("users", "user", "5", "", &date, data, true)
+	err = indexer.Update("users", "user", "5", "", "", &date, data, true)
 	// So here's the deal. Flushing does seem to work, you just have to give the
 	// channel a moment to recieve the message ...
 	//	<- time.After(time.Millisecond * 20)
@@ -180,9 +180,9 @@ func TestBulkSmallBatch(t *testing.T) {
 	indexer.Start()
 	<-time.After(time.Millisecond * 20)
 
-	indexer.Index("users", "user", "2", "", &date, data, true)
-	indexer.Index("users", "user", "3", "", &date, data, true)
-	indexer.Index("users", "user", "4", "", &date, data, true)
+	indexer.Index("users", "user", "2", "", "", &date, data, true)
+	indexer.Index("users", "user", "3", "", "", &date, data, true)
+	indexer.Index("users", "user", "4", "", "", &date, data, true)
 	<-time.After(time.Millisecond * 200)
 	//	indexer.Flush()
 	indexer.Stop()
@@ -231,7 +231,7 @@ func XXXTestBulkErrors(t *testing.T) {
 		for i := 0; i < 20; i++ {
 			date := time.Unix(1257894000, 0)
 			data := map[string]interface{}{"name": "smurfs", "age": 22, "date": date}
-			indexer.Index("users", "user", strconv.Itoa(i), "", &date, data, true)
+			indexer.Index("users", "user", strconv.Itoa(i), "", "", &date, data, true)
 		}
 	}()
 	var errBuf *ErrorBuffer
@@ -271,7 +271,7 @@ func BenchmarkSend(b *testing.B) {
 		about := make([]byte, 1000)
 		rand.Read(about)
 		data := map[string]interface{}{"name": "smurfs", "age": 22, "date": time.Unix(1257894000, 0), "about": about}
-		indexer.Index("users", "user", strconv.Itoa(i), "", nil, data, true)
+		indexer.Index("users", "user", strconv.Itoa(i), "", "", nil, data, true)
 	}
 	log.Printf("Sent %d messages in %d sets totaling %d bytes \n", b.N, sets, totalBytes)
 	if indexer.NumErrors() != 0 {
@@ -305,7 +305,7 @@ func BenchmarkSendBytes(b *testing.B) {
 		return indexer.Send(buf)
 	}
 	for i := 0; i < b.N; i++ {
-		indexer.Index("users", "user", strconv.Itoa(i), "", nil, body, true)
+		indexer.Index("users", "user", strconv.Itoa(i), "", "", nil, body, true)
 	}
 	log.Printf("Sent %d messages in %d sets totaling %d bytes \n", b.N, sets, totalBytes)
 	if indexer.NumErrors() != 0 {
diff --git a/lib/coretest_test.go b/lib/coretest_test.go
index 9af08bce..6d2acf72 100644
--- a/lib/coretest_test.go
+++ b/lib/coretest_test.go
@@ -179,7 +179,7 @@ func LoadTestData() {
 				log.Println("HM, already exists? ", ge.Url)
 			}
 			docsm[id] = true
-			indexer.Index(testIndex, ge.Type, id, "", &ge.Created, line, true)
+			indexer.Index(testIndex, ge.Type, id, "", "", &ge.Created, line, true)
 			docCt++
 		} else {
 			log.Println("ERROR? ", string(line))

From 1c8bddf1c93ad451ab7320983ea8104e9d1490f9 Mon Sep 17 00:00:00 2001
From: Mike Yoon <mike@yaki.org>
Date: Mon, 8 Jun 2015 21:52:23 -0700
Subject: [PATCH 23/81] Make boolean filters variadic

---
 lib/searchfilter.go      | 25 +++++++++++++------------
 lib/searchfilter_test.go |  3 +--
 2 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/lib/searchfilter.go b/lib/searchfilter.go
index 3070b745..b4323553 100644
--- a/lib/searchfilter.go
+++ b/lib/searchfilter.go
@@ -154,9 +154,9 @@ type FilterOp struct {
 	RangeMap        map[string]RangeFilter `json:"range,omitempty"`
 	ExistsProp      *PropertyPathMarker    `json:"exists,omitempty"`
 	MissingProp     *PropertyPathMarker    `json:"missing,omitempty"`
-	AndFilters      []FilterOp             `json:"and,omitempty"`
-	OrFilters       []FilterOp             `json:"or,omitempty"`
-	NotFilters      []FilterOp             `json:"not,omitempty"`
+	AndFilters      []*FilterOp            `json:"and,omitempty"`
+	OrFilters       []*FilterOp            `json:"or,omitempty"`
+	NotFilters      []*FilterOp            `json:"not,omitempty"`
 	LimitProp       *LimitFilter           `json:"limit,omitempty"`
 	TypeProp        *TypeFilter            `json:"type,omitempty"`
 	IdsProp         *IdsFilter             `json:"ids,omitempty"`
@@ -217,31 +217,32 @@ func (f *FilterOp) Term(field string, value interface{}) *FilterOp {
 	return f
 }
 
-func (f *FilterOp) And(filter *FilterOp) *FilterOp {
+func (f *FilterOp) And(filters ...*FilterOp) *FilterOp {
 	if len(f.AndFilters) == 0 {
-		f.AndFilters = []FilterOp{*filter}
+		f.AndFilters = filters[:]
 	} else {
-		f.AndFilters = append(f.AndFilters, *filter)
+		f.AndFilters = append(f.AndFilters, filters...)
 	}
 
 	return f
 }
 
-func (f *FilterOp) Or(filter *FilterOp) *FilterOp {
+func (f *FilterOp) Or(filters ...*FilterOp) *FilterOp {
 	if len(f.OrFilters) == 0 {
-		f.OrFilters = []FilterOp{*filter}
+		f.OrFilters = filters[:]
 	} else {
-		f.OrFilters = append(f.OrFilters, *filter)
+		f.OrFilters = append(f.OrFilters, filters...)
 	}
 
 	return f
 }
 
-func (f *FilterOp) Not(filter *FilterOp) *FilterOp {
+func (f *FilterOp) Not(filters ...*FilterOp) *FilterOp {
 	if len(f.NotFilters) == 0 {
-		f.NotFilters = []FilterOp{*filter}
+		f.NotFilters = filters[:]
+
 	} else {
-		f.NotFilters = append(f.NotFilters, *filter)
+		f.NotFilters = append(f.NotFilters, filters...)
 	}
 
 	return f
diff --git a/lib/searchfilter_test.go b/lib/searchfilter_test.go
index 34781616..dfb0c049 100644
--- a/lib/searchfilter_test.go
+++ b/lib/searchfilter_test.go
@@ -49,8 +49,7 @@ func TestAndDsl(t *testing.T) {
 }
 
 func TestOrDsl(t *testing.T) {
-	filter := Filter().Or(Filter().Term("test", "asdf")).
-		Or(Filter().Range("rangefield", 1, 2, 3, 4, "+08:00"))
+	filter := Filter().Or(Filter().Term("test", "asdf"), Filter().Range("rangefield", 1, 2, 3, 4, "+08:00"))
 	actual := GetJson(filter)
 
 	actualFilters := actual["or"].([]interface{})

From e88a14e76289b7abbea49acb6fd1ac374441ef52 Mon Sep 17 00:00:00 2001
From: Mike Yoon <mike@yaki.org>
Date: Tue, 16 Jun 2015 01:12:34 -0700
Subject: [PATCH 24/81] Add highlighting api support

---
 lib/searchhighlight.go      | 130 ++++++++++++++++++++++++++++++++++++
 lib/searchhighlight_test.go |  63 +++++++++++++++++
 lib/searchsearch.go         |   6 ++
 lib/shared_test.go          |  17 +++++
 4 files changed, 216 insertions(+)
 create mode 100644 lib/searchhighlight.go
 create mode 100644 lib/searchhighlight_test.go

diff --git a/lib/searchhighlight.go b/lib/searchhighlight.go
new file mode 100644
index 00000000..15b0ca5a
--- /dev/null
+++ b/lib/searchhighlight.go
@@ -0,0 +1,130 @@
+package elastigo
+
+import "encoding/json"
+
+func NewHighlight() *HighlightDsl {
+	return &HighlightDsl{}
+}
+
+type HighlightDsl struct {
+	Settings  *HighlightEmbed           `-`
+	TagSchema string                    `json:"tag_schema,omitempty"`
+	Fields    map[string]HighlightEmbed `json:"fields,omitempty"`
+}
+
+func NewHighlightOpts() *HighlightEmbed {
+	return &HighlightEmbed{}
+}
+
+type HighlightEmbed struct {
+	BoundaryCharsVal   string    `json:"boundary_chars,omitempty"`
+	BoundaryMaxScanVal int       `json:"boundary_max_scan,omitempty"`
+	PreTags            []string  `json:"pre_tags,omitempty"`
+	PostTags           []string  `json:"post_tags,omitempty"`
+	FragmentSizeVal    int       `json:"fragment_size,omitempty"`
+	NumOfFragmentsVal  int       `json:"number_of_fragments,omitempty"`
+	HighlightQuery     *QueryDsl `json:"highlight_query,omitempty"`
+	MatchedFieldsVal   []string  `json:"matched_fields,omitempty"`
+	OrderVal           string    `json:"order,omitempty"`
+	TypeVal            string    `json:"type,omitempty"`
+}
+
+// Custom marshalling
+func (t *HighlightDsl) MarshalJSON() ([]byte, error) {
+	m := make(map[string]interface{})
+
+	if t.Fields != nil {
+		m["fields"] = t.Fields
+	}
+
+	if t.TagSchema != "" {
+		m["tag_schema"] = t.TagSchema
+	}
+
+	//This is terrible :(
+	if t.Settings != nil {
+		embed, _ := json.Marshal(t.Settings)
+		json.Unmarshal(embed, &m)
+	}
+
+	return json.Marshal(m)
+}
+
+func (h *HighlightDsl) AddField(name string, settings *HighlightEmbed) *HighlightDsl {
+	if h.Fields == nil {
+		h.Fields = make(map[string]HighlightEmbed)
+	}
+
+	if settings != nil {
+		h.Fields[name] = *settings
+	} else {
+		h.Fields[name] = HighlightEmbed{}
+	}
+
+	return h
+}
+
+func (h *HighlightDsl) Schema(schema string) *HighlightDsl {
+	h.TagSchema = schema
+	return h
+}
+
+func (h *HighlightDsl) SetOptions(options *HighlightEmbed) *HighlightDsl {
+	h.Settings = options
+	return h
+}
+
+func (o *HighlightEmbed) BoundaryChars(chars string) *HighlightEmbed {
+	o.BoundaryCharsVal = chars
+	return o
+}
+
+func (o *HighlightEmbed) BoundaryMaxScan(max int) *HighlightEmbed {
+	o.BoundaryMaxScanVal = max
+	return o
+}
+
+func (he *HighlightEmbed) FragSize(size int) *HighlightEmbed {
+	he.FragmentSizeVal = size
+	return he
+}
+
+func (he *HighlightEmbed) NumFrags(numFrags int) *HighlightEmbed {
+	he.NumOfFragmentsVal = numFrags
+	return he
+}
+
+func (he *HighlightEmbed) MatchedFields(fields ...string) *HighlightEmbed {
+	he.MatchedFieldsVal = fields
+	return he
+}
+
+func (he *HighlightEmbed) Order(order string) *HighlightEmbed {
+	he.OrderVal = order
+	return he
+}
+
+func (he *HighlightEmbed) Tags(pre string, post string) *HighlightEmbed {
+	if he == nil {
+		he = &HighlightEmbed{}
+	}
+
+	if he.PreTags == nil {
+		he.PreTags = []string{pre}
+	} else {
+		he.PreTags = append(he.PreTags, pre)
+	}
+
+	if he.PostTags == nil {
+		he.PostTags = []string{post}
+	} else {
+		he.PostTags = append(he.PostTags, post)
+	}
+
+	return he
+}
+
+func (he *HighlightEmbed) Type(highlightType string) *HighlightEmbed {
+	he.TypeVal = highlightType
+	return he
+}
diff --git a/lib/searchhighlight_test.go b/lib/searchhighlight_test.go
new file mode 100644
index 00000000..ef2fe28d
--- /dev/null
+++ b/lib/searchhighlight_test.go
@@ -0,0 +1,63 @@
+package elastigo
+
+import (
+	"github.com/bmizerany/assert"
+	"testing"
+)
+
+func TestEmbedDsl(t *testing.T) {
+	highlight := NewHighlight().SetOptions(NewHighlightOpts().
+		Tags("<div>", "</div>").
+		BoundaryChars("asdf").BoundaryMaxScan(100).
+		FragSize(10).NumFrags(50).
+		Order("order").Type("fdsa").
+		MatchedFields("1", "2"))
+
+	actual := GetJson(highlight)
+
+	assert.Equal(t, "<div>", actual["pre_tags"].([]interface{})[0])
+	assert.Equal(t, "</div>", actual["post_tags"].([]interface{})[0])
+	assert.Equal(t, "asdf", actual["boundary_chars"])
+	assert.Equal(t, float64(100), actual["boundary_max_scan"])
+	assert.Equal(t, float64(10), actual["fragment_size"])
+	assert.Equal(t, float64(50), actual["number_of_fragments"])
+	assert.Equal(t, "1", actual["matched_fields"].([]interface{})[0])
+	assert.Equal(t, "2", actual["matched_fields"].([]interface{})[1])
+	assert.Equal(t, "order", actual["order"])
+	assert.Equal(t, "fdsa", actual["type"])
+}
+
+func TestFieldDsl(t *testing.T) {
+	highlight := NewHighlight().AddField("whatever", NewHighlightOpts().
+		Tags("<div>", "</div>").
+		BoundaryChars("asdf").BoundaryMaxScan(100).
+		FragSize(10).NumFrags(50).
+		Order("order").Type("fdsa").
+		MatchedFields("1", "2"))
+
+	actual := GetJson(highlight)["fields"].(map[string]interface{})["whatever"].(map[string]interface{})
+
+	assert.Equal(t, "<div>", actual["pre_tags"].([]interface{})[0])
+	assert.Equal(t, "</div>", actual["post_tags"].([]interface{})[0])
+	assert.Equal(t, "asdf", actual["boundary_chars"])
+	assert.Equal(t, float64(100), actual["boundary_max_scan"])
+	assert.Equal(t, float64(10), actual["fragment_size"])
+	assert.Equal(t, float64(50), actual["number_of_fragments"])
+	assert.Equal(t, "1", actual["matched_fields"].([]interface{})[0])
+	assert.Equal(t, "2", actual["matched_fields"].([]interface{})[1])
+	assert.Equal(t, "order", actual["order"])
+	assert.Equal(t, "fdsa", actual["type"])
+}
+
+func TestEmbedAndFieldDsl(t *testing.T) {
+	highlight := NewHighlight().
+		SetOptions(NewHighlightOpts().Tags("<div>", "</div>")).
+		AddField("afield", NewHighlightOpts().Type("something"))
+
+	actual := GetJson(highlight)
+	actualField := actual["fields"].(map[string]interface{})["afield"].(map[string]interface{})
+
+	assert.Equal(t, "<div>", actual["pre_tags"].([]interface{})[0])
+	assert.Equal(t, "</div>", actual["post_tags"].([]interface{})[0])
+	assert.Equal(t, "something", actualField["type"])
+}
diff --git a/lib/searchsearch.go b/lib/searchsearch.go
index 820f23d9..ee1f629c 100644
--- a/lib/searchsearch.go
+++ b/lib/searchsearch.go
@@ -49,6 +49,7 @@ type SearchDsl struct {
 	SortBody      []*SortDsl               `json:"sort,omitempty"`
 	FilterVal     *FilterWrap              `json:"filter,omitempty"`
 	AggregatesVal map[string]*AggregateDsl `json:"aggregations,omitempty"`
+	HighlightVal  *HighlightDsl            `json:"highlight,omitempty"`
 }
 
 func (s *SearchDsl) Bytes(conn *Conn) ([]byte, error) {
@@ -199,3 +200,8 @@ func (s *SearchDsl) SearchType(searchType string) *SearchDsl {
 	s.args["search_type"] = searchType
 	return s
 }
+
+func (s *SearchDsl) Highlight(highlight *HighlightDsl) *SearchDsl {
+	s.HighlightVal = highlight
+	return s
+}
diff --git a/lib/shared_test.go b/lib/shared_test.go
index ea81c7c1..59d2d288 100644
--- a/lib/shared_test.go
+++ b/lib/shared_test.go
@@ -14,6 +14,7 @@ package elastigo
 import (
 	"flag"
 	"log"
+	"encoding/json"
 )
 
 var (
@@ -21,3 +22,19 @@ var (
 	eshost   *string = flag.String("host", "localhost", "Elasticsearch Server Host Address")
 	logLevel *string = flag.String("logging", "info", "Which log level: [debug,info,warn,error,fatal]")
 )
+
+func GetJson(input interface{}) map[string]interface{} {
+	var result map[string]interface{}
+	bytes, _ := json.Marshal(input)
+
+	json.Unmarshal(bytes, &result)
+	return result
+}
+
+func HasKey(input map[string]interface{}, key string) bool {
+	if _, ok := input[key]; ok {
+		return true
+	}
+
+	return false
+}
\ No newline at end of file

From b4c2ab278f1df643a7acab20bf8383d9c4199a17 Mon Sep 17 00:00:00 2001
From: Mike Yoon <mike@yaki.org>
Date: Tue, 16 Jun 2015 23:11:08 -0700
Subject: [PATCH 25/81] Add error handling

---
 lib/searchhighlight.go      | 18 +++++++++++++-----
 lib/searchhighlight_test.go | 10 +++++++---
 lib/shared_test.go          | 11 +++++++----
 3 files changed, 27 insertions(+), 12 deletions(-)

diff --git a/lib/searchhighlight.go b/lib/searchhighlight.go
index 15b0ca5a..ac74947d 100644
--- a/lib/searchhighlight.go
+++ b/lib/searchhighlight.go
@@ -41,13 +41,21 @@ func (t *HighlightDsl) MarshalJSON() ([]byte, error) {
 		m["tag_schema"] = t.TagSchema
 	}
 
-	//This is terrible :(
-	if t.Settings != nil {
-		embed, _ := json.Marshal(t.Settings)
-		json.Unmarshal(embed, &m)
+	if t.Settings == nil {
+		return json.Marshal(m)
 	}
 
-	return json.Marshal(m)
+	//This is terrible :(, could use structs package to avoid extra serialization.
+	embed, err := json.Marshal(t.Settings)
+	if err == nil {
+		err = json.Unmarshal(embed, &m)
+	}
+
+	if err == nil {
+		return json.Marshal(m)
+	}
+
+	return nil, err
 }
 
 func (h *HighlightDsl) AddField(name string, settings *HighlightEmbed) *HighlightDsl {
diff --git a/lib/searchhighlight_test.go b/lib/searchhighlight_test.go
index ef2fe28d..ca5b9304 100644
--- a/lib/searchhighlight_test.go
+++ b/lib/searchhighlight_test.go
@@ -13,8 +13,9 @@ func TestEmbedDsl(t *testing.T) {
 		Order("order").Type("fdsa").
 		MatchedFields("1", "2"))
 
-	actual := GetJson(highlight)
+	actual, err := GetJson(highlight)
 
+	assert.Equal(t, nil, err)
 	assert.Equal(t, "<div>", actual["pre_tags"].([]interface{})[0])
 	assert.Equal(t, "</div>", actual["post_tags"].([]interface{})[0])
 	assert.Equal(t, "asdf", actual["boundary_chars"])
@@ -35,8 +36,10 @@ func TestFieldDsl(t *testing.T) {
 		Order("order").Type("fdsa").
 		MatchedFields("1", "2"))
 
-	actual := GetJson(highlight)["fields"].(map[string]interface{})["whatever"].(map[string]interface{})
+	result, err := GetJson(highlight)
+	actual := result["fields"].(map[string]interface{})["whatever"].(map[string]interface{})
 
+	assert.Equal(t, nil, err)
 	assert.Equal(t, "<div>", actual["pre_tags"].([]interface{})[0])
 	assert.Equal(t, "</div>", actual["post_tags"].([]interface{})[0])
 	assert.Equal(t, "asdf", actual["boundary_chars"])
@@ -54,9 +57,10 @@ func TestEmbedAndFieldDsl(t *testing.T) {
 		SetOptions(NewHighlightOpts().Tags("<div>", "</div>")).
 		AddField("afield", NewHighlightOpts().Type("something"))
 
-	actual := GetJson(highlight)
+	actual, err := GetJson(highlight)
 	actualField := actual["fields"].(map[string]interface{})["afield"].(map[string]interface{})
 
+	assert.Equal(t, nil, err)
 	assert.Equal(t, "<div>", actual["pre_tags"].([]interface{})[0])
 	assert.Equal(t, "</div>", actual["post_tags"].([]interface{})[0])
 	assert.Equal(t, "something", actualField["type"])
diff --git a/lib/shared_test.go b/lib/shared_test.go
index 59d2d288..2a3d8de4 100644
--- a/lib/shared_test.go
+++ b/lib/shared_test.go
@@ -23,12 +23,15 @@ var (
 	logLevel *string = flag.String("logging", "info", "Which log level: [debug,info,warn,error,fatal]")
 )
 
-func GetJson(input interface{}) map[string]interface{} {
+func GetJson(input interface{}) (map[string]interface{}, error) {
 	var result map[string]interface{}
-	bytes, _ := json.Marshal(input)
+	bytes, err := json.Marshal(input)
 
-	json.Unmarshal(bytes, &result)
-	return result
+	if err == nil {
+		err = json.Unmarshal(bytes, &result)
+	}
+
+	return result, err
 }
 
 func HasKey(input map[string]interface{}, key string) bool {

From a8427f0448215e0bc592bad493cd1028e69fcb6c Mon Sep 17 00:00:00 2001
From: Brian Downs <brian.downs@gmail.com>
Date: Sun, 21 Jun 2015 12:55:59 -0700
Subject: [PATCH 26/81] added status field to CatIndexInfo struct

---
 lib/catresponses.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/catresponses.go b/lib/catresponses.go
index 0f679f3b..55c4d9e9 100644
--- a/lib/catresponses.go
+++ b/lib/catresponses.go
@@ -2,6 +2,7 @@ package elastigo
 
 type CatIndexInfo struct {
 	Health   string
+	Status   string
 	Name     string
 	Shards   int
 	Replicas int

From ef77478369bee396c2effaffa1d0f5cfadc463f4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Erlend=20R=C3=B8sj=C3=B8?= <erlend@erlend.net>
Date: Thu, 2 Jul 2015 10:53:26 +0200
Subject: [PATCH 27/81] Added SetFromUrl function to connection

---
 lib/connection.go      | 28 +++++++++++++++++++
 lib/connection_test.go | 62 ++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 90 insertions(+)
 create mode 100644 lib/connection_test.go

diff --git a/lib/connection.go b/lib/connection.go
index 72da350f..895eb4fc 100644
--- a/lib/connection.go
+++ b/lib/connection.go
@@ -12,9 +12,11 @@
 package elastigo
 
 import (
+	"errors"
 	"fmt"
 	hostpool "github.com/bitly/go-hostpool"
 	"net/http"
+	"net/url"
 	"runtime"
 	"strings"
 	"sync"
@@ -61,6 +63,32 @@ func NewConn() *Conn {
 	}
 }
 
+func (c *Conn) SetFromUrl(u string) error {
+	if u == "" {
+		return errors.New("Url is empty")
+	}
+
+	parsedUrl, err := url.Parse(u)
+	if err != nil {
+		return err
+	}
+
+	c.Protocol = parsedUrl.Scheme
+	host, portNum := splitHostnamePartsFromHost(parsedUrl.Host, c.Port)
+	c.Port = portNum
+	c.Domain = host
+
+	if parsedUrl.User != nil {
+		c.Username = parsedUrl.User.Username()
+		password, passwordIsSet := parsedUrl.User.Password()
+		if passwordIsSet {
+			c.Password = password
+		}
+	}
+
+	return nil
+}
+
 func (c *Conn) SetPort(port string) {
 	c.Port = port
 }
diff --git a/lib/connection_test.go b/lib/connection_test.go
new file mode 100644
index 00000000..5719b017
--- /dev/null
+++ b/lib/connection_test.go
@@ -0,0 +1,62 @@
+// Copyright 2013 Matthew Baird
+// Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//     http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package elastigo
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/bmizerany/assert"
+)
+
+func TestSetFromUrl(t *testing.T) {
+	c := NewConn()
+
+	err := c.SetFromUrl("http://localhost")
+	exp := "localhost"
+	assert.T(t, c.Domain == exp && err == nil, fmt.Sprintf("Expected %s, got: %s", exp, c.Domain))
+
+	c = NewConn()
+
+	err = c.SetFromUrl("http://localhost:9200")
+	exp = "9200"
+	assert.T(t, c.Port == exp && err == nil, fmt.Sprintf("Expected %s, got: %s", exp, c.Port))
+
+	c = NewConn()
+
+	err = c.SetFromUrl("http://localhost:9200")
+	exp = "localhost"
+	assert.T(t, c.Domain == exp && err == nil, fmt.Sprintf("Expected %s, got: %s", exp, c.Domain))
+
+	c = NewConn()
+
+	err = c.SetFromUrl("http://someuser@localhost:9200")
+	exp = "someuser"
+	assert.T(t, c.Username == exp && err == nil, fmt.Sprintf("Expected %s, got: %s", exp, c.Username))
+
+	c = NewConn()
+
+	err = c.SetFromUrl("http://someuser:password@localhost:9200")
+	exp = "password"
+	assert.T(t, c.Password == exp && err == nil, fmt.Sprintf("Expected %s, got: %s", exp, c.Password))
+
+	c = NewConn()
+
+	err = c.SetFromUrl("http://someuser:password@localhost:9200")
+	exp = "someuser"
+	assert.T(t, c.Username == exp && err == nil, fmt.Sprintf("Expected %s, got: %s", exp, c.Username))
+
+	c = NewConn()
+
+	err = c.SetFromUrl("")
+	exp = "Url is empty"
+	assert.T(t, err != nil && err.Error() == exp, fmt.Sprintf("Expected %s, got: %s", exp, err.Error()))
+}

From c7a562a02e7d2850049050a465d9bdbf3cb66421 Mon Sep 17 00:00:00 2001
From: Dimitri Roche <dimroc@gmail.com>
Date: Mon, 6 Jul 2015 14:49:48 -0400
Subject: [PATCH 28/81] Update Percolate to unmarshal response into a valid
 struct

Currently, the existing `Match` struct does not comply
with json returned from a percolate query.

This fixes that.

Example percolate response as of ES 1.6.0:


```json
{"took":9,"_shards":{"total":3,"successful":3,"failed":0},"total":3,
"matches":[{"_index":"test-geoevents-20150706-115234","_id":"nyc,West Village"},{"_index":"test-geoevents-20150706-115234","_id":"nyc,Manhattan"},{"_index":"test-geoevents-20150706-115234","_id":"nyc,Greenwich Village"}]}
```
---
 lib/corepercolate.go | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/lib/corepercolate.go b/lib/corepercolate.go
index cd6df1b2..e9f459f5 100644
--- a/lib/corepercolate.go
+++ b/lib/corepercolate.go
@@ -16,6 +16,19 @@ import (
 	"fmt"
 )
 
+//{"took":9,"_shards":{"total":3,"successful":3,"failed":0},"total":3,
+//"matches":[{"_index":"test-geoevents-20150706-115234","_id":"nyc,West Village"},{"_index":"test-geoevents-20150706-115234","_id":"nyc,Manhattan"},{"_index":"test-geoevents-20150706-115234","_id":"nyc,Greenwich Village"}]}
+
+type PercolatorResult struct {
+	SearchResult
+	Matches []PercolatorMatch `json:"matches"`
+}
+
+type PercolatorMatch struct {
+	Index string `json:"_index"`
+	Id    string `json:"_id"`
+}
+
 // RegisterPercolate allows the caller to register queries against an index,
 // and then send percolate requests which include a doc, and getting back the
 // queries that match on that doc out of the set of registered queries.  Think
@@ -42,9 +55,9 @@ func (c *Conn) RegisterPercolate(index string, name string, args map[string]inte
 	return retval, err
 }
 
-func (c *Conn) Percolate(index string, _type string, name string, args map[string]interface{}, doc string) (Match, error) {
+func (c *Conn) Percolate(index string, _type string, name string, args map[string]interface{}, doc string) (PercolatorResult, error) {
 	var url string
-	var retval Match
+	var retval PercolatorResult
 	url = fmt.Sprintf("/%s/%s/_percolate", index, _type)
 	body, err := c.DoCommand("GET", url, args, doc)
 	if err != nil {

From 68660f4157c717b17ab14cb13ab65d4b237e1ac8 Mon Sep 17 00:00:00 2001
From: Ben Aldrich <ben.aldrich@logrhythm.com>
Date: Tue, 7 Jul 2015 15:58:41 -0600
Subject: [PATCH 29/81] testing with elasticsearch 1.5.2 variables didn't align

---
 lib/catindexinfo.go      | 29 +++++++++++----------
 lib/catindexinfo_test.go | 56 +++++++++++++++++++++++-----------------
 lib/catresponses.go      |  1 +
 3 files changed, 50 insertions(+), 36 deletions(-)

diff --git a/lib/catindexinfo.go b/lib/catindexinfo.go
index c02a37ca..0e48cc1e 100644
--- a/lib/catindexinfo.go
+++ b/lib/catindexinfo.go
@@ -8,50 +8,53 @@ import (
 
 var ErrInvalidIndexLine = errors.New("Cannot parse indexline")
 
-// Create an IndexInfo from the string _cat/indices would produce
+//Create an IndexInfo from the string _cat/indices would produce
+//EX: health status index    pri rep docs.count docs.deleted store.size pri.store.size
+//green open logs-2015-06-19   2 0 135389346 0 53048922233 53048922233
 func NewCatIndexInfo(indexLine string) (catIndex *CatIndexInfo, err error) {
 	split := strings.Fields(indexLine)
-	if len(split) < 4 {
+	if len(split) < 5 {
 		return nil, ErrInvalidIndexLine
 	}
 	catIndex = &CatIndexInfo{}
 	catIndex.Store = CatIndexStore{}
 	catIndex.Docs = CatIndexDocs{}
 	catIndex.Health = split[0]
-	catIndex.Name = split[1]
-	catIndex.Shards, err = strconv.Atoi(split[2])
+	catIndex.Status = split[1]
+	catIndex.Name = split[2]
+	catIndex.Shards, err = strconv.Atoi(split[3])
 	if err != nil {
 		catIndex.Shards = 0
 	}
-	catIndex.Replicas, err = strconv.Atoi(split[3])
+	catIndex.Replicas, err = strconv.Atoi(split[4])
 	if err != nil {
 		catIndex.Replicas = 0
 	}
-	if len(split) == 4 {
+	if len(split) == 5 {
 		return catIndex, nil
 	}
-	catIndex.Docs.Count, err = strconv.ParseInt(split[4], 10, 64)
+	catIndex.Docs.Count, err = strconv.ParseInt(split[5], 10, 64)
 	if err != nil {
 		catIndex.Docs.Count = 0
 	}
-	if len(split) == 5 {
+	if len(split) == 6 {
 		return catIndex, nil
 	}
-	catIndex.Docs.Deleted, err = strconv.ParseInt(split[5], 10, 64)
+	catIndex.Docs.Deleted, err = strconv.ParseInt(split[6], 10, 64)
 	if err != nil {
 		catIndex.Docs.Deleted = 0
 	}
-	if len(split) == 6 {
+	if len(split) == 7 {
 		return catIndex, nil
 	}
-	catIndex.Store.Size, err = strconv.ParseInt(split[6], 10, 64)
+	catIndex.Store.Size, err = strconv.ParseInt(split[7], 10, 64)
 	if err != nil {
 		catIndex.Store.Size = 0
 	}
-	if len(split) == 7 {
+	if len(split) == 8 {
 		return catIndex, nil
 	}
-	catIndex.Store.PriSize, err = strconv.ParseInt(split[7], 10, 64)
+	catIndex.Store.PriSize, err = strconv.ParseInt(split[8], 10, 64)
 	if err != nil {
 		catIndex.Store.PriSize = 0
 	}
diff --git a/lib/catindexinfo_test.go b/lib/catindexinfo_test.go
index 1eb69396..f4b28c91 100644
--- a/lib/catindexinfo_test.go
+++ b/lib/catindexinfo_test.go
@@ -10,22 +10,26 @@ func TestCatIndexInfo(t *testing.T) {
 		_, err := NewCatIndexInfo("red ")
 		So(err, ShouldNotBeNil)
 	})
-	Convey("Create index line from a bad shards index listing", t, func() {
-		i, err := NewCatIndexInfo("red foo-2000-01-01-bar a 1 1234 3 11000 13000")
+	Convey("catIndex Create index line from a bad shards index listing", t, func() {
+		// green  open   twitter    5   1      11434            0       64mb           32mb
+		//green open logs-2015-06-19   2 1 135389346 20 53048922233 53048922233
+		i, err := NewCatIndexInfo("green open logs-2015-06-19   2 1 135389346 20 53048922233 53048922233")
 		So(err, ShouldBeNil)
-		So(i.Health, ShouldEqual, "red")
-		So(i.Name, ShouldEqual, "foo-2000-01-01-bar")
-		So(i.Shards, ShouldEqual, 0)
+		So(i.Health, ShouldEqual, "green")
+		So(i.Status, ShouldEqual, "open")
+		So(i.Name, ShouldEqual, "logs-2015-06-19")
+		So(i.Shards, ShouldEqual, 2)
 		So(i.Replicas, ShouldEqual, 1)
-		So(i.Docs.Count, ShouldEqual, 1234)
-		So(i.Docs.Deleted, ShouldEqual, 3)
-		So(i.Store.Size, ShouldEqual, 11000)
-		So(i.Store.PriSize, ShouldEqual, 13000)
+		So(i.Docs.Count, ShouldEqual, 135389346)
+		So(i.Docs.Deleted, ShouldEqual, 20)
+		So(i.Store.Size, ShouldEqual, 53048922233)
+		So(i.Store.PriSize, ShouldEqual, 53048922233)
 	})
-	Convey("Create index line from a bad replicas index listing", t, func() {
-		i, err := NewCatIndexInfo("red foo-2000-01-01-bar 2 a 1234 3 11000 13000")
+	Convey("catIndex Create index line from a bad replicas index listing", t, func() {
+		i, err := NewCatIndexInfo("red open foo-2000-01-01-bar 2 0 1234 3 11000 13000")
 		So(err, ShouldBeNil)
 		So(i.Health, ShouldEqual, "red")
+		So(i.Status, ShouldEqual, "open")
 		So(i.Name, ShouldEqual, "foo-2000-01-01-bar")
 		So(i.Shards, ShouldEqual, 2)
 		So(i.Replicas, ShouldEqual, 0)
@@ -34,9 +38,10 @@ func TestCatIndexInfo(t *testing.T) {
 		So(i.Store.Size, ShouldEqual, 11000)
 		So(i.Store.PriSize, ShouldEqual, 13000)
 	})
-	Convey("Create index line from a complete index listing", t, func() {
-		i, err := NewCatIndexInfo("red foo-2000-01-01-bar 2 1 1234 3 11000 13000")
+	Convey("catIndex Create index line from a complete index listing", t, func() {
+		i, err := NewCatIndexInfo("red closed foo-2000-01-01-bar 2 1 1234 3 11000 13000")
 		So(err, ShouldBeNil)
+		So(i.Status, ShouldEqual, "closed")
 		So(i.Health, ShouldEqual, "red")
 		So(i.Name, ShouldEqual, "foo-2000-01-01-bar")
 		So(i.Shards, ShouldEqual, 2)
@@ -46,10 +51,11 @@ func TestCatIndexInfo(t *testing.T) {
 		So(i.Store.Size, ShouldEqual, 11000)
 		So(i.Store.PriSize, ShouldEqual, 13000)
 	})
-	Convey("Create index line from a bad docs index listing", t, func() {
-		i, err := NewCatIndexInfo("red foo-2000-01-01-bar 2 1 a 3 11000 13000")
+	Convey("catIndex Create index line from a bad docs index listing", t, func() {
+		i, err := NewCatIndexInfo("red open foo-2000-01-01-bar 2 1 a 3 11000 13000")
 		So(err, ShouldBeNil)
 		So(i.Health, ShouldEqual, "red")
+		So(i.Status, ShouldEqual, "open")
 		So(i.Name, ShouldEqual, "foo-2000-01-01-bar")
 		So(i.Shards, ShouldEqual, 2)
 		So(i.Replicas, ShouldEqual, 1)
@@ -58,10 +64,11 @@ func TestCatIndexInfo(t *testing.T) {
 		So(i.Store.Size, ShouldEqual, 11000)
 		So(i.Store.PriSize, ShouldEqual, 13000)
 	})
-	Convey("Create index line from a bad deletes index listing", t, func() {
-		i, err := NewCatIndexInfo("red foo-2000-01-01-bar 2 1 1234 a 11000 13000")
+	Convey("catIndex Create index line from a bad deletes index listing", t, func() {
+		i, err := NewCatIndexInfo("red open foo-2000-01-01-bar 2 1 1234 a 11000 13000")
 		So(err, ShouldBeNil)
 		So(i.Health, ShouldEqual, "red")
+		So(i.Status, ShouldEqual, "open")
 		So(i.Name, ShouldEqual, "foo-2000-01-01-bar")
 		So(i.Shards, ShouldEqual, 2)
 		So(i.Replicas, ShouldEqual, 1)
@@ -70,10 +77,11 @@ func TestCatIndexInfo(t *testing.T) {
 		So(i.Store.Size, ShouldEqual, 11000)
 		So(i.Store.PriSize, ShouldEqual, 13000)
 	})
-	Convey("Create index line from a kinda short index listing", t, func() {
-		i, err := NewCatIndexInfo("red foo-2000-01-01-bar 2 1 1234")
+	Convey("catIndex Create index line from a kinda short index listing", t, func() {
+		i, err := NewCatIndexInfo("red open foo-2000-01-01-bar 2 1 1234")
 		So(err, ShouldBeNil)
 		So(i.Health, ShouldEqual, "red")
+		So(i.Status, ShouldEqual, "open")
 		So(i.Name, ShouldEqual, "foo-2000-01-01-bar")
 		So(i.Shards, ShouldEqual, 2)
 		So(i.Replicas, ShouldEqual, 1)
@@ -82,10 +90,11 @@ func TestCatIndexInfo(t *testing.T) {
 		So(i.Store.Size, ShouldEqual, 0)
 		So(i.Store.PriSize, ShouldEqual, 0)
 	})
-	Convey("Create index line from a kinda sorta short index listing", t, func() {
-		i, err := NewCatIndexInfo("red foo-2000-01-01-bar 2 1 1234 3")
+	Convey("catIndex Create index line from a kinda sorta short index listing", t, func() {
+		i, err := NewCatIndexInfo("red open foo-2000-01-01-bar 2 1 1234 3")
 		So(err, ShouldBeNil)
 		So(i.Health, ShouldEqual, "red")
+		So(i.Status, ShouldEqual, "open")
 		So(i.Name, ShouldEqual, "foo-2000-01-01-bar")
 		So(i.Shards, ShouldEqual, 2)
 		So(i.Replicas, ShouldEqual, 1)
@@ -94,10 +103,11 @@ func TestCatIndexInfo(t *testing.T) {
 		So(i.Store.Size, ShouldEqual, 0)
 		So(i.Store.PriSize, ShouldEqual, 0)
 	})
-	Convey("Create index line from a short index listing", t, func() {
-		i, err := NewCatIndexInfo("red foo-2000-01-01-bar 2 1")
+	Convey("catIndex Create index line from a short index listing", t, func() {
+		i, err := NewCatIndexInfo("red open foo-2000-01-01-bar 2 1")
 		So(err, ShouldBeNil)
 		So(i.Health, ShouldEqual, "red")
+		So(i.Status, ShouldEqual, "open")
 		So(i.Name, ShouldEqual, "foo-2000-01-01-bar")
 		So(i.Shards, ShouldEqual, 2)
 		So(i.Replicas, ShouldEqual, 1)
diff --git a/lib/catresponses.go b/lib/catresponses.go
index 0f679f3b..55c4d9e9 100644
--- a/lib/catresponses.go
+++ b/lib/catresponses.go
@@ -2,6 +2,7 @@ package elastigo
 
 type CatIndexInfo struct {
 	Health   string
+	Status   string
 	Name     string
 	Shards   int
 	Replicas int

From de547697de6d4c85ac6c049d38d6b64a2d42a111 Mon Sep 17 00:00:00 2001
From: Ben Aldrich <ben.aldrich@logrhythm.com>
Date: Tue, 7 Jul 2015 15:59:59 -0600
Subject: [PATCH 30/81] cleanup up comment

---
 lib/catindexinfo.go      | 4 ++--
 lib/catindexinfo_test.go | 2 --
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/lib/catindexinfo.go b/lib/catindexinfo.go
index 0e48cc1e..ea826bbd 100644
--- a/lib/catindexinfo.go
+++ b/lib/catindexinfo.go
@@ -9,8 +9,8 @@ import (
 var ErrInvalidIndexLine = errors.New("Cannot parse indexline")
 
 //Create an IndexInfo from the string _cat/indices would produce
-//EX: health status index    pri rep docs.count docs.deleted store.size pri.store.size
-//green open logs-2015-06-19   2 0 135389346 0 53048922233 53048922233
+//EX: health status index    pri rep docs.count docs.deleted store.size  pri.store.size
+//green open logs-2015-06-19   2 0   135389346  0            53048922233 53048922233
 func NewCatIndexInfo(indexLine string) (catIndex *CatIndexInfo, err error) {
 	split := strings.Fields(indexLine)
 	if len(split) < 5 {
diff --git a/lib/catindexinfo_test.go b/lib/catindexinfo_test.go
index f4b28c91..d09f6cd9 100644
--- a/lib/catindexinfo_test.go
+++ b/lib/catindexinfo_test.go
@@ -11,8 +11,6 @@ func TestCatIndexInfo(t *testing.T) {
 		So(err, ShouldNotBeNil)
 	})
 	Convey("catIndex Create index line from a bad shards index listing", t, func() {
-		// green  open   twitter    5   1      11434            0       64mb           32mb
-		//green open logs-2015-06-19   2 1 135389346 20 53048922233 53048922233
 		i, err := NewCatIndexInfo("green open logs-2015-06-19   2 1 135389346 20 53048922233 53048922233")
 		So(err, ShouldBeNil)
 		So(i.Health, ShouldEqual, "green")

From 15969a5bc0c940a38de9ef7d710324aa7ed1aa3c Mon Sep 17 00:00:00 2001
From: Ben Aldrich <ben.aldrich@logrhythm.com>
Date: Thu, 9 Jul 2015 15:00:32 -0600
Subject: [PATCH 31/81] fix cat indices to call out columns

---
 lib/catindexinfo.go | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/catindexinfo.go b/lib/catindexinfo.go
index ea826bbd..44ca2d80 100644
--- a/lib/catindexinfo.go
+++ b/lib/catindexinfo.go
@@ -64,7 +64,8 @@ func NewCatIndexInfo(indexLine string) (catIndex *CatIndexInfo, err error) {
 // Pull all the index info from the connection
 func (c *Conn) GetCatIndexInfo(pattern string) (catIndices []CatIndexInfo) {
 	catIndices = make([]CatIndexInfo, 0)
-	args := map[string]interface{}{"bytes": "b"}
+	//force it to only show the fileds we know about
+	args := map[string]interface{}{"bytes": "b", "h": "health,status,index,pri,rep,docs.count,docs.deleted,store.size,pri.store.size"}
 	indices, err := c.DoCommand("GET", "/_cat/indices/"+pattern, args, nil)
 	if err == nil {
 		indexLines := strings.Split(string(indices[:]), "\n")

From cda563c7c350db3f703e3711bea4839bf169d02b Mon Sep 17 00:00:00 2001
From: Ben Aldrich <ben.aldrich@logrhythm.com>
Date: Thu, 9 Jul 2015 15:06:33 -0600
Subject: [PATCH 32/81] fix shards cat api as well

---
 lib/catshardinfo.go | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/catshardinfo.go b/lib/catshardinfo.go
index b4b1d502..c93366b9 100644
--- a/lib/catshardinfo.go
+++ b/lib/catshardinfo.go
@@ -90,7 +90,8 @@ func (s *CatShardInfo) String() string {
 // Get all the shards, even the bad ones
 func (c *Conn) GetCatShards() (shards CatShards) {
 	shards = make(CatShards, 0)
-	args := map[string]interface{}{"bytes": "b"}
+	//force it to only respond with the columns we know about and in a forced order
+	args := map[string]interface{}{"bytes": "b", "h": "index,shard,prirep,state,docs,store,ip,node"}
 	s, err := c.DoCommand("GET", "/_cat/shards", args, nil)
 	if err == nil {
 		catShardLines := strings.Split(string(s[:]), "\n")

From 9b38f64e8cddc44e0b5ef82cf4cc15d6926f046a Mon Sep 17 00:00:00 2001
From: Nicolas Donna <donnanicolas@gmail.com>
Date: Thu, 30 Apr 2015 19:07:12 -0300
Subject: [PATCH 33/81] Added parent parameter to bulk indexer

---
 lib/corebulk.go      | 24 +++++++++++++++---------
 lib/corebulk_test.go | 18 +++++++++---------
 lib/coretest_test.go |  2 +-
 3 files changed, 25 insertions(+), 19 deletions(-)

diff --git a/lib/corebulk.go b/lib/corebulk.go
index 660078ed..77818746 100644
--- a/lib/corebulk.go
+++ b/lib/corebulk.go
@@ -280,9 +280,9 @@ func (b *BulkIndexer) shutdown() {
 // The index bulk API adds or updates a typed JSON document to a specific index, making it searchable.
 // it operates by buffering requests, and ocassionally flushing to elasticsearch
 // http://www.elasticsearch.org/guide/reference/api/bulk.html
-func (b *BulkIndexer) Index(index string, _type string, id, ttl string, date *time.Time, data interface{}, refresh bool) error {
+func (b *BulkIndexer) Index(index string, _type string, id, parent, ttl string, date *time.Time, data interface{}, refresh bool) error {
 	//{ "index" : { "_index" : "test", "_type" : "type1", "_id" : "1" } }
-	by, err := WriteBulkBytes("index", index, _type, id, ttl, date, data, refresh)
+	by, err := WriteBulkBytes("index", index, _type, id, parent, ttl, date, data, refresh)
 	if err != nil {
 		return err
 	}
@@ -290,9 +290,9 @@ func (b *BulkIndexer) Index(index string, _type string, id, ttl string, date *ti
 	return nil
 }
 
-func (b *BulkIndexer) Update(index string, _type string, id, ttl string, date *time.Time, data interface{}, refresh bool) error {
+func (b *BulkIndexer) Update(index string, _type string, id, parent, ttl string, date *time.Time, data interface{}, refresh bool) error {
 	//{ "index" : { "_index" : "test", "_type" : "type1", "_id" : "1" } }
-	by, err := WriteBulkBytes("update", index, _type, id, ttl, date, data, refresh)
+	by, err := WriteBulkBytes("update", index, _type, id, parent, ttl, date, data, refresh)
 	if err != nil {
 		return err
 	}
@@ -306,14 +306,14 @@ func (b *BulkIndexer) Delete(index, _type, id string, refresh bool) {
 	return
 }
 
-func (b *BulkIndexer) UpdateWithWithScript(index string, _type string, id, ttl string, date *time.Time, script string, refresh bool) error {
+func (b *BulkIndexer) UpdateWithWithScript(index string, _type string, id, parent, ttl string, date *time.Time, script string, refresh bool) error {
 
 	var data map[string]interface{} = make(map[string]interface{})
 	data["script"] = script
-	return b.Update(index, _type, id, ttl, date, data, refresh)
+	return b.Update(index, _type, id, parent, ttl, date, data, refresh)
 }
 
-func (b *BulkIndexer) UpdateWithPartialDoc(index string, _type string, id, ttl string, date *time.Time, partialDoc interface{}, upsert bool, refresh bool) error {
+func (b *BulkIndexer) UpdateWithPartialDoc(index string, _type string, id, parent, ttl string, date *time.Time, partialDoc interface{}, upsert bool, refresh bool) error {
 
 	var data map[string]interface{} = make(map[string]interface{})
 
@@ -321,7 +321,7 @@ func (b *BulkIndexer) UpdateWithPartialDoc(index string, _type string, id, ttl s
 	if upsert {
 		data["doc_as_upsert"] = true
 	}
-	return b.Update(index, _type, id, ttl, date, data, refresh)
+	return b.Update(index, _type, id, parent, ttl, date, data, refresh)
 }
 
 // This does the actual send of a buffer, which has already been formatted
@@ -354,7 +354,7 @@ func (b *BulkIndexer) Send(buf *bytes.Buffer) error {
 
 // Given a set of arguments for index, type, id, data create a set of bytes that is formatted for bulkd index
 // http://www.elasticsearch.org/guide/reference/api/bulk.html
-func WriteBulkBytes(op string, index string, _type string, id, ttl string, date *time.Time, data interface{}, refresh bool) ([]byte, error) {
+func WriteBulkBytes(op string, index string, _type string, id, parent, ttl string, date *time.Time, data interface{}, refresh bool) ([]byte, error) {
 	// only index and update are currently supported
 	if op != "index" && op != "update" {
 		return nil, errors.New(fmt.Sprintf("Operation '%s' is not yet supported", op))
@@ -373,6 +373,12 @@ func WriteBulkBytes(op string, index string, _type string, id, ttl string, date
 		buf.WriteString(`"`)
 	}
 
+	if len(parent) > 0 {
+		buf.WriteString(`,"_parent":"`)
+		buf.WriteString(parent)
+		buf.WriteString(`"`)
+	}
+
 	if op == "update" {
 		buf.WriteString(`,"retry_on_conflict":3`)
 	}
diff --git a/lib/corebulk_test.go b/lib/corebulk_test.go
index 3a6b615e..1acc5eab 100644
--- a/lib/corebulk_test.go
+++ b/lib/corebulk_test.go
@@ -75,7 +75,7 @@ func TestBulkIndexerBasic(t *testing.T) {
 		"date": "yesterday",
 	}
 
-	err := indexer.Index(testIndex, "user", "1", "", &date, data, true)
+	err := indexer.Index(testIndex, "user", "1", "", "", &date, data, true)
 
 	waitFor(func() bool {
 		return len(buffers) > 0
@@ -87,7 +87,7 @@ func TestBulkIndexerBasic(t *testing.T) {
 	expectedBytes := 144
 	assert.T(t, totalBytesSent == expectedBytes, fmt.Sprintf("Should have sent %v bytes but was %v", expectedBytes, totalBytesSent))
 
-	err = indexer.Index(testIndex, "user", "2", "", nil, data, true)
+	err = indexer.Index(testIndex, "user", "2", "", "", nil, data, true)
 	<-time.After(time.Millisecond * 10) // we need to wait for doc to hit send channel
 	// this will test to ensure that Flush actually catches a doc
 	indexer.Flush()
@@ -134,7 +134,7 @@ func XXXTestBulkUpdate(t *testing.T) {
 	data := map[string]interface{}{
 		"script": "ctx._source.count += 2",
 	}
-	err = indexer.Update("users", "user", "5", "", &date, data, true)
+	err = indexer.Update("users", "user", "5", "", "", &date, data, true)
 	// So here's the deal. Flushing does seem to work, you just have to give the
 	// channel a moment to recieve the message ...
 	//	<- time.After(time.Millisecond * 20)
@@ -180,9 +180,9 @@ func TestBulkSmallBatch(t *testing.T) {
 	indexer.Start()
 	<-time.After(time.Millisecond * 20)
 
-	indexer.Index("users", "user", "2", "", &date, data, true)
-	indexer.Index("users", "user", "3", "", &date, data, true)
-	indexer.Index("users", "user", "4", "", &date, data, true)
+	indexer.Index("users", "user", "2", "", "", &date, data, true)
+	indexer.Index("users", "user", "3", "", "", &date, data, true)
+	indexer.Index("users", "user", "4", "", "", &date, data, true)
 	<-time.After(time.Millisecond * 200)
 	//	indexer.Flush()
 	indexer.Stop()
@@ -231,7 +231,7 @@ func XXXTestBulkErrors(t *testing.T) {
 		for i := 0; i < 20; i++ {
 			date := time.Unix(1257894000, 0)
 			data := map[string]interface{}{"name": "smurfs", "age": 22, "date": date}
-			indexer.Index("users", "user", strconv.Itoa(i), "", &date, data, true)
+			indexer.Index("users", "user", strconv.Itoa(i), "", "", &date, data, true)
 		}
 	}()
 	var errBuf *ErrorBuffer
@@ -271,7 +271,7 @@ func BenchmarkSend(b *testing.B) {
 		about := make([]byte, 1000)
 		rand.Read(about)
 		data := map[string]interface{}{"name": "smurfs", "age": 22, "date": time.Unix(1257894000, 0), "about": about}
-		indexer.Index("users", "user", strconv.Itoa(i), "", nil, data, true)
+		indexer.Index("users", "user", strconv.Itoa(i), "", "", nil, data, true)
 	}
 	log.Printf("Sent %d messages in %d sets totaling %d bytes \n", b.N, sets, totalBytes)
 	if indexer.NumErrors() != 0 {
@@ -305,7 +305,7 @@ func BenchmarkSendBytes(b *testing.B) {
 		return indexer.Send(buf)
 	}
 	for i := 0; i < b.N; i++ {
-		indexer.Index("users", "user", strconv.Itoa(i), "", nil, body, true)
+		indexer.Index("users", "user", strconv.Itoa(i), "", "", nil, body, true)
 	}
 	log.Printf("Sent %d messages in %d sets totaling %d bytes \n", b.N, sets, totalBytes)
 	if indexer.NumErrors() != 0 {
diff --git a/lib/coretest_test.go b/lib/coretest_test.go
index 9af08bce..6d2acf72 100644
--- a/lib/coretest_test.go
+++ b/lib/coretest_test.go
@@ -179,7 +179,7 @@ func LoadTestData() {
 				log.Println("HM, already exists? ", ge.Url)
 			}
 			docsm[id] = true
-			indexer.Index(testIndex, ge.Type, id, "", &ge.Created, line, true)
+			indexer.Index(testIndex, ge.Type, id, "", "", &ge.Created, line, true)
 			docCt++
 		} else {
 			log.Println("ERROR? ", string(line))

From edd785533425251c431d05a726ab524161024b3f Mon Sep 17 00:00:00 2001
From: "Keene, Ramin" <Ramin.Keene@exchange.nordstrom.com>
Date: Thu, 16 Jul 2015 12:28:08 -0700
Subject: [PATCH 34/81] add support for _ttl mapping field when using
 MappingOptions struct to define mappings

---
 lib/indicesputmapping.go      | 6 ++++++
 lib/indicesputmapping_test.go | 2 ++
 2 files changed, 8 insertions(+)

diff --git a/lib/indicesputmapping.go b/lib/indicesputmapping.go
index dcfb2eda..aab0d228 100644
--- a/lib/indicesputmapping.go
+++ b/lib/indicesputmapping.go
@@ -28,6 +28,7 @@ type MappingOptions struct {
 	Routing    *RoutingOptions        `json:"_routing,omitempty"`
 	Size       *SizeOptions           `json:"_size,omitempty"`
 	Source     *SourceOptions         `json:"_source,omitempty"`
+	TTL        *TTLOptions            `json:"_ttl,omitempty"`
 	Type       *TypeOptions           `json:"_type,omitempty"`
 	Properties map[string]interface{} `json:"properties"`
 }
@@ -66,6 +67,11 @@ type TypeOptions struct {
 	Index string `json:"index,omitempty"`
 }
 
+type TTLOptions struct {
+	Enabled bool `json:"enabled"`
+	Default string `json:"default,omitempty"`
+}
+
 type IdOptions struct {
 	Index string `json:"index,omitempty"`
 	Path  string `json:"path,omitempty"`
diff --git a/lib/indicesputmapping_test.go b/lib/indicesputmapping_test.go
index 3a4b7bef..8da3ca34 100644
--- a/lib/indicesputmapping_test.go
+++ b/lib/indicesputmapping_test.go
@@ -99,6 +99,7 @@ func TestPutMapping(t *testing.T) {
 		Timestamp: TimestampOptions{Enabled: true},
 		Id:        IdOptions{Index: "analyzed", Path: "id"},
 		Parent:    &ParentOptions{Type: "testParent"},
+		TTL:       &TTLOptions{Enabled: true, Default: "1w"},
 		Properties: map[string]interface{}{
 			// special properties that can't be expressed as tags
 			"multi_analyze": map[string]interface{}{
@@ -114,6 +115,7 @@ func TestPutMapping(t *testing.T) {
 		Timestamp: TimestampOptions{Enabled: true},
 		Id:        IdOptions{Index: "analyzed", Path: "id"},
 		Parent:    &ParentOptions{Type: "testParent"},
+		TTL:       &TTLOptions{Enabled: true, Default: "1w"},
 		Properties: map[string]interface{}{
 			"NoJson":        map[string]string{"type": "string"},
 			"dontIndex":     map[string]string{"index": "no"},

From 43070ecda6aed9e9672c687da45bfc6c7276b286 Mon Sep 17 00:00:00 2001
From: Martin Gonzalez <tinchogon34@gmail.com>
Date: Wed, 12 Aug 2015 10:30:25 -0300
Subject: [PATCH 35/81] Partial Fix percolator result structure

---
 lib/baseresponse.go | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/lib/baseresponse.go b/lib/baseresponse.go
index a2a8b7dc..41770dfa 100644
--- a/lib/baseresponse.go
+++ b/lib/baseresponse.go
@@ -118,9 +118,14 @@ type ExtendedStatus struct {
 	ShardsStatus Status     `json:"_shards"`
 }
 
+type MatchRes struct {
+	Index string `json:"_index"`
+	Id    string `json:"_id"`
+}
+
 type Match struct {
 	OK          bool         `json:"ok"`
-	Matches     []string     `json:"matches"`
+	Matches     []MatchRes   `json:"matches"`
 	Explanation *Explanation `json:"explanation,omitempty"`
 }
 

From 49a12c9bf0e7e5b3dbe9d5a6fe107334a49b261b Mon Sep 17 00:00:00 2001
From: Cheney <me@chenye.org>
Date: Mon, 24 Aug 2015 18:01:22 +0800
Subject: [PATCH 36/81] update bulk api retry_on_conflict params

---
 lib/corebulk.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/corebulk.go b/lib/corebulk.go
index 5010f843..9a3c2dba 100644
--- a/lib/corebulk.go
+++ b/lib/corebulk.go
@@ -375,7 +375,7 @@ func WriteBulkBytes(op string, index string, _type string, id, ttl string, date
 	}
 
 	if op == "update" {
-		buf.WriteString(`,"retry_on_conflict":3`)
+		buf.WriteString(`,"_retry_on_conflict":3`)
 	}
 
 	if len(ttl) > 0 {

From a710d5bdd0c2cc16344e2c6f86b458d2d98b0ccc Mon Sep 17 00:00:00 2001
From: Robert Weber <robert.weber@logrhythm.com>
Date: Mon, 24 Aug 2015 14:52:49 -0600
Subject: [PATCH 37/81] Fixed args for the parent parameter PR

---
 lib/coreexample_test.go | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/lib/coreexample_test.go b/lib/coreexample_test.go
index 231ebd2f..608ee4a7 100644
--- a/lib/coreexample_test.go
+++ b/lib/coreexample_test.go
@@ -14,8 +14,9 @@ package elastigo_test
 import (
 	"bytes"
 	"fmt"
-	elastigo "github.com/mattbaird/elastigo/lib"
 	"strconv"
+
+	elastigo "github.com/mattbaird/elastigo/lib"
 )
 
 // The simplest usage of background bulk indexing
@@ -24,7 +25,7 @@ func ExampleBulkIndexer_simple() {
 
 	indexer := c.NewBulkIndexerErrors(10, 60)
 	indexer.Start()
-	indexer.Index("twitter", "user", "1", "", nil, `{"name":"bob"}`, true)
+	indexer.Index("twitter", "user", "1", "", "", nil, `{"name":"bob"}`, true)
 	indexer.Stop()
 }
 
@@ -45,7 +46,7 @@ func ExampleBulkIndexer_responses() {
 	}
 	indexer.Start()
 	for i := 0; i < 20; i++ {
-		indexer.Index("twitter", "user", strconv.Itoa(i), "", nil, `{"name":"bob"}`, true)
+		indexer.Index("twitter", "user", strconv.Itoa(i), "", "", nil, `{"name":"bob"}`, true)
 	}
 	indexer.Stop()
 }

From 6eac474ed28dfe7874579792f90b8640933791c1 Mon Sep 17 00:00:00 2001
From: Dimitri Roche <dimroc@gmail.com>
Date: Tue, 25 Aug 2015 13:16:22 +0200
Subject: [PATCH 38/81] Fix RegisterPercolate to actually work with ES 1.6+

Unsure if this ever worked with any version of ES.
---
 lib/corepercolate.go | 18 ++++--------------
 1 file changed, 4 insertions(+), 14 deletions(-)

diff --git a/lib/corepercolate.go b/lib/corepercolate.go
index e9f459f5..cc38bfa2 100644
--- a/lib/corepercolate.go
+++ b/lib/corepercolate.go
@@ -16,9 +16,6 @@ import (
 	"fmt"
 )
 
-//{"took":9,"_shards":{"total":3,"successful":3,"failed":0},"total":3,
-//"matches":[{"_index":"test-geoevents-20150706-115234","_id":"nyc,West Village"},{"_index":"test-geoevents-20150706-115234","_id":"nyc,Manhattan"},{"_index":"test-geoevents-20150706-115234","_id":"nyc,Greenwich Village"}]}
-
 type PercolatorResult struct {
 	SearchResult
 	Matches []PercolatorMatch `json:"matches"`
@@ -29,19 +26,12 @@ type PercolatorMatch struct {
 	Id    string `json:"_id"`
 }
 
-// RegisterPercolate allows the caller to register queries against an index,
-// and then send percolate requests which include a doc, and getting back the
-// queries that match on that doc out of the set of registered queries.  Think
-// of it as the reverse operation of indexing and then searching. Instead of
-// sending docs, indexing them, and then running queries. One sends queries,
-// registers them, and then sends docs and finds out which queries match that
-// doc.
-// see http://www.elasticsearch.org/guide/reference/api/percolate.html
-func (c *Conn) RegisterPercolate(index string, name string, args map[string]interface{}, query OneTermQuery) (BaseResponse, error) {
+// See http://www.elasticsearch.org/guide/reference/api/percolate.html
+func (c *Conn) RegisterPercolate(index string, id string, data interface{}) (BaseResponse, error) {
 	var url string
 	var retval BaseResponse
-	url = fmt.Sprintf("/_percolator/%s/%s", index, name)
-	body, err := c.DoCommand("PUT", url, args, query)
+	url = fmt.Sprintf("/%s/.percolator/%s", index, id)
+	body, err := c.DoCommand("PUT", url, nil, data)
 	if err != nil {
 		return retval, err
 	}

From 9940c38cc55fa6d425053bd557635cef33486a6c Mon Sep 17 00:00:00 2001
From: Dimitri Roche <dimroc@gmail.com>
Date: Tue, 25 Aug 2015 13:16:37 +0200
Subject: [PATCH 39/81] Add test coverage of both RegisterPercolate and
 Percolate

---
 lib/corepercolate_test.go | 64 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 64 insertions(+)
 create mode 100644 lib/corepercolate_test.go

diff --git a/lib/corepercolate_test.go b/lib/corepercolate_test.go
new file mode 100644
index 00000000..55a0713a
--- /dev/null
+++ b/lib/corepercolate_test.go
@@ -0,0 +1,64 @@
+package elastigo
+
+import (
+	. "github.com/smartystreets/goconvey/convey"
+	"testing"
+)
+
+const (
+	percIndexName = "test-perc-index"
+)
+
+func TestPercolate(t *testing.T) {
+	Convey("With a registered percolator", t, func() {
+		c := NewTestConn()
+		_, createErr := c.CreateIndex(percIndexName)
+		So(createErr, ShouldBeNil)
+		defer c.DeleteIndex(percIndexName)
+
+		options := `{
+      "percType": {
+        "properties": {
+          "message": {
+            "type": "string"
+          }
+        }
+      }
+    }`
+
+		err := c.PutMappingFromJSON(percIndexName, "percType", []byte(options))
+		So(err, ShouldBeNil)
+
+		data := `{
+      "query": {
+        "match": {
+          "message": "bonsai tree"
+        }
+      }
+    }`
+
+		_, err = c.RegisterPercolate(percIndexName, "PERCID", data)
+		So(err, ShouldBeNil)
+
+		Convey("That matches the document", func() {
+			// Should return the percolator id (registered query)
+			doc := `{"doc": { "message": "A new bonsai tree in the office" }}`
+
+			result, err := c.Percolate(percIndexName, "percType", "", nil, doc)
+			So(err, ShouldBeNil)
+			So(len(result.Matches), ShouldEqual, 1)
+			match := result.Matches[0]
+			So(match.Id, ShouldEqual, "PERCID")
+			So(match.Index, ShouldEqual, percIndexName)
+		})
+
+		Convey("That does not match the document", func() {
+			// Should NOT return the percolator id (registered query)
+			doc := `{"doc": { "message": "Barren wasteland with no matches" }}`
+
+			result, err := c.Percolate(percIndexName, "percType", "", nil, doc)
+			So(err, ShouldBeNil)
+			So(len(result.Matches), ShouldEqual, 0)
+		})
+	})
+}

From 2ff99561cba57275e2fa6e098710eeb71343cb35 Mon Sep 17 00:00:00 2001
From: Paul Oldenburg <paul.oldenburg@logrhythm.com>
Date: Thu, 27 Aug 2015 11:54:38 -0600
Subject: [PATCH 40/81] Increase the wait so that the index will have time to
 commit

---
 lib/corebulk_test.go | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/lib/corebulk_test.go b/lib/corebulk_test.go
index 1acc5eab..d8486503 100644
--- a/lib/corebulk_test.go
+++ b/lib/corebulk_test.go
@@ -76,10 +76,10 @@ func TestBulkIndexerBasic(t *testing.T) {
 	}
 
 	err := indexer.Index(testIndex, "user", "1", "", "", &date, data, true)
-
 	waitFor(func() bool {
 		return len(buffers) > 0
 	}, 5)
+
 	// part of request is url, so lets factor that in
 	//totalBytesSent = totalBytesSent - len(*eshost)
 	assert.T(t, len(buffers) == 1, fmt.Sprintf("Should have sent one operation but was %d", len(buffers)))
@@ -88,7 +88,10 @@ func TestBulkIndexerBasic(t *testing.T) {
 	assert.T(t, totalBytesSent == expectedBytes, fmt.Sprintf("Should have sent %v bytes but was %v", expectedBytes, totalBytesSent))
 
 	err = indexer.Index(testIndex, "user", "2", "", "", nil, data, true)
-	<-time.After(time.Millisecond * 10) // we need to wait for doc to hit send channel
+	waitFor(func() bool {
+		return len(buffers) > 1
+	}, 5)
+
 	// this will test to ensure that Flush actually catches a doc
 	indexer.Flush()
 	totalBytesSent = totalBytesSent - len(*eshost)

From 080a19c96415eee7cef38e873ce9ee78057a1457 Mon Sep 17 00:00:00 2001
From: Paul Oldenburg <paul.oldenburg@logrhythm.com>
Date: Thu, 27 Aug 2015 13:01:32 -0600
Subject: [PATCH 41/81] Add new test set up with mock data

---
 lib/baserequest.go       |   5 +
 lib/coresearch_test.go   | 114 ++++++++--------
 lib/coretest_test.go     |   2 +
 lib/request.go           |  22 ++--
 lib/searchfilter_test.go | 154 +++++++++++-----------
 lib/searchsearch_test.go | 274 +++++++++++++++++++--------------------
 lib/testsetup.go         |  70 ++++++++++
 7 files changed, 352 insertions(+), 289 deletions(-)
 create mode 100644 lib/testsetup.go

diff --git a/lib/baserequest.go b/lib/baserequest.go
index e44ee9e9..49790a81 100644
--- a/lib/baserequest.go
+++ b/lib/baserequest.go
@@ -18,6 +18,7 @@ import (
 	"io"
 	"io/ioutil"
 	"log"
+	"net/http/httputil"
 	"time"
 )
 
@@ -51,6 +52,10 @@ func (c *Conn) DoCommand(method string, url string, args map[string]interface{},
 		}
 	}
 
+	// uncomment this to print out the request that hits the wire
+	reqbuf, err := httputil.DumpRequest(req.Request, true)
+	log.Println(fmt.Sprintf("\n========= req:\nURL: %s\n%s", req.URL, bytes.NewBuffer(reqbuf).String()))
+
 	// Copy request body for tracer
 	if c.RequestTracer != nil {
 		requestBody, err := ioutil.ReadAll(req.Body)
diff --git a/lib/coresearch_test.go b/lib/coresearch_test.go
index f443e107..a16d6fec 100644
--- a/lib/coresearch_test.go
+++ b/lib/coresearch_test.go
@@ -13,71 +13,71 @@ package elastigo
 
 import (
 	"encoding/json"
-	"fmt"
-	"github.com/bmizerany/assert"
+	. "github.com/smartystreets/goconvey/convey"
 	"testing"
 )
 
-func ok(t *testing.T, err error) {
-	if err != nil {
-		t.Fatal(err)
-	}
-}
-
-func TestSearchResultToJSON(t *testing.T) {
-	c := NewTestConn()
-
-	qry := map[string]interface{}{
-		"query": map[string]interface{}{
-			"wildcard": map[string]string{"actor": "a*"},
-		},
-	}
-	var args map[string]interface{}
-	out, err := c.Search("github", "", args, qry)
-	ok(t, err)
-
-	_, err = json.Marshal(out.Hits.Hits)
-	ok(t, err)
-}
-
 type SuggestTest struct {
 	Completion string `json:"completion"`
 }
 
 type hash map[string]interface{}
 
-func TestSuggest(t *testing.T) {
+func TestCoreSearch(t *testing.T) {
+
 	c := NewTestConn()
-	mappingOpts := MappingOptions{Properties: hash{
-		"completion": hash{
-			"type": "completion",
-		},
-	}}
-	err := c.PutMapping("github", "SuggestTest", SuggestTest{}, mappingOpts)
-	ok(t, err)
-
-	_, err = c.UpdateWithPartialDoc("github", "SuggestTest", "1", nil, SuggestTest{"foobar"}, true)
-	ok(t, err)
-
-	query := hash{"completion_completion": hash{
-		"text": "foo",
-		"completion": hash{
-			"size":  10,
-			"field": "completion",
-		},
-	}}
-
-	_, err = c.Refresh("github")
-	ok(t, err)
-
-	res, err := c.Suggest("github", nil, query)
-	ok(t, err)
-
-	opts, err := res.Result("completion_completion")
-	ok(t, err)
-
-	first := opts[0]
-	assert.T(t, len(first.Options) > 0, "Length of first.Options was 0.")
-	text := first.Options[0].Text
-	assert.T(t, text == "foobar", fmt.Sprintf("Expected foobar, got: %s", text))
+	c.CreateIndex("github")
+	waitFor(func() bool { return false }, 5)
+
+	defer func() {
+		c.DeleteIndex("github")
+	}()
+
+	Convey("Convert a search result to JSON", t, func() {
+
+		qry := map[string]interface{}{
+			"query": map[string]interface{}{
+				"wildcard": map[string]string{"actor": "a*"},
+			},
+		}
+		var args map[string]interface{}
+		out, err := c.Search("github", "", args, qry)
+		So(err, ShouldBeNil)
+
+		_, err = json.Marshal(out.Hits.Hits)
+		So(err, ShouldBeNil)
+	})
+
+	Convey("Update a document and verify that it is reflected", t, func() {
+		mappingOpts := MappingOptions{Properties: hash{
+			"completion": hash{
+				"type": "completion",
+			},
+		}}
+		err := c.PutMapping("github", "SuggestTest", SuggestTest{}, mappingOpts)
+		So(err, ShouldBeNil)
+
+		_, err = c.UpdateWithPartialDoc("github", "SuggestTest", "1", nil, SuggestTest{"foobar"}, true)
+		So(err, ShouldBeNil)
+
+		query := hash{"completion_completion": hash{
+			"text": "foo",
+			"completion": hash{
+				"size":  10,
+				"field": "completion",
+			},
+		}}
+
+		_, err = c.Refresh("github")
+		So(err, ShouldBeNil)
+
+		res, err := c.Suggest("github", nil, query)
+		So(err, ShouldBeNil)
+
+		opts, err := res.Result("completion_completion")
+		So(err, ShouldBeNil)
+
+		So(len(opts[0].Options), ShouldBeGreaterThan, 0)
+		So(opts[0].Options[0].Text, ShouldEqual, "foobar")
+	})
 }
diff --git a/lib/coretest_test.go b/lib/coretest_test.go
index 6d2acf72..33090cf0 100644
--- a/lib/coretest_test.go
+++ b/lib/coretest_test.go
@@ -116,6 +116,8 @@ func LoadTestData() {
 	c := NewConn()
 	c.Domain = *eshost
 
+	fmt.Println("Loading test data")
+
 	c.DeleteIndex(testIndex)
 
 	docCt := 0
diff --git a/lib/request.go b/lib/request.go
index b4ac4b34..56ad377a 100644
--- a/lib/request.go
+++ b/lib/request.go
@@ -54,16 +54,20 @@ func (r *Request) SetBody(body io.Reader) {
 		rc = ioutil.NopCloser(body)
 	}
 	r.Body = rc
-	if body != nil {
-		switch v := body.(type) {
-		case *strings.Reader:
-			r.ContentLength = int64(v.Len())
-		case *bytes.Reader:
-			r.ContentLength = int64(v.Len())
-		case *bytes.Buffer:
-			r.ContentLength = int64(v.Len())
+	r.ContentLength = -1
+	/*
+		if body != nil {
+			switch v := body.(type) {
+			case *strings.Reader:
+				r.ContentLength = int64(v.Len())
+			case *bytes.Reader:
+				r.ContentLength = int64(v.Len())
+			case *bytes.Buffer:
+				r.ContentLength = int64(v.Len())
+			}
 		}
-	}
+		fmt.Printf("SetBody() - ContentLength: %d  body.(type): %T\n", r.ContentLength)
+	*/
 }
 
 func (r *Request) Do(v interface{}) (int, []byte, error) {
diff --git a/lib/searchfilter_test.go b/lib/searchfilter_test.go
index 7f533891..defc9c1e 100644
--- a/lib/searchfilter_test.go
+++ b/lib/searchfilter_test.go
@@ -12,96 +12,88 @@
 package elastigo
 
 import (
-	"fmt"
-	//"github.com/araddon/gou"
-	"github.com/bmizerany/assert"
+	. "github.com/smartystreets/goconvey/convey"
 	"testing"
 )
 
 func TestFilters(t *testing.T) {
-	c := NewTestConn()
 
-	// search for docs that are missing repository.name
-	qry := Search("github").Filter(
-		Filter().Exists("repository.name"),
-	)
-	out, err := qry.Result(c)
-	assert.T(t, err == nil, t, "should not have error")
-	expectedDocs := 10
-	expectedHits := 7695
-	assert.T(t, out.Hits.Len() == expectedDocs, fmt.Sprintf("Should have %v docs got %v", expectedDocs, out.Hits.Len()))
-	assert.T(t, out.Hits.Total == expectedHits, fmt.Sprintf("Should have %v total got %v", expectedHits, out.Hits.Total))
-	qry = Search("github").Filter(
-		Filter().Missing("repository.name"),
-	)
-	expectedHits = 390
-	out, _ = qry.Result(c)
-	assert.T(t, out.Hits.Len() == expectedDocs, fmt.Sprintf("Should have %v docs got %v", expectedDocs, out.Hits.Len()))
-	assert.T(t, out.Hits.Total == expectedHits, fmt.Sprintf("Should have %v total got %v", expectedHits, out.Hits.Total))
+	c := NewTestConn()
+	PopulateTestDB(t, c)
+	defer func() {
+		TearDownTestDB(c)
+	}()
 
-	//actor_attributes: {type: "User",
-	qry = Search("github").Filter(
-		Filter().Terms("actor_attributes.location", "portland"),
-	)
-	out, _ = qry.Result(c)
-	expectedDocs = 10
-	expectedHits = 71
-	assert.T(t, out.Hits.Len() == expectedDocs, fmt.Sprintf("Should have %v docs got %v", expectedDocs, out.Hits.Len()))
-	assert.T(t, out.Hits.Total == expectedHits, fmt.Sprintf("Should have %v total got %v", expectedHits, out.Hits.Total))
+	Convey("Exists filter", t, func() {
+		qry := Search("oilers").Filter(
+			Filter().Exists("goals"),
+		)
+		out, err := qry.Result(c)
+		So(err, ShouldBeNil)
+		So(out, ShouldNotBeNil)
+		So(out.Hits, ShouldNotBeNil)
+		So(out.Hits.Len(), ShouldEqual, 10)
+		So(out.Hits.Total, ShouldEqual, 12)
+	})
 
-	/*
-		Should this be an AND by default?
-	*/
-	qry = Search("github").Filter(
-		Filter().Terms("actor_attributes.location", "portland"),
-		Filter().Terms("repository.has_wiki", true),
-	)
-	out, err = qry.Result(c)
-	expectedDocs = 10
-	expectedHits = 44
-	assert.T(t, err == nil, t, "should not have error")
-	assert.T(t, out.Hits.Len() == expectedDocs, fmt.Sprintf("Should have %v docs got %v", expectedDocs, out.Hits.Len()))
-	assert.T(t, out.Hits.Total == expectedHits, fmt.Sprintf("Should have %v total got %v", expectedHits, out.Hits.Total))
+	Convey("Missing filter", t, func() {
+		qry := Search("oilers").Filter(
+			Filter().Missing("goals"),
+		)
+		out, err := qry.Result(c)
+		So(err, ShouldBeNil)
+		So(out, ShouldNotBeNil)
+		So(out.Hits, ShouldNotBeNil)
+		So(out.Hits.Total, ShouldEqual, 2)
+	})
 
-	// NOW, lets try with two query calls instead of one
-	qry = Search("github").Filter(
-		Filter().Terms("actor_attributes.location", "portland"),
-	)
-	qry.Filter(
-		Filter().Terms("repository.has_wiki", true),
-	)
-	out, err = qry.Result(c)
-	//gou.Debug(out)
-	assert.T(t, err == nil, t, "should not have error")
-	assert.T(t, out.Hits.Len() == expectedDocs, fmt.Sprintf("Should have %v docs got %v", expectedDocs, out.Hits.Len()))
-	assert.T(t, out.Hits.Total == expectedHits, fmt.Sprintf("Should have %v total got %v", expectedHits, out.Hits.Total))
+	Convey("Terms filter", t, func() {
+		qry := Search("oilers").Filter(
+			Filter().Terms("pos", "rw", "lw"),
+		)
+		out, err := qry.Result(c)
+		So(err, ShouldBeNil)
+		So(out, ShouldNotBeNil)
+		So(out.Hits, ShouldNotBeNil)
+		So(out.Hits.Total, ShouldEqual, 6)
+	})
 
-	qry = Search("github").Filter(
-		"or",
-		Filter().Terms("actor_attributes.location", "portland"),
-		Filter().Terms("repository.has_wiki", true),
-	)
-	out, err = qry.Result(c)
-	expectedHits = 6676
-	assert.T(t, err == nil, t, "should not have error")
-	assert.T(t, out.Hits.Len() == expectedDocs, fmt.Sprintf("Should have %v docs got %v", expectedDocs, out.Hits.Len()))
-	assert.T(t, out.Hits.Total == expectedHits, fmt.Sprintf("Should have %v total got %v", expectedHits, out.Hits.Total))
-}
-
-func TestFilterRange(t *testing.T) {
-	c := NewTestConn()
+	Convey("Filter involving an AND", t, func() {
+		qry := Search("oilers").Filter(
+			Filter().Terms("pos", "lw"),
+			Filter().Exists("PIM"),
+		)
+		out, err := qry.Result(c)
+		So(err, ShouldBeNil)
+		So(out, ShouldNotBeNil)
+		So(out.Hits, ShouldNotBeNil)
+		So(out.Hits.Total, ShouldEqual, 2)
+	})
 
-	// now lets filter range for repositories with more than 100 forks
-	out, _ := Search("github").Size("25").Filter(
-		Range().Field("repository.forks").From("100"),
-	).Result(c)
-	if out == nil || &out.Hits == nil {
-		t.Fail()
-		return
-	}
-	expectedDocs := 25
-	expectedHits := 725
+	Convey("Filterng filter results", t, func() {
+		qry := Search("oilers").Filter(
+			Filter().Terms("pos", "lw"),
+		)
+		qry.Filter(
+			Filter().Exists("PIM"),
+		)
+		out, err := qry.Result(c)
+		So(err, ShouldBeNil)
+		So(out, ShouldNotBeNil)
+		So(out.Hits, ShouldNotBeNil)
+		So(out.Hits.Total, ShouldEqual, 2)
+	})
 
-	assert.T(t, out.Hits.Len() == expectedDocs, fmt.Sprintf("Should have %v docs got %v", expectedDocs, out.Hits.Len()))
-	assert.T(t, out.Hits.Total == expectedHits, fmt.Sprintf("Should have total %v got %v", expectedHits, out.Hits.Total))
+	Convey("Filter involving OR", t, func() {
+		qry := Search("oilers").Filter(
+			"or",
+			Filter().Terms("pos", "g"),
+			Range().Field("goals").Gt(80),
+		)
+		out, err := qry.Result(c)
+		So(err, ShouldBeNil)
+		So(out, ShouldNotBeNil)
+		So(out.Hits, ShouldNotBeNil)
+		So(out.Hits.Total, ShouldEqual, 3)
+	})
 }
diff --git a/lib/searchsearch_test.go b/lib/searchsearch_test.go
index 4c44fd9d..df8d3c53 100644
--- a/lib/searchsearch_test.go
+++ b/lib/searchsearch_test.go
@@ -12,156 +12,145 @@
 package elastigo
 
 import (
-	"fmt"
 	"github.com/araddon/gou"
-	"github.com/bmizerany/assert"
+	. "github.com/smartystreets/goconvey/convey"
 	"testing"
 )
 
-func TestSearchRequest(t *testing.T) {
-	c := NewTestConn()
-
-	qry := map[string]interface{}{
-		"query": map[string]interface{}{
-			"wildcard": map[string]string{"actor": "a*"},
-		},
-	}
-	out, err := c.Search("github", "", nil, qry)
-	//log.Println(out)
-	assert.T(t, &out != nil && err == nil, t, "Should get docs")
-	expectedDocs := 10
-	expectedHits := 621
-	assert.T(t, out.Hits.Len() == expectedDocs, t, fmt.Sprintf("Should have %v docs but was %v", expectedDocs, out.Hits.Len()))
-	assert.T(t, out.Hits.Total == expectedHits, t, fmt.Sprintf("Should have %v hits but was %v", expectedHits, out.Hits.Total))
-}
-
-func TestSearchSimple(t *testing.T) {
-	c := NewTestConn()
-
-	// searching without faceting
-	qry := Search("github").Pretty().Query(
-		Query().Search("add"),
-	)
-	out, _ := qry.Result(c)
-	// how many different docs used the word "add"
-	expectedDocs := 10
-	expectedHits := 494
-	assert.T(t, out.Hits.Len() == expectedDocs, fmt.Sprintf("Should have %v docs %v", expectedDocs, out.Hits.Len()))
-	assert.T(t, out.Hits.Total == expectedHits, fmt.Sprintf("Should have %v total= %v", expectedHits, out.Hits.Total))
+func TestSearch(t *testing.T) {
 
-	// now the same result from a "Simple" search
-	out, _ = Search("github").Search("add").Result(c)
-	assert.T(t, out.Hits.Len() == expectedDocs, fmt.Sprintf("Should have %v docs %v", expectedDocs, out.Hits.Len()))
-	assert.T(t, out.Hits.Total == expectedHits, fmt.Sprintf("Should have %v total= %v", expectedHits, out.Hits.Total))
-}
-
-func TestSearchRequestQueryString(t *testing.T) {
 	c := NewTestConn()
-
-	out, err := c.SearchUri("github", "", map[string]interface{}{"q": "actor:a*"})
-	expectedHits := 621
-	assert.T(t, &out != nil && err == nil, "Should get docs")
-	assert.T(t, out.Hits.Total == expectedHits, fmt.Sprintf("Should have %v hits but was %v", expectedHits, out.Hits.Total))
+	PopulateTestDB(t, c)
+	defer func() {
+		TearDownTestDB(c)
+	}()
+
+	Convey("Wildcard request query", t, func() {
+
+		qry := map[string]interface{}{
+			"query": map[string]interface{}{
+				"wildcard": map[string]string{"name": "*hu*"},
+			},
+		}
+		out, err := c.Search("oilers", "", nil, qry)
+
+		So(err, ShouldBeNil)
+		So(out, ShouldNotBeNil)
+		So(out.Hits, ShouldNotBeNil)
+		So(out.Hits.Total, ShouldEqual, 3)
+	})
+
+	Convey("Simple search", t, func() {
+
+		// searching without faceting
+		qry := Search("oilers").Pretty().Query(
+			Query().Search("dave"),
+		)
+
+		// how many different docs used the word "dave"
+		out, err := qry.Result(c)
+		So(err, ShouldBeNil)
+		So(out, ShouldNotBeNil)
+		So(out.Hits, ShouldNotBeNil)
+		So(out.Hits.Total, ShouldEqual, 2)
+
+		out, _ = Search("oilers").Search("dave").Result(c)
+		So(err, ShouldBeNil)
+		So(out, ShouldNotBeNil)
+		So(out.Hits, ShouldNotBeNil)
+		So(out.Hits.Total, ShouldEqual, 2)
+	})
+
+	Convey("URL Request query string", t, func() {
+
+		out, err := c.SearchUri("oilers", "", map[string]interface{}{"q": "pos:*w"})
+
+		So(err, ShouldBeNil)
+		So(out, ShouldNotBeNil)
+		So(out.Hits, ShouldNotBeNil)
+		So(out.Hits.Total, ShouldEqual, 6)
+	})
+
+	//	A faceted search for what "type" of events there are
+	//	- since we are not specifying an elasticsearch type it searches all ()
+	//
+	//	{
+	//	    "terms" : {
+	//	      "_type" : "terms",
+	//	      "missing" : 0,
+	//	      "total" : 7561,
+	//	      "other" : 0,
+	//	      "terms" : [ {
+	//	        "term" : "pushevent",
+	//	        "count" : 4185
+	//	      }, {
+	//	        "term" : "createevent",
+	//	        "count" : 786
+	//	      }.....]
+	//	    }
+	//	 }
+
+	Convey("Facet search simple", t, func() {
+
+		qry := Search("oilers").Pretty().Facet(
+			Facet().Fields("teams").Size("4"),
+		).Query(
+			Query().All(),
+		).Size("1")
+		out, err := qry.Result(c)
+		So(err, ShouldBeNil)
+		So(out, ShouldNotBeNil)
+
+		h := gou.NewJsonHelper(out.Facets)
+		So(h.Int("teams.total"), ShouldEqual, 37)
+		So(h.Int("teams.missing"), ShouldEqual, 0)
+		So(len(h.List("teams.terms")), ShouldEqual, 4)
+
+		// change the size
+		qry.FacetVal.Size("20")
+		out, err = qry.Result(c)
+		So(err, ShouldBeNil)
+		So(out, ShouldNotBeNil)
+
+		h = gou.NewJsonHelper(out.Facets)
+		So(h.Int("teams.total"), ShouldEqual, 37)
+		So(len(h.List("teams.terms")), ShouldEqual, 11)
+
+	})
+
+	Convey("Facet search with type", t, func() {
+
+		out, err := Search("oilers").Type("heyday").Pretty().Facet(
+			Facet().Fields("teams").Size("4"),
+		).Query(
+			Query().All(),
+		).Result(c)
+		So(err, ShouldBeNil)
+		So(out, ShouldNotBeNil)
+
+		h := gou.NewJsonHelper(out.Facets)
+		So(h.Int("teams.total"), ShouldEqual, 37)
+		So(len(h.List("teams.terms")), ShouldEqual, 4)
+	})
+
+	Convey("Facet search with range", t, func() {
+
+		qry := Search("oilers").Pretty().Facet(
+			Facet().Fields("teams").Size("20"),
+		).Query(
+			Query().Search("*w*"),
+		)
+		out, err := qry.Result(c)
+		So(err, ShouldBeNil)
+		So(out, ShouldNotBeNil)
+
+		h := gou.NewJsonHelper(out.Facets)
+		So(h.Int("teams.total"), ShouldEqual, 20)
+		So(len(h.List("teams.terms")), ShouldEqual, 7)
+	})
 }
 
-func TestSearchFacetOne(t *testing.T) {
-	/*
-		A faceted search for what "type" of events there are
-		- since we are not specifying an elasticsearch type it searches all ()
-
-		{
-		    "terms" : {
-		      "_type" : "terms",
-		      "missing" : 0,
-		      "total" : 7561,
-		      "other" : 0,
-		      "terms" : [ {
-		        "term" : "pushevent",
-		        "count" : 4185
-		      }, {
-		        "term" : "createevent",
-		        "count" : 786
-		      }.....]
-		    }
-		 }
-
-	*/
-	c := NewTestConn()
-
-	qry := Search("github").Pretty().Facet(
-		Facet().Fields("type").Size("25"),
-	).Query(
-		Query().All(),
-	).Size("1")
-	out, err := qry.Result(c)
-	//log.Println(string(out.Facets))
-	//gou.Debug(out)
-	assert.T(t, out != nil && err == nil, "Should have output")
-	if out == nil {
-		t.Fail()
-		return
-	}
-	h := gou.NewJsonHelper(out.Facets)
-	expectedTotal := 8084
-	expectedTerms := 16
-	assert.T(t, h.Int("type.total") == expectedTotal, fmt.Sprintf("Should have %v results %v", expectedTotal, h.Int("type.total")))
-	assert.T(t, len(h.List("type.terms")) == expectedTerms, fmt.Sprintf("Should have %v event types, %v", expectedTerms, len(h.List("type.terms"))))
-
-	// Now, lets try changing size to 10
-	qry.FacetVal.Size("10")
-	out, err = qry.Result(c)
-	h = gou.NewJsonHelper(out.Facets)
-
-	// still same doc count
-	assert.T(t, h.Int("type.total") == expectedTotal, fmt.Sprintf("Should have %v results %v", expectedTotal, h.Int("type.total")))
-	// make sure size worked
-	expectedTerms = 10
-	assert.T(t, len(h.List("type.terms")) == expectedTerms, fmt.Sprintf("Should have %v event types, got %v", expectedTerms, len(h.List("type.terms"))))
-
-	// now, lets add a type (out of the 16)
-	out, _ = Search("github").Type("IssueCommentEvent").Pretty().Facet(
-		Facet().Fields("type").Size("25"),
-	).Query(
-		Query().All(),
-	).Result(c)
-	h = gou.NewJsonHelper(out.Facets)
-	//log.Println(string(out.Facets))
-	// still same doc count
-	expectedTotal = 685
-	assert.T(t, h.Int("type.total") == expectedTotal, fmt.Sprintf("Should have %v results %v", expectedTotal, h.Int("type.total")))
-	// we should only have one facettype because we limited to one type
-	assert.T(t, len(h.List("type.terms")) == 1, fmt.Sprintf("Should have 1 event types, %v", len(h.List("type.terms"))))
-
-	// now, add a second type (chained)
-	out, _ = Search("github").Type("IssueCommentEvent").Type("PushEvent").Pretty().Facet(
-		Facet().Fields("type").Size("25"),
-	).Query(
-		Query().All(),
-	).Result(c)
-	h = gou.NewJsonHelper(out.Facets)
-	// still same doc count
-	expectedTotal = 4941
-	expectedTerms = 2
-	assert.T(t, h.Int("type.total") == expectedTotal, fmt.Sprintf("Should have %v results %v", expectedTotal, h.Int("type.total")))
-	// make sure we now have 2 types
-	assert.T(t, len(h.List("type.terms")) == expectedTerms, fmt.Sprintf("Should have %v event types, %v", expectedTerms, len(h.List("type.terms"))))
-
-	//and instead of faceting on type, facet on userid
-	// now, add a second type (chained)
-	out, _ = Search("github").Type("IssueCommentEvent,PushEvent").Pretty().Facet(
-		Facet().Fields("actor").Size("500"),
-	).Query(
-		Query().All(),
-	).Result(c)
-	h = gou.NewJsonHelper(out.Facets)
-	// still same doc count
-	expectedTotal = 5168
-	expectedTerms = 500
-	assert.T(t, h.Int("actor.total") == expectedTotal, t, fmt.Sprintf("Should have %v results %v", expectedTotal, h.Int("actor.total")))
-	// make sure size worked
-	assert.T(t, len(h.List("actor.terms")) == expectedTerms, t, fmt.Sprintf("Should have %v users, %v", expectedTerms, len(h.List("actor.terms"))))
-
-}
+/*
 
 func TestSearchFacetRange(t *testing.T) {
 	c := NewTestConn()
@@ -361,3 +350,4 @@ func TestSearchSortOrder(t *testing.T) {
 		fmt.Sprintf("Should have %v watchers, got %v", watchers, h3.Int("repository.watchers")))
 
 }
+*/
diff --git a/lib/testsetup.go b/lib/testsetup.go
new file mode 100644
index 00000000..e8cc9a89
--- /dev/null
+++ b/lib/testsetup.go
@@ -0,0 +1,70 @@
+package elastigo
+
+import (
+	"testing"
+	"time"
+)
+
+/*
+// elastigo Conn adapter to avoid a circular dependency
+type conn interface {
+	CreateIndex(name string) (interface{}, error)
+	DeleteIndex(name string) (interface{}, error)
+
+	Index(index string, _type string, id string, args map[string]interface{}, data interface{}) (interface{}, error)
+}
+*/
+
+func newIndexWorker(c *Conn, t *testing.T) func(interface{}) {
+
+	return func(d interface{}) {
+		_, err := c.Index("oilers", "heyday", "", nil, d)
+		if err != nil {
+			t.Fatalf("Index failed: %s", err)
+		}
+	}
+}
+
+func PopulateTestDB(t *testing.T, c *Conn) {
+
+	// it is not technically necessary to create an index here
+	c.CreateIndex("oilers")
+
+	idx := newIndexWorker(c, t)
+
+	idx(`{"name": "Mark Messier",   "jersey": 11, "pos": "LW", "goals": 37, "PIM": 165, 
+			"born": "19610118", "teams": ["EDM", "NYR", "VAN"]}`)
+	idx(`{"name": "Wayne Gretzky",  "jersey": 99, "pos": "C",  "goals": 87,
+			"born": "19610126", "teams": ["EDM", "NYR", "STL"]}`)
+	idx(`{"name": "Paul Coffey",    "jersey": 7,  "pos": "D",  "goals": 40,
+			"born": "19610601", "teams": ["EDM", "DET"]}`)
+	idx(`{"name": "Jari Kurri",     "jersey": 17, "pos": "RW", "goals": 52,
+			"born": "19600518", "teams": ["EDM", "VAN"]}`)
+	idx(`{"name": "Glenn Anderson", "jersey": 9,  "pos": "RW", "goals": 54,
+			"born": "19601002", "teams": ["EDM", "NYR", "TOR", "STL"]}`)
+	idx(`{"name": "Ken Linseman",   "jersey": 13, "pos": "C",  "goals": 18,
+			"born": "19580811", "teams": ["EDM", "TOR"]}`)
+	idx(`{"name": "Pat Hughes",     "jersey": 16, "pos": "RW", "goals": 27,
+			"born": "19550325", "teams": ["EDM", "MTL", "PIT"]}`)
+	idx(`{"name": "Dave Hunter",    "jersey": 12, "pos": "LW", "goals": 22,
+			"born": "19580101", "teams": ["EDM", "PIT"]}`)
+	idx(`{"name": "Kevin Lowe",     "jersey": 4,  "pos": "D",  "goals": 4,
+			"born": "19590415", "teams": ["EDM", "NYR"]}`)
+	idx(`{"name": "Charlie Huddy",  "jersey": 22, "pos": "D",  "goals": 8,
+			"born": "19590602", "teams": ["EDM", "BUF", "STL"]}`)
+	idx(`{"name": "Randy Gregg",    "jersey": 21, "pos": "D",  "goals": 13,
+			"born": "19560219", "teams": ["EDM", "VAN"]}`)
+	idx(`{"name": "Dave Semenko",   "jersey": 27, "pos": "LW", "goals": 4, "PIM": 118,
+			"born": "19570712", "teams": ["EDM"]}`)
+	idx(`{"name": "Grant Fuhr",     "jersey": 31, "pos": "G",  "GAA": 3.91,
+			"born": "19620928", "teams": ["EDM", "TOR", "BUF", "STL"]}`)
+	idx(`{"name": "Andy Moog",      "jersey": 35, "pos": "G",  "GAA": 3.77,
+			"born": "19600218", "teams": ["EDM", "BOS", "DAL", "MTL"]}`)
+
+	// HACK to let the ES magic happen
+	time.Sleep(time.Second)
+}
+
+func TearDownTestDB(c *Conn) {
+	c.DeleteIndex("oilers")
+}

From 92bbdaa2a3fcff7f6bee91b0827508b4fe93e13d Mon Sep 17 00:00:00 2001
From: Paul Oldenburg <paul.oldenburg@logrhythm.com>
Date: Thu, 27 Aug 2015 13:42:52 -0600
Subject: [PATCH 42/81] Converted test case to use mock test data

---
 lib/searchfacet_test.go | 40 ++++++++++++++++++++++------------------
 1 file changed, 22 insertions(+), 18 deletions(-)

diff --git a/lib/searchfacet_test.go b/lib/searchfacet_test.go
index c7c253e7..c860a8e4 100644
--- a/lib/searchfacet_test.go
+++ b/lib/searchfacet_test.go
@@ -12,29 +12,33 @@
 package elastigo
 
 import (
-	//"encoding/json"
-	"fmt"
 	"github.com/araddon/gou"
-	"github.com/bmizerany/assert"
+	. "github.com/smartystreets/goconvey/convey"
 	"testing"
 )
 
 func TestFacetRegex(t *testing.T) {
+
 	c := NewTestConn()
+	PopulateTestDB(t, c)
+	defer func() {
+		TearDownTestDB(c)
+	}()
+
+	Convey("Wildcard request query", t, func() {
+
+		// This is a possible solution for auto-complete
+		out, err := Search("oilers").Size("0").Facet(
+			Facet().Regex("name", "[jk].*").Size("8"),
+		).Result(c)
+		So(err, ShouldBeNil)
+		So(out, ShouldNotBeNil)
 
-	// This is a possible solution for auto-complete
-	out, _ := Search("github").Size("0").Facet(
-		Facet().Regex("repository.name", "no.*").Size("8"),
-	).Result(c)
-	if out == nil || &out.Hits == nil {
-		t.Fail()
-		return
-	}
-	//Debug(string(out.Facets))
-	fh := gou.NewJsonHelper([]byte(out.Facets))
-	facets := fh.Helpers("/repository.name/terms")
-	assert.T(t, len(facets) == 8, fmt.Sprintf("Should have 8? but was %v", len(facets)))
-	// for _, f := range facets {
-	// 	Debug(f)
-	// }
+		// Debug(string(out.Facets))
+		fh := gou.NewJsonHelper([]byte(out.Facets))
+		facets := fh.Helpers("/name/terms")
+		So(err, ShouldBeNil)
+		So(facets, ShouldNotBeNil)
+		So(len(facets), ShouldEqual, 4)
+	})
 }

From e2b3d92e834727a1eb27177cd4ee41c1acf65e84 Mon Sep 17 00:00:00 2001
From: Paul Oldenburg <paul.oldenburg@logrhythm.com>
Date: Thu, 27 Aug 2015 13:45:56 -0600
Subject: [PATCH 43/81] Comment out debug prints

---
 lib/baserequest.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/baserequest.go b/lib/baserequest.go
index 49790a81..5d3ce0b0 100644
--- a/lib/baserequest.go
+++ b/lib/baserequest.go
@@ -53,8 +53,8 @@ func (c *Conn) DoCommand(method string, url string, args map[string]interface{},
 	}
 
 	// uncomment this to print out the request that hits the wire
-	reqbuf, err := httputil.DumpRequest(req.Request, true)
-	log.Println(fmt.Sprintf("\n========= req:\nURL: %s\n%s", req.URL, bytes.NewBuffer(reqbuf).String()))
+	//reqbuf, err := httputil.DumpRequest(req.Request, true)
+	//log.Println(fmt.Sprintf("\n========= req:\nURL: %s\n%s", req.URL, bytes.NewBuffer(reqbuf).String()))
 
 	// Copy request body for tracer
 	if c.RequestTracer != nil {

From 498af6c2fcc809c6190fdd693d2bea04c0d8fba1 Mon Sep 17 00:00:00 2001
From: Paul Oldenburg <paul.oldenburg@logrhythm.com>
Date: Thu, 27 Aug 2015 13:57:32 -0600
Subject: [PATCH 44/81] Fixed broken import

---
 lib/baserequest.go | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/baserequest.go b/lib/baserequest.go
index 5d3ce0b0..ed975f4a 100644
--- a/lib/baserequest.go
+++ b/lib/baserequest.go
@@ -18,7 +18,7 @@ import (
 	"io"
 	"io/ioutil"
 	"log"
-	"net/http/httputil"
+	//"net/http/httputil"
 	"time"
 )
 
@@ -53,6 +53,7 @@ func (c *Conn) DoCommand(method string, url string, args map[string]interface{},
 	}
 
 	// uncomment this to print out the request that hits the wire
+	//   (requires net/http/httputil)
 	//reqbuf, err := httputil.DumpRequest(req.Request, true)
 	//log.Println(fmt.Sprintf("\n========= req:\nURL: %s\n%s", req.URL, bytes.NewBuffer(reqbuf).String()))
 

From 01a358c0798295d0d89202c90c22d4e83a27a891 Mon Sep 17 00:00:00 2001
From: Andrew Snodgrass <asnodgrass@anypresence.com>
Date: Thu, 27 Aug 2015 14:07:31 -0600
Subject: [PATCH 45/81] Check for nil body

---
 lib/baserequest.go | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/lib/baserequest.go b/lib/baserequest.go
index e44ee9e9..c5e0c5a3 100644
--- a/lib/baserequest.go
+++ b/lib/baserequest.go
@@ -53,13 +53,17 @@ func (c *Conn) DoCommand(method string, url string, args map[string]interface{},
 
 	// Copy request body for tracer
 	if c.RequestTracer != nil {
-		requestBody, err := ioutil.ReadAll(req.Body)
-		if err != nil {
-			return body, err
-		}
+		rbody := ""
+		if req.Body != nil {
+			requestBody, err := ioutil.ReadAll(req.Body)
+			if err != nil {
+				return body, err
+			}
 
-		req.SetBody(bytes.NewReader(requestBody))
-		c.RequestTracer(req.Method, req.URL.String(), string(requestBody))
+			req.SetBody(bytes.NewReader(requestBody))
+			rbody = string(requestBody)
+		}
+		c.RequestTracer(req.Method, req.URL.String(), rbody)
 	}
 
 	httpStatusCode, body, err = req.Do(&response)

From 4f6d6dd192acaa6f52cbfb84e78567f50d6a8ec1 Mon Sep 17 00:00:00 2001
From: Paul Oldenburg <paul.oldenburg@logrhythm.com>
Date: Thu, 27 Aug 2015 14:14:20 -0600
Subject: [PATCH 46/81] Removed test logging and commented code

---
 lib/coretest_test.go |  2 --
 lib/request.go       | 13 -------------
 2 files changed, 15 deletions(-)

diff --git a/lib/coretest_test.go b/lib/coretest_test.go
index 33090cf0..6d2acf72 100644
--- a/lib/coretest_test.go
+++ b/lib/coretest_test.go
@@ -116,8 +116,6 @@ func LoadTestData() {
 	c := NewConn()
 	c.Domain = *eshost
 
-	fmt.Println("Loading test data")
-
 	c.DeleteIndex(testIndex)
 
 	docCt := 0
diff --git a/lib/request.go b/lib/request.go
index 56ad377a..e42e13b3 100644
--- a/lib/request.go
+++ b/lib/request.go
@@ -55,19 +55,6 @@ func (r *Request) SetBody(body io.Reader) {
 	}
 	r.Body = rc
 	r.ContentLength = -1
-	/*
-		if body != nil {
-			switch v := body.(type) {
-			case *strings.Reader:
-				r.ContentLength = int64(v.Len())
-			case *bytes.Reader:
-				r.ContentLength = int64(v.Len())
-			case *bytes.Buffer:
-				r.ContentLength = int64(v.Len())
-			}
-		}
-		fmt.Printf("SetBody() - ContentLength: %d  body.(type): %T\n", r.ContentLength)
-	*/
 }
 
 func (r *Request) Do(v interface{}) (int, []byte, error) {

From 559349d2b231bfc8e93b3e09ac55bcf697e36ef3 Mon Sep 17 00:00:00 2001
From: Paul Oldenburg <paul.oldenburg@logrhythm.com>
Date: Thu, 27 Aug 2015 14:16:37 -0600
Subject: [PATCH 47/81] Changed test name

---
 lib/searchfacet_test.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/searchfacet_test.go b/lib/searchfacet_test.go
index c860a8e4..e7001431 100644
--- a/lib/searchfacet_test.go
+++ b/lib/searchfacet_test.go
@@ -25,7 +25,7 @@ func TestFacetRegex(t *testing.T) {
 		TearDownTestDB(c)
 	}()
 
-	Convey("Wildcard request query", t, func() {
+	Convey("Facted regex query", t, func() {
 
 		// This is a possible solution for auto-complete
 		out, err := Search("oilers").Size("0").Facet(

From b1a40030b3fc44b6085c15c007547f51045fbd10 Mon Sep 17 00:00:00 2001
From: Paul Oldenburg <paul.oldenburg@logrhythm.com>
Date: Thu, 27 Aug 2015 16:07:13 -0600
Subject: [PATCH 48/81] Converted more test cases and altered the mock data
 mapping

---
 lib/searchsearch_test.go | 313 ++++++++++++++++-----------------------
 lib/testsetup.go         |  35 +++--
 2 files changed, 147 insertions(+), 201 deletions(-)

diff --git a/lib/searchsearch_test.go b/lib/searchsearch_test.go
index df8d3c53..688c9ccf 100644
--- a/lib/searchsearch_test.go
+++ b/lib/searchsearch_test.go
@@ -22,7 +22,7 @@ func TestSearch(t *testing.T) {
 	c := NewTestConn()
 	PopulateTestDB(t, c)
 	defer func() {
-		TearDownTestDB(c)
+		//TearDownTestDB(c)
 	}()
 
 	Convey("Wildcard request query", t, func() {
@@ -133,7 +133,7 @@ func TestSearch(t *testing.T) {
 		So(len(h.List("teams.terms")), ShouldEqual, 4)
 	})
 
-	Convey("Facet search with range", t, func() {
+	Convey("Facet search with wildcard", t, func() {
 
 		qry := Search("oilers").Pretty().Facet(
 			Facet().Fields("teams").Size("20"),
@@ -148,206 +148,145 @@ func TestSearch(t *testing.T) {
 		So(h.Int("teams.total"), ShouldEqual, 20)
 		So(len(h.List("teams.terms")), ShouldEqual, 7)
 	})
-}
 
-/*
+	Convey("Facet search with range", t, func() {
 
-func TestSearchFacetRange(t *testing.T) {
-	c := NewTestConn()
+		qry := Search("oilers").Pretty().Facet(
+			Facet().Fields("teams").Size("20"),
+		).Query(
+			Query().Range(
+				Range().Field("dob").From("19600101").To("19621231"),
+			).Search("*w*"),
+		)
+		out, err := qry.Result(c)
+		So(err, ShouldBeNil)
+		So(out, ShouldNotBeNil)
 
-	// ok, now lets try facet but on actor field with a range
-	qry := Search("github").Pretty().Facet(
-		Facet().Fields("actor").Size("500"),
-	).Query(
-		Query().Search("add"),
-	)
-	out, err := qry.Result(c)
-	assert.T(t, out != nil && err == nil, t, "Should have output")
-
-	if out == nil {
-		t.Fail()
-		return
-	}
-	//log.Println(string(out.Facets))
-	h := gou.NewJsonHelper(out.Facets)
-	expectedActorTotal := 521
-	// how many different docs used the word "add", during entire time range
-	assert.T(t, h.Int("actor.total") == expectedActorTotal, fmt.Sprintf("Should have %v results %v", expectedActorTotal, h.Int("actor.total")))
-	// make sure size worked
-	expectedTerms := 366
-	assert.T(t, len(h.List("actor.terms")) == expectedTerms,
-		fmt.Sprintf("Should have %v unique userids, %v", expectedTerms, len(h.List("actor.terms"))))
-
-	// ok, repeat but with a range showing different results
-	qry = Search("github").Pretty().Facet(
-		Facet().Fields("actor").Size("500"),
-	).Query(
-		Query().Range(
-			Range().Field("created_at").From("2012-12-10T15:00:00-08:00").To("2012-12-10T15:10:00-08:00"),
-		).Search("add"),
-	)
-	out, err = qry.Result(c)
-	assert.T(t, out != nil && err == nil, t, "Should have output")
-
-	if out == nil {
-		t.Fail()
-		return
-	}
-	//log.Println(string(out.Facets))
-	h = gou.NewJsonHelper(out.Facets)
-	// how many different events used the word "add", during time range?
-	expectedActorTotal = 97
-	expectedTerms = 71
-	assert.T(t, h.Int("actor.total") == expectedActorTotal, fmt.Sprintf("Should have %v results %v", expectedActorTotal, h.Int("actor.total")))
-	// make sure size worked
-	assert.T(t, len(h.List("actor.terms")) == expectedTerms,
-		fmt.Sprintf("Should have %v event types, %v", expectedTerms, len(h.List("actor.terms"))))
+		h := gou.NewJsonHelper(out.Facets)
+		So(h.Int("teams.total"), ShouldEqual, 12)
+		So(len(h.List("teams.terms")), ShouldEqual, 5)
+	})
 
-}
+	Convey("Search query with terms", t, func() {
 
-func TestSearchTerm(t *testing.T) {
-	c := NewTestConn()
+		qry := Search("oilers").Query(
+			Query().Term("teams", "nyr"),
+		)
+		out, err := qry.Result(c)
+		So(err, ShouldBeNil)
+		So(out, ShouldNotBeNil)
+		So(out.Hits.Len(), ShouldEqual, 4)
+		So(out.Hits.Total, ShouldEqual, 4)
+	})
 
-	// ok, now lets try searching with term query (specific field/term)
-	qry := Search("github").Query(
-		Query().Term("repository.name", "jasmine"),
-	)
-	out, _ := qry.Result(c)
-	// how many different docs have jasmine in repository.name?
-	expectedDocs := 4
-	expectedHits := 4
-	assert.T(t, out.Hits.Len() == expectedDocs, fmt.Sprintf("Should have %v docs %v", expectedDocs, out.Hits.Len()))
-	assert.T(t, out.Hits.Total == expectedHits, fmt.Sprintf("Should have %v total= %v", expectedHits, out.Hits.Total))
-}
+	Convey("Search query with fields", t, func() {
 
-func TestSearchFields(t *testing.T) {
-	c := NewTestConn()
+		qry := Search("oilers").Query(
+			Query().Fields("teams", "nyr", "", ""),
+		)
+		out, err := qry.Result(c)
+		So(err, ShouldBeNil)
+		So(out, ShouldNotBeNil)
+		So(out.Hits.Len(), ShouldEqual, 4)
+		So(out.Hits.Total, ShouldEqual, 4)
+	})
 
-	// same as terms, search using fields:
-	//    how many different docs have jasmine in repository.name?
-	qry := Search("github").Query(
-		Query().Fields("repository.name", "jasmine", "", ""),
-	)
-	out, _ := qry.Result(c)
-	expectedDocs := 4
-	expectedHits := 4
-	assert.T(t, out.Hits.Len() == expectedDocs, fmt.Sprintf("Should have %v docs %v", expectedDocs, out.Hits.Len()))
-	assert.T(t, out.Hits.Total == expectedHits, fmt.Sprintf("Should have %v total= %v", expectedHits, out.Hits.Total))
-}
+	Convey("Search query with fields exist and missing", t, func() {
 
-func TestSearchMissingExists(t *testing.T) {
-	c := NewTestConn()
+		qry := Search("oilers").Filter(
+			Filter().Exists("PIM"),
+		)
+		out, err := qry.Result(c)
+		So(err, ShouldBeNil)
+		So(out, ShouldNotBeNil)
+		So(out.Hits.Len(), ShouldEqual, 2)
+		So(out.Hits.Total, ShouldEqual, 2)
 
-	// search for docs that are missing repository.name
-	qry := Search("github").Filter(
-		Filter().Exists("repository.name"),
-	)
-	out, _ := qry.Result(c)
-	expectedDocs := 10
-	expectedTotal := 7695
-	assert.T(t, out.Hits.Len() == expectedDocs, fmt.Sprintf("Should have %v docs %v", expectedDocs, out.Hits.Len()))
-	assert.T(t, out.Hits.Total == expectedTotal, fmt.Sprintf("Should have %v total= %v", expectedTotal, out.Hits.Total))
-
-	qry = Search("github").Filter(
-		Filter().Missing("repository.name"),
-	)
-	out, _ = qry.Result(c)
-	expectedDocs = 10
-	expectedTotal = 390
-	assert.T(t, out.Hits.Len() == expectedDocs, fmt.Sprintf("Should have %v docs %v", expectedDocs, out.Hits.Len()))
-	assert.T(t, out.Hits.Total == expectedTotal, fmt.Sprintf("Should have %v total= %v", expectedTotal, out.Hits.Total))
-}
+		qry = Search("oilers").Filter(
+			Filter().Missing("PIM"),
+		)
+		out, err = qry.Result(c)
+		So(err, ShouldBeNil)
+		So(out, ShouldNotBeNil)
+		So(out.Hits.Len(), ShouldEqual, 10)
+		So(out.Hits.Total, ShouldEqual, 12)
+	})
 
-func TestSearchFilterQuery(t *testing.T) {
-	c := NewTestConn()
+	Convey("Search with query and filter", t, func() {
 
-	// compound query + filter with query being wildcard
-	out, _ := Search("github").Size("25").Query(
-		Query().Fields("repository.name", "jas*", "", ""),
-	).Filter(
-		Filter().Terms("repository.has_wiki", true),
-	).Result(c)
-	if out == nil || &out.Hits == nil {
-		t.Fail()
-		return
-	}
-
-	expectedDocs := 7
-	expectedTotal := 7
-	assert.T(t, out.Hits.Len() == expectedDocs, fmt.Sprintf("Should have %v docs %v", expectedDocs, out.Hits.Len()))
-	assert.T(t, out.Hits.Total == expectedTotal, fmt.Sprintf("Should have %v total= %v", expectedTotal, out.Hits.Total))
-}
+		out, err := Search("oilers").Size("25").Query(
+			Query().Fields("name", "*d*", "", ""),
+		).Filter(
+			Filter().Terms("teams", "stl"),
+		).Result(c)
+		So(err, ShouldBeNil)
+		So(out, ShouldNotBeNil)
+		So(out.Hits.Len(), ShouldEqual, 2)
+		So(out.Hits.Total, ShouldEqual, 2)
+	})
 
-func TestSearchRange(t *testing.T) {
-	c := NewTestConn()
+	Convey("Search with range", t, func() {
 
-	// now lets filter by a subset of the total time
-	out, _ := Search("github").Size("25").Query(
-		Query().Range(
-			Range().Field("created_at").From("2012-12-10T15:00:00-08:00").To("2012-12-10T15:10:00-08:00"),
-		).Search("add"),
-	).Result(c)
-	assert.T(t, out != nil && &out.Hits != nil, "Must not have nil results, or hits")
-	assert.T(t, out.Hits.Len() == 25, fmt.Sprintf("Should have 25 docs %v", out.Hits.Len()))
-	assert.T(t, out.Hits.Total == 92, fmt.Sprintf("Should have total=92 but was %v", out.Hits.Total))
-}
+		out, err := Search("oilers").Size("25").Query(
+			Query().Range(
+				Range().Field("dob").From("19600101").To("19621231"),
+			).Search("*w*"),
+		).Result(c)
+		So(err, ShouldBeNil)
+		So(out, ShouldNotBeNil)
+		So(out.Hits.Len(), ShouldEqual, 4)
+		So(out.Hits.Total, ShouldEqual, 4)
+	})
 
-func TestSearchSortOrder(t *testing.T) {
-	c := NewTestConn()
+	Convey("Search with sorting desc", t, func() {
+
+		out, err := Search("oilers").Pretty().Query(
+			Query().All(),
+		).Sort(
+			Sort("dob").Desc(),
+		).Result(c)
+		So(err, ShouldBeNil)
+		So(out, ShouldNotBeNil)
+		So(out.Hits.Len(), ShouldEqual, 10)
+		So(out.Hits.Total, ShouldEqual, 14)
+
+		b, err := out.Hits.Hits[0].Source.MarshalJSON()
+		h1 := gou.NewJsonHelper(b)
+		So(h1.String("name"), ShouldEqual, "Grant Fuhr")
+	})
+
+	Convey("Search with sorting asc", t, func() {
+
+		out, err := Search("oilers").Pretty().Query(
+			Query().All(),
+		).Sort(
+			Sort("dob"),
+		).Result(c)
+		So(err, ShouldBeNil)
+		So(out, ShouldNotBeNil)
+		So(out.Hits.Len(), ShouldEqual, 10)
+		So(out.Hits.Total, ShouldEqual, 14)
 
-	// ok, now lets try sorting by repository watchers descending
-	qry := Search("github").Pretty().Query(
-		Query().All(),
-	).Sort(
-		Sort("repository.watchers").Desc(),
-	)
-	out, _ := qry.Result(c)
-
-	// how many different docs used the word "add", during entire time range
-	expectedDocs := 10
-	expectedTotal := 8085
-	assert.T(t, out.Hits.Len() == expectedDocs, fmt.Sprintf("Should have %v docs %v", expectedDocs, out.Hits.Len()))
-	assert.T(t, out.Hits.Total == expectedTotal, fmt.Sprintf("Should have %v total= %v", expectedTotal, out.Hits.Total))
-	b, err := out.Hits.Hits[0].Source.MarshalJSON()
-	assert.T(t, err == nil, fmt.Sprintf("Should not have returned an error: %v", err))
-	h1 := gou.NewJsonHelper(b)
-	assert.T(t, h1.Int("repository.watchers") == 41377,
-		fmt.Sprintf("Should have 41377 watchers= %v", h1.Int("repository.watchers")))
-
-	// ascending
-	out, _ = Search("github").Pretty().Query(
-		Query().All(),
-	).Sort(
-		Sort("repository.watchers"),
-	).Result(c)
-	// how many different docs used the word "add", during entire time range
-	assert.T(t, out.Hits.Len() == expectedDocs, fmt.Sprintf("Should have %v docs %v", expectedDocs, out.Hits.Len()))
-	assert.T(t, out.Hits.Total == expectedTotal, fmt.Sprintf("Should have %v total got %v", expectedTotal, out.Hits.Total))
-	b, err = out.Hits.Hits[0].Source.MarshalJSON()
-	assert.T(t, err == nil, fmt.Sprintf("Should not have returned an error: %v", err))
-	h2 := gou.NewJsonHelper(b)
-	assert.T(t, h2.Int("repository.watchers") == 0,
-		fmt.Sprintf("Should have 0 watchers= %v", h2.Int("repository.watchers")))
-
-	// sort descending with search
-	out, _ = Search("github").Pretty().Size("5").Query(
-		Query().Search("python"),
-	).Sort(
-		Sort("repository.watchers").Desc(),
-	).Result(c)
-	// how many different docs used the word "add", during entire time range
-	expectedDocs = 5
-	expectedTotal = 734
-
-	assert.T(t, out.Hits.Len() == expectedDocs, fmt.Sprintf("Should have %v docs %v", expectedDocs, out.Hits.Len()))
-	assert.T(t, out.Hits.Total == expectedTotal, fmt.Sprintf("Should have %v total got %v", expectedTotal, out.Hits.Total))
-
-	b, err = out.Hits.Hits[0].Source.MarshalJSON()
-	assert.T(t, err == nil, fmt.Sprintf("Should not have returned an error: %v", err))
-	h3 := gou.NewJsonHelper(b)
-	watchers := 8659
-	assert.T(t, h3.Int("repository.watchers") == watchers,
-		fmt.Sprintf("Should have %v watchers, got %v", watchers, h3.Int("repository.watchers")))
+		b, err := out.Hits.Hits[0].Source.MarshalJSON()
+		h1 := gou.NewJsonHelper(b)
+		So(h1.String("name"), ShouldEqual, "Pat Hughes")
+	})
+
+	Convey("Search with sorting desc with query", t, func() {
+
+		out, err := Search("oilers").Pretty().Query(
+			Query().Search("*w*"),
+		).Sort(
+			Sort("dob").Desc(),
+		).Result(c)
+		So(err, ShouldBeNil)
+		So(out, ShouldNotBeNil)
+		So(out.Hits.Len(), ShouldEqual, 8)
+		So(out.Hits.Total, ShouldEqual, 8)
+
+		b, err := out.Hits.Hits[0].Source.MarshalJSON()
+		h1 := gou.NewJsonHelper(b)
+		So(h1.String("name"), ShouldEqual, "Wayne Gretzky")
+	})
 
 }
-*/
diff --git a/lib/testsetup.go b/lib/testsetup.go
index e8cc9a89..19f83c8a 100644
--- a/lib/testsetup.go
+++ b/lib/testsetup.go
@@ -30,36 +30,43 @@ func PopulateTestDB(t *testing.T, c *Conn) {
 	// it is not technically necessary to create an index here
 	c.CreateIndex("oilers")
 
+	// set the mapping for dob to be a date so it can be used for range searches
+	_, err := c.DoCommand("PUT", "/oilers/heyday/_mapping?ignore_conflicts", nil,
+		string(`{"heyday": {"properties": {"dob": {"type": "date"}}}}`))
+	if err != nil {
+		t.Fatal("Error setting dob mapping")
+	}
+
 	idx := newIndexWorker(c, t)
 
 	idx(`{"name": "Mark Messier",   "jersey": 11, "pos": "LW", "goals": 37, "PIM": 165, 
-			"born": "19610118", "teams": ["EDM", "NYR", "VAN"]}`)
+			"dob": "19610118", "teams": ["EDM", "NYR", "VAN"]}`)
 	idx(`{"name": "Wayne Gretzky",  "jersey": 99, "pos": "C",  "goals": 87,
-			"born": "19610126", "teams": ["EDM", "NYR", "STL"]}`)
+			"dob": "19610126", "teams": ["EDM", "NYR", "STL"]}`)
 	idx(`{"name": "Paul Coffey",    "jersey": 7,  "pos": "D",  "goals": 40,
-			"born": "19610601", "teams": ["EDM", "DET"]}`)
+			"dob": "19610601", "teams": ["EDM", "DET"]}`)
 	idx(`{"name": "Jari Kurri",     "jersey": 17, "pos": "RW", "goals": 52,
-			"born": "19600518", "teams": ["EDM", "VAN"]}`)
+			"dob": "19600518", "teams": ["EDM", "VAN"]}`)
 	idx(`{"name": "Glenn Anderson", "jersey": 9,  "pos": "RW", "goals": 54,
-			"born": "19601002", "teams": ["EDM", "NYR", "TOR", "STL"]}`)
+			"dob": "19601002", "teams": ["EDM", "NYR", "TOR", "STL"]}`)
 	idx(`{"name": "Ken Linseman",   "jersey": 13, "pos": "C",  "goals": 18,
-			"born": "19580811", "teams": ["EDM", "TOR"]}`)
+			"dob": "19580811", "teams": ["EDM", "TOR"]}`)
 	idx(`{"name": "Pat Hughes",     "jersey": 16, "pos": "RW", "goals": 27,
-			"born": "19550325", "teams": ["EDM", "MTL", "PIT"]}`)
+			"dob": "19550325", "teams": ["EDM", "MTL", "PIT"]}`)
 	idx(`{"name": "Dave Hunter",    "jersey": 12, "pos": "LW", "goals": 22,
-			"born": "19580101", "teams": ["EDM", "PIT"]}`)
+			"dob": "19580101", "teams": ["EDM", "PIT"]}`)
 	idx(`{"name": "Kevin Lowe",     "jersey": 4,  "pos": "D",  "goals": 4,
-			"born": "19590415", "teams": ["EDM", "NYR"]}`)
+			"dob": "19590415", "teams": ["EDM", "NYR"]}`)
 	idx(`{"name": "Charlie Huddy",  "jersey": 22, "pos": "D",  "goals": 8,
-			"born": "19590602", "teams": ["EDM", "BUF", "STL"]}`)
+			"dob": "19590602", "teams": ["EDM", "BUF", "STL"]}`)
 	idx(`{"name": "Randy Gregg",    "jersey": 21, "pos": "D",  "goals": 13,
-			"born": "19560219", "teams": ["EDM", "VAN"]}`)
+			"dob": "19560219", "teams": ["EDM", "VAN"]}`)
 	idx(`{"name": "Dave Semenko",   "jersey": 27, "pos": "LW", "goals": 4, "PIM": 118,
-			"born": "19570712", "teams": ["EDM"]}`)
+			"dob": "19570712", "teams": ["EDM"]}`)
 	idx(`{"name": "Grant Fuhr",     "jersey": 31, "pos": "G",  "GAA": 3.91,
-			"born": "19620928", "teams": ["EDM", "TOR", "BUF", "STL"]}`)
+			"dob": "19620928", "teams": ["EDM", "TOR", "BUF", "STL"]}`)
 	idx(`{"name": "Andy Moog",      "jersey": 35, "pos": "G",  "GAA": 3.77,
-			"born": "19600218", "teams": ["EDM", "BOS", "DAL", "MTL"]}`)
+			"dob": "19600218", "teams": ["EDM", "BOS", "DAL", "MTL"]}`)
 
 	// HACK to let the ES magic happen
 	time.Sleep(time.Second)

From 3e59754ba024bdf613756930b071fda08d27e53e Mon Sep 17 00:00:00 2001
From: Paul Oldenburg <paul.oldenburg@logrhythm.com>
Date: Mon, 31 Aug 2015 14:34:52 -0600
Subject: [PATCH 49/81] Adding cat node functionality for reporting stats for
 cluster nodes

---
 lib/catnodeinfo.go      | 243 ++++++++++++++++++++++++++++++++++++++++
 lib/catnodeinfo_test.go |  86 ++++++++++++++
 lib/catresponses.go     |  73 ++++++++++++
 3 files changed, 402 insertions(+)
 create mode 100644 lib/catnodeinfo.go
 create mode 100644 lib/catnodeinfo_test.go

diff --git a/lib/catnodeinfo.go b/lib/catnodeinfo.go
new file mode 100644
index 00000000..f9c95c98
--- /dev/null
+++ b/lib/catnodeinfo.go
@@ -0,0 +1,243 @@
+package elastigo
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+)
+
+// newCatNodeInfo returns an instance of CatNodeInfo populated with the
+// the information in the cat output indexLine which contains the
+// specified fields. An err is returned if a field is not known.
+func newCatNodeInfo(fields []string, indexLine string) (catNode *CatNodeInfo, err error) {
+
+	split := strings.Fields(indexLine)
+	catNode = &CatNodeInfo{}
+
+	// Check the fields length compared to the number of stats
+	lf, ls := len(fields), len(split)
+	if lf > ls {
+		return nil, fmt.Errorf("Number of fields (%d) greater than number of stats (%d)", lf, ls)
+	}
+
+	// Populate the apropriate field in CatNodeInfo
+	for i, field := range fields {
+		switch field {
+		case "id", "nodeId":
+			catNode.Id = split[i]
+		case "pid", "p":
+			i, _ := strconv.Atoi(split[i])
+			catNode.PID = int32(i)
+		case "host", "h":
+			catNode.Host = split[i]
+		case "ip", "i":
+			catNode.IP = split[i]
+		case "port", "po":
+			i, _ := strconv.Atoi(split[i])
+			catNode.Port = int16(i)
+		case "version", "v":
+			catNode.Version = split[i]
+		case "build", "b":
+			catNode.Build = split[i]
+		case "jdk", "j":
+			catNode.JDK = split[i]
+		case "disk.avail", "d", "disk", "diskAvail":
+			catNode.DiskAvail = split[i]
+		case "heap.current", "hc", "heapCurrent":
+			catNode.HeapCur = split[i]
+		case "heap.percent", "hp", "heapPercent":
+			i, _ := strconv.Atoi(split[i])
+			catNode.HeapPerc = int16(i)
+		case "heap.max", "hm", "heapMax":
+			catNode.HeapMax = split[i]
+		case "ram.current", "rc", "ramCurrent":
+			catNode.RamCur = split[i]
+		case "ram.percent", "rp", "ramPercent":
+			i, _ := strconv.Atoi(split[i])
+			catNode.RamPerc = int16(i)
+		case "ram.max", "rm", "ramMax":
+			catNode.RamMax = split[i]
+		case "file_desc.current", "fdc", "fileDescriptorCurrent":
+			catNode.FileDescCur = split[i]
+		case "file_desc.percent", "fdp", "fileDescriptorPercent":
+			i, _ := strconv.Atoi(split[i])
+			catNode.FileDescPerc = int16(i)
+		case "file_desc.max", "fdm", "fileDescriptorMax":
+			catNode.FileDescMax = split[i]
+		case "load", "l":
+			catNode.Load = split[i]
+		case "uptime", "u":
+			catNode.UpTime = split[i]
+		case "node.role", "r", "role", "dc", "nodeRole":
+			catNode.NodeRole = split[i]
+		case "master", "m":
+			catNode.Master = split[i]
+		case "name", "n":
+			catNode.Name = split[i]
+		case "completion.size", "cs", "completionSize":
+			catNode.CmpltSize = split[i]
+		case "fielddata.memory_size", "fm", "fielddataMemory":
+			catNode.FieldMem = split[i]
+		case "fielddata.evictions", "fe", "fieldataEvictions":
+			i, _ := strconv.Atoi(split[i])
+			catNode.FieldEvict = int32(i)
+		case "filter_cache.memory_size", "fcm", "filterCacheMemory":
+			catNode.FiltMem = split[i]
+		case "filter_cache.evictions", "fce", "filterCacheEvictions":
+			i, _ := strconv.Atoi(split[i])
+			catNode.FiltEvict = int32(i)
+		case "flush.total", "ft", "flushTotal":
+			i, _ := strconv.Atoi(split[i])
+			catNode.FlushTotal = int32(i)
+		case "flush.total_time", "ftt", "flushTotalTime":
+			i, _ := strconv.Atoi(split[i])
+			catNode.FlushTotalTime = int32(i)
+		case "get.current", "gc", "getCurrent":
+			i, _ := strconv.Atoi(split[i])
+			catNode.GetCur = int32(i)
+		case "get.time", "gti", "getTime":
+			catNode.GetTime = split[i]
+		case "get.total", "gto", "getTotal":
+			i, _ := strconv.Atoi(split[i])
+			catNode.GetTotal = int32(i)
+		case "get.exists_time", "geti", "getExistsTime":
+			catNode.GetExistsTime = split[i]
+		case "get.exists_total", "geto", "getExistsTotal":
+			i, _ := strconv.Atoi(split[i])
+			catNode.GetExistsTotal = int32(i)
+		case "get.missing_time", "gmti", "getMissingTime":
+			catNode.GetMissingTime = split[i]
+		case "get.missing_total", "gmto", "getMissingTotal":
+			i, _ := strconv.Atoi(split[i])
+			catNode.GetMissingTotal = int32(i)
+		case "id_cache.memory_size", "im", "idCacheMemory":
+			catNode.IDCacheMemory = split[i]
+		case "indexing.delete_current", "idc", "indexingDeleteCurrent":
+			i, _ := strconv.Atoi(split[i])
+			catNode.IdxDelCur = int32(i)
+		case "indexing.delete_time", "idti", "indexingDeleteime":
+			catNode.IdxDelTime = split[i]
+		case "indexing.delete_total", "idto", "indexingDeleteTotal":
+			i, _ := strconv.Atoi(split[i])
+			catNode.IdxDelTotal = int32(i)
+		case "indexing.index_current", "iic", "indexingIndexCurrent":
+			i, _ := strconv.Atoi(split[i])
+			catNode.IdxIdxCur = int32(i)
+		case "indexing.index_time", "iiti", "indexingIndexTime":
+			catNode.IdxIdxTime = split[i]
+		case "indexing.index_total", "iito", "indexingIndexTotal":
+			i, _ := strconv.Atoi(split[i])
+			catNode.IdxIdxTotal = int32(i)
+		case "merges.current", "mc", "mergesCurrent":
+			i, _ := strconv.Atoi(split[i])
+			catNode.MergCur = int32(i)
+		case "merges.current_docs", "mcd", "mergesCurrentDocs":
+			i, _ := strconv.Atoi(split[i])
+			catNode.MergCurDocs = int32(i)
+		case "merges.current_size", "mcs", "mergesCurrentSize":
+			catNode.MergCurSize = split[i]
+		case "merges.total", "mt", "mergesTotal":
+			i, _ := strconv.Atoi(split[i])
+			catNode.MergTotal = int32(i)
+		case "merges.total_docs", "mtd", "mergesTotalDocs":
+			i, _ := strconv.Atoi(split[i])
+			catNode.MergTotalDocs = int32(i)
+		case "merges.total_size", "mts", "mergesTotalSize":
+			catNode.MergTotalSize = split[i]
+		case "merges.total_time", "mtt", "mergesTotalTime":
+			catNode.MergTotalTime = split[i]
+		case "percolate.current", "pc", "percolateCurrent":
+			i, _ := strconv.Atoi(split[i])
+			catNode.PercCur = int32(i)
+		case "percolate.memory_size", "pm", "percolateMemory":
+			catNode.PercMem = split[i]
+		case "percolate.queries", "pq", "percolateQueries":
+			i, _ := strconv.Atoi(split[i])
+			catNode.PercQueries = int32(i)
+		case "percolate.time", "pti", "percolateTime":
+			catNode.PercTime = split[i]
+		case "percolate.total", "pto", "percolateTotal":
+			i, _ := strconv.Atoi(split[i])
+			catNode.PercTotal = int32(i)
+		case "refesh.total", "rto", "refreshTotal":
+			i, _ := strconv.Atoi(split[i])
+			catNode.RefreshTotal = int32(i)
+		case "refresh.time", "rti", "refreshTime":
+			catNode.RefreshTime = split[i]
+		case "search.fetch_current", "sfc", "searchFetchCurrent":
+			i, _ := strconv.Atoi(split[i])
+			catNode.SearchFetchCur = int32(i)
+		case "search.fetch_time", "sfti", "searchFetchTime":
+			catNode.SearchFetchTime = split[i]
+		case "search.fetch_total", "sfto", "searchFetchTotal":
+			i, _ := strconv.Atoi(split[i])
+			catNode.SearchFetchTotal = int32(i)
+		case "search.open_contexts", "so", "searchOpenContexts":
+			i, _ := strconv.Atoi(split[i])
+			catNode.SearchOpenContexts = int32(i)
+		case "search.query_current", "sqc", "searchQueryCurrent":
+			i, _ := strconv.Atoi(split[i])
+			catNode.SearchQueryCur = int32(i)
+		case "search.query_time", "sqti", "searchQueryTime":
+			catNode.SearchQueryTime = split[i]
+		case "search.query_total", "sqto", "searchQueryTotal":
+			i, _ := strconv.Atoi(split[i])
+			catNode.SearchQueryTotal = int32(i)
+		case "segments.count", "sc", "segmentsCount":
+			i, _ := strconv.Atoi(split[i])
+			catNode.SegCount = int32(i)
+		case "segments.memory", "sm", "segmentsMemory":
+			catNode.SegMem = split[i]
+		case "segments.index_writer_memory", "siwm", "segmentsIndexWriterMemory":
+			catNode.SegIdxWriterMem = split[i]
+		case "segments.index_writer_max_memory", "siwmx", "segmentsIndexWriterMaxMemory":
+			catNode.SegIdxWriterMax = split[i]
+		case "segments.version_map_memory", "svmm", "segmentsVersionMapMemory":
+			catNode.SegVerMapMem = split[i]
+		default:
+			return nil, fmt.Errorf("Invalid cat nodes field: %s", field)
+		}
+	}
+
+	return catNode, nil
+}
+
+// GetCatNodeInfo issues an elasticsearch cat nodes request with the specified
+// fields and returns a list of CatNodeInfos, one for each node, whose requested
+// members are populated with statistics. If fields is nil or empty, the default
+// cat output is used.
+func (c *Conn) GetCatNodeInfo(fields []string) (catNodes []CatNodeInfo, err error) {
+	catNodes = make([]CatNodeInfo, 0)
+
+	// Issue a request for stats on the requested fields
+	var args map[string]interface{}
+	if len(fields) > 0 {
+		args = map[string]interface{}{"h": strings.Join(fields, ",")}
+	} else {
+		fields = []string{"host", "ip", "heap.percent", "ram.percent", "load",
+			"node.role", "master", "name"}
+	}
+	indices, err := c.DoCommand("GET", "/_cat/nodes/", args, nil)
+	if err != nil {
+		return catNodes, err
+	}
+
+	// Create a CatIndexInfo for each line in the response
+	indexLines := strings.Split(string(indices[:]), "\n")
+	for _, index := range indexLines {
+
+		// Ignore empty output lines
+		if len(index) < 1 {
+			continue
+		}
+
+		// Create a CatNodeInfo and append it to the result
+		ci, err := newCatNodeInfo(fields, index)
+		if ci != nil {
+			catNodes = append(catNodes, *ci)
+		} else if err != nil {
+			return catNodes, err
+		}
+	}
+	return catNodes, nil
+}
diff --git a/lib/catnodeinfo_test.go b/lib/catnodeinfo_test.go
new file mode 100644
index 00000000..225602bc
--- /dev/null
+++ b/lib/catnodeinfo_test.go
@@ -0,0 +1,86 @@
+package elastigo
+
+import (
+	. "github.com/smartystreets/goconvey/convey"
+	"testing"
+)
+
+func TestCatNode(t *testing.T) {
+
+	c := NewTestConn()
+
+	// Typical cat nodes call
+	Convey("Basic cat nodes", t, func() {
+
+		fields := []string{"fm", "fe", "fcm", "fce", "ft", "ftt", "im", "rp"}
+		catNodes, err := c.GetCatNodeInfo(fields)
+
+		So(err, ShouldBeNil)
+		So(catNodes, ShouldNotBeNil)
+		So(len(catNodes), ShouldBeGreaterThan, 0)
+
+		for _, catNode := range catNodes {
+			So(catNode.FieldMem, ShouldNotBeEmpty)
+			So(catNode.FiltMem, ShouldNotBeEmpty)
+			So(catNode.IDCacheMemory, ShouldNotBeEmpty)
+			So(catNode.RamPerc, ShouldNotBeEmpty)
+		}
+	})
+
+	// Cat nodes call with default arguments
+	Convey("Cat nodes with default arguments", t, func() {
+
+		fields := []string{}
+		catNodes, err := c.GetCatNodeInfo(fields)
+
+		So(err, ShouldBeNil)
+		So(catNodes, ShouldNotBeNil)
+		So(len(catNodes), ShouldBeGreaterThan, 0)
+
+		for _, catNode := range catNodes {
+			So(catNode.Host, ShouldNotBeEmpty)
+			So(catNode.IP, ShouldNotBeEmpty)
+			So(catNode.NodeRole, ShouldNotBeEmpty)
+			So(catNode.Name, ShouldNotBeEmpty)
+		}
+	})
+
+	// Cat nodes with all parameters
+	Convey("Cat nodes with all output fields", t, func() {
+
+		fields := []string{
+			"id", "pid", "h", "i", "po", "v", "b", "j", "d", "hc", "hp", "hm",
+			"rc", "rp", "rm", "fdc", "fdp", "fdm", "l", "u", "r", "m", "n",
+			"cs", "fm", "fe", "fcm", "fce", "ft", "ftt", "gc", "gti", "gto",
+			"geti", "geto", "gmti", "gmto", "im", "idc", "idti", "idto", "iic",
+			"iiti", "iito", "mc", "mcd", "mcs", "mt", "mtd", "mts", "mtt",
+			"pc", "pm", "pq", "pti", "pto", "rto", "rti", "sfc", "sfti", "sfto",
+			"so", "sqc", "sqti", "sqto", "sc", "sm", "siwm", "siwmx", "svmm",
+		}
+		catNodes, err := c.GetCatNodeInfo(fields)
+
+		So(err, ShouldBeNil)
+		So(catNodes, ShouldNotBeNil)
+		So(len(catNodes), ShouldBeGreaterThan, 0)
+
+		for _, catNode := range catNodes {
+			So(catNode.Host, ShouldNotBeEmpty)
+			So(catNode.IP, ShouldNotBeEmpty)
+			So(catNode.NodeRole, ShouldNotBeEmpty)
+			So(catNode.Name, ShouldNotBeEmpty)
+			So(catNode.MergTotalSize, ShouldNotBeEmpty)
+			So(catNode.GetMissingTime, ShouldNotBeEmpty)
+			So(catNode.SegIdxWriterMem, ShouldNotBeEmpty)
+		}
+	})
+
+	// Test field name validation
+	Convey("Invalid field error behavior", t, func() {
+
+		fields := []string{"fm", "bogus"}
+		_, err := c.GetCatNodeInfo(fields)
+
+		So(err, ShouldNotBeNil)
+		So(err.Error(), ShouldEqual, "Number of fields (2) greater than number of stats (1)")
+	})
+}
diff --git a/lib/catresponses.go b/lib/catresponses.go
index 55c4d9e9..41e91e8e 100644
--- a/lib/catresponses.go
+++ b/lib/catresponses.go
@@ -30,3 +30,76 @@ type CatShardInfo struct {
 	NodeIP    string
 	NodeName  string
 }
+
+type CatNodeInfo struct {
+	Id                 string
+	PID                int32
+	Host               string
+	IP                 string
+	Port               int16
+	Version            string
+	Build              string
+	JDK                string
+	DiskAvail          string
+	HeapCur            string
+	HeapPerc           int16
+	HeapMax            string
+	RamCur             string
+	RamPerc            int16
+	RamMax             string
+	FileDescCur        string
+	FileDescPerc       int16
+	FileDescMax        string
+	Load               string
+	UpTime             string
+	NodeRole           string
+	Master             string
+	Name               string
+	CmpltSize          string
+	FieldMem           string
+	FieldEvict         int32
+	FiltMem            string
+	FiltEvict          int32
+	FlushTotal         int32
+	FlushTotalTime     int32
+	GetCur             int32
+	GetTime            string
+	GetTotal           int32
+	GetExistsTime      string
+	GetExistsTotal     int32
+	GetMissingTime     string
+	GetMissingTotal    int32
+	IDCacheMemory      string
+	IdxDelCur          int32
+	IdxDelTime         string
+	IdxDelTotal        int32
+	IdxIdxCur          int32
+	IdxIdxTime         string
+	IdxIdxTotal        int32
+	MergCur            int32
+	MergCurDocs        int32
+	MergCurSize        string
+	MergTotal          int32
+	MergTotalDocs      int32
+	MergTotalSize      string
+	MergTotalTime      string
+	PercCur            int32
+	PercMem            string
+	PercQueries        int32
+	PercTime           string
+	PercTotal          int32
+	RefreshTotal       int32
+	RefreshTime        string
+	SearchFetchCur     int32
+	SearchFetchTime    string
+	SearchFetchTotal   int32
+	SearchOpenContexts int32
+	SearchQueryCur     int32
+	SearchQueryTime    string
+	SearchQueryTotal   int32
+	SegCount           int32
+	SegMem             string
+	SegIdxWriterMem    string
+	SegIdxWriterMax    string
+	SegVerMapMem       string
+}

From d91c599e0110bc77714c3984d8c3d4a4249e9650 Mon Sep 17 00:00:00 2001
From: Paul Oldenburg <paul.oldenburg@logrhythm.com>
Date: Mon, 31 Aug 2015 14:45:02 -0600
Subject: [PATCH 50/81] Removed redundant comments on tests

---
 lib/catnodeinfo_test.go | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/lib/catnodeinfo_test.go b/lib/catnodeinfo_test.go
index 225602bc..ab0893ae 100644
--- a/lib/catnodeinfo_test.go
+++ b/lib/catnodeinfo_test.go
@@ -9,7 +9,6 @@ func TestCatNode(t *testing.T) {
 
 	c := NewTestConn()
 
-	// Typical cat nodes call
 	Convey("Basic cat nodes", t, func() {
 
 		fields := []string{"fm", "fe", "fcm", "fce", "ft", "ftt", "im", "rp"}
@@ -27,7 +26,6 @@ func TestCatNode(t *testing.T) {
 		}
 	})
 
-	// Cat nodes call with default arguments
 	Convey("Cat nodes with default arguments", t, func() {
 
 		fields := []string{}
@@ -45,7 +43,6 @@ func TestCatNode(t *testing.T) {
 		}
 	})
 
-	// Cat nodes with all parameters
 	Convey("Cat nodes with all output fields", t, func() {
 
 		fields := []string{
@@ -74,7 +71,6 @@ func TestCatNode(t *testing.T) {
 		}
 	})
 
-	// Test field name validation
 	Convey("Invalid field error behavior", t, func() {
 
 		fields := []string{"fm", "bogus"}

From 8ab979457e3e9287a02ce7f98dda1f2c63f2f1de Mon Sep 17 00:00:00 2001
From: Paul Oldenburg <paul.oldenburg@logrhythm.com>
Date: Mon, 31 Aug 2015 16:37:39 -0600
Subject: [PATCH 51/81] Removing a commented out import

---
 lib/baserequest.go | 1 -
 1 file changed, 1 deletion(-)

diff --git a/lib/baserequest.go b/lib/baserequest.go
index ed975f4a..55ebd6f8 100644
--- a/lib/baserequest.go
+++ b/lib/baserequest.go
@@ -18,7 +18,6 @@ import (
 	"io"
 	"io/ioutil"
 	"log"
-	//"net/http/httputil"
 	"time"
 )
 

From ec873838b7628946bf72ba5d1031b4ce6cc59561 Mon Sep 17 00:00:00 2001
From: Paul Oldenburg <paul.oldenburg@logrhythm.com>
Date: Tue, 1 Sep 2015 08:28:10 -0600
Subject: [PATCH 52/81] Changing data type of FlushTotalTime from int to string

---
 lib/catnodeinfo.go  | 3 +--
 lib/catresponses.go | 2 +-
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/lib/catnodeinfo.go b/lib/catnodeinfo.go
index f9c95c98..d1a9d827 100644
--- a/lib/catnodeinfo.go
+++ b/lib/catnodeinfo.go
@@ -90,8 +90,7 @@ func newCatNodeInfo(fields []string, indexLine string) (catNode *CatNodeInfo, er
 			i, _ := strconv.Atoi(split[i])
 			catNode.FlushTotal = int32(i)
 		case "flush.total_time", "ftt", "flushTotalTime":
-			i, _ := strconv.Atoi(split[i])
-			catNode.FlushTotalTime = int32(i)
+			catNode.FlushTotalTime = split[i]
 		case "get.current", "gc", "getCurrent":
 			i, _ := strconv.Atoi(split[i])
 			catNode.GetCur = int32(i)
diff --git a/lib/catresponses.go b/lib/catresponses.go
index 41e91e8e..7ce2e28d 100644
--- a/lib/catresponses.go
+++ b/lib/catresponses.go
@@ -61,7 +61,7 @@ type CatNodeInfo struct {
 	FiltMem            string
 	FiltEvict          int32
 	FlushTotal         int32
-	FlushTotalTime     int32
+	FlushTotalTime     string
 	GetCur             int32
 	GetTime            string
 	GetTotal           int32

From e6da24a8ed9d2078e8ac94bd7f50acdb80680760 Mon Sep 17 00:00:00 2001
From: Paul Oldenburg <paul.oldenburg@logrhythm.com>
Date: Tue, 1 Sep 2015 09:32:12 -0600
Subject: [PATCH 53/81] Adding formatting field bytes to cat nodes command

---
 lib/catnodeinfo.go | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/lib/catnodeinfo.go b/lib/catnodeinfo.go
index d1a9d827..bce84cf8 100644
--- a/lib/catnodeinfo.go
+++ b/lib/catnodeinfo.go
@@ -206,16 +206,20 @@ func newCatNodeInfo(fields []string, indexLine string) (catNode *CatNodeInfo, er
 // members are populated with statistics. If fields is nil or empty, the default
 // cat output is used.
 func (c *Conn) GetCatNodeInfo(fields []string) (catNodes []CatNodeInfo, err error) {
+
 	catNodes = make([]CatNodeInfo, 0)
 
-	// Issue a request for stats on the requested fields
-	var args map[string]interface{}
-	if len(fields) > 0 {
-		args = map[string]interface{}{"h": strings.Join(fields, ",")}
-	} else {
+	// If no fields have been specified, use the "default" arrangement
+	if len(fields) < 1 {
 		fields = []string{"host", "ip", "heap.percent", "ram.percent", "load",
 			"node.role", "master", "name"}
 	}
+
+	// Issue a request for stats on the requested fields
+	args := map[string]interface{}{
+		"bytes": "b",
+		"h":     strings.Join(fields, ","),
+	}
 	indices, err := c.DoCommand("GET", "/_cat/nodes/", args, nil)
 	if err != nil {
 		return catNodes, err

From e0959d440a3a9e3167603e83d2ea20cb23ca2071 Mon Sep 17 00:00:00 2001
From: Paul Oldenburg <paul.oldenburg@logrhythm.com>
Date: Wed, 2 Sep 2015 09:27:48 -0600
Subject: [PATCH 54/81] Handle fixed-width cat nodes output

---
 fixedwidth/fixedwidth.go      | 217 ++++++++++++++++++++++++++++++++++
 fixedwidth/fixedwidth_test.go |  77 ++++++++++++
 lib/catnodeinfo.go            | 172 +++++++++++++--------------
 lib/catnodeinfo_test.go       |   9 +-
 4 files changed, 383 insertions(+), 92 deletions(-)
 create mode 100644 fixedwidth/fixedwidth.go
 create mode 100644 fixedwidth/fixedwidth_test.go

diff --git a/fixedwidth/fixedwidth.go b/fixedwidth/fixedwidth.go
new file mode 100644
index 00000000..9a514777
--- /dev/null
+++ b/fixedwidth/fixedwidth.go
@@ -0,0 +1,217 @@
+package fixedwidth
+
+import (
+	"bytes"
+	"fmt"
+	"log"
+	"regexp"
+	"strings"
+)
+
+type Field struct {
+	Header string
+	Width  int
+}
+
+func (f *Field) String() string {
+	return fmt.Sprintf("Field{Header:\"%s\", Width:%d}", f.Header, f.Width)
+}
+
+type Column struct {
+	FieldInfo Field
+	Data      []string
+}
+
+func (c *Column) String() string {
+	d := ""
+	if len(c.Data) > 0 {
+		d = fmt.Sprint("'", strings.Join(c.Data, "', '"), "'")
+	}
+	return fmt.Sprintf("Column{FieldInfo:%s, Data:[%s]}", c.FieldInfo.String(), d)
+}
+
+type FixedWidthTable []Column
+
+var tokenRegexp *regexp.Regexp
+
+// getTokenRegexp returns the compiled Regexp object corresponding
+// to a single field header with trailing whitespace. This method
+// caches the compiled Regexp instance in the field tokenRegexp.
+func getTokenRegexp() *regexp.Regexp {
+
+	if tokenRegexp == nil {
+		t, err := regexp.Compile("(\\S+(?:\\s+)?)")
+		if err != nil {
+			log.Fatal(err)
+		}
+		tokenRegexp = t
+	}
+	return tokenRegexp
+}
+
+// parseHeaderLine assumes that the input is the header line of a fixed
+// width table, whose header names do not contain whitespace. It
+// returns the parsed header structure with the width of the last
+// field being -1.
+func parseHeaderLine(line string) (fields []Field) {
+
+	fields = []Field{}
+
+	for _, token := range getTokenRegexp().FindAllString(line, -1) {
+		fields = append(fields, Field{strings.Trim(token, " "), len(token)})
+	}
+	if len(fields) > 0 {
+		fields[len(fields)-1].Width = -1
+	}
+
+	return
+}
+
+// parseDataLines treats the specified lines as the data and the specified
+// fields as the headers for the data. One column is built and returned for
+// each field whose data vector is populated with width-parsed values from
+// the corresponding position in each line.
+func parseDataLines(lines []string, fields []Field) (t FixedWidthTable, err error) {
+
+	// Initialize a new Column slice
+	t = []Column{}
+	for _, f := range fields {
+		t = append(t, Column{FieldInfo: f, Data: []string{}})
+	}
+
+	// Parse columns, removing the values from each line
+	for c, f := range fields {
+
+		w := f.Width
+		for r := 0; r < len(lines); r++ {
+			
+			if len(lines[r]) < 1 {
+				continue
+			}
+			
+			line := lines[r]
+			if w > 0 && len(line) > w {
+				t[c].Data = append(t[c].Data, strings.Trim(line[0:w], " "))
+				lines[r] = line[w:]
+			} else {
+				t[c].Data = append(t[c].Data, line)
+			}
+		}
+	}
+
+	return t, nil
+}
+
+func (t *FixedWidthTable) Width() int {
+
+	w := 0
+	if t != nil {
+		w = len(*t)
+	}
+	return w
+}
+
+func (tp *FixedWidthTable) Height() int {
+
+	t := *tp
+	h := 0
+	if t != nil && len(t) > 0 {
+		h = len(t[0].Data)
+	}
+
+	return h
+}
+
+// Item returns the data item at the specified row and column from the
+// given FixedWidthTable using zero-based indexing. If the specified
+// indices are out of bounds, an empty string is returned.
+func (tp *FixedWidthTable) Item(row int, col int) string {
+
+	t := *tp
+
+	switch {
+	case t == nil || len(t) < 1 || len(t[0].Data) < 1:
+		//log.Fatalf("Input c is nil, empty or has no Data")
+		return ""
+	case col < 0 || col >= len(t):
+		//log.Fatalf("Index col (%d) is out of range [0,%d]", col, len(t)-1)
+		return ""
+	case row < 0 || row >= len(t[0].Data):
+		//log.Fatalf("Index row (%d) is out of range [0,%d]", row, len(t[0].Data)-1)
+		return ""
+	}
+
+	return t[col].Data[row]
+}
+
+// RowMap returns a map containing all of the rows data, indexed by
+// the header name for each column. If the table is empty, or the
+// row is out of bounds, an empty map is returned.
+func (tp *FixedWidthTable) RowMap(row int) map[string]string {
+
+	t := *tp
+	m := map[string]string{}
+	if row < 0 || row >= t.Height() || t.Width() < 1 {
+		return m
+	}
+
+	for col, c := range t {
+		m[c.FieldInfo.Header] = t.Item(row, col)
+	}
+
+	return m
+}
+
+// String returns the string representation of the fixed width table,
+// basically the inverse of NewFixedWidthTable().
+func (tp *FixedWidthTable) String() string {
+
+	t := *tp
+	var buffer bytes.Buffer
+	var pat = make(map[int]string)
+
+	for col, c := range t {
+		pat[col] = fmt.Sprint("%-", c.FieldInfo.Width, "s")
+		buffer.WriteString(fmt.Sprintf(pat[col], c.FieldInfo.Header))
+	}
+	buffer.WriteString("\n")
+
+	for r := 0; r < t.Height(); r++ {
+
+		for c, col := range t {
+			buffer.WriteString(fmt.Sprintf(pat[c], col.Data[r]))
+		}
+		if r < t.Height()-1 {
+			buffer.WriteString("\n")
+		}
+	}
+
+	return buffer.String()
+}
+
+// NewFixedWidthTable assumes that the input is a newline-delimited
+// set of lines, the first of which is a space-delimited fixed width
+// header line. The trailing spaces in each header field are assumed to
+// be included in the width of that field. The remaining lines should 
+// contain data for each field in columns of the same width. For
+// example:
+//  
+//   field1    f2 field3  field4
+//   data1     d2 data3   data four
+//   data fivesd6 datasevsdata 8888 888 88
+//
+func NewFixedWidthTable(data []byte) (*FixedWidthTable, error) {
+
+	lines := strings.Split(string(data[:]), "\n")
+	if len(lines) < 2 {
+		return nil, fmt.Errorf("At least two input lines are required, recieved %d", len(lines))
+	}
+	
+	f := parseHeaderLine(lines[0])
+	t, err := parseDataLines(lines[1:], f)
+	if err != nil {
+		return nil, err
+	}
+
+	return &t, nil
+}
diff --git a/fixedwidth/fixedwidth_test.go b/fixedwidth/fixedwidth_test.go
new file mode 100644
index 00000000..9589c23e
--- /dev/null
+++ b/fixedwidth/fixedwidth_test.go
@@ -0,0 +1,77 @@
+package fixedwidth
+
+import (
+	"bytes"
+	. "github.com/smartystreets/goconvey/convey"
+	"testing"
+)
+
+func TestFixedWidth(t *testing.T) {
+
+	Convey("Analyze basic header line", t, func() {
+		// 0----*----1----*----2----*----3----*----4----*----5----*----6
+		fields := parseHeaderLine("field1    field2  field3")
+		So(len(fields), ShouldEqual, 3)
+		So(fields[0].Header, ShouldEqual, "field1")
+		So(fields[0].Width, ShouldEqual, 10)
+		So(fields[1].Header, ShouldEqual, "field2")
+		So(fields[1].Width, ShouldEqual, 8)
+		So(fields[2].Header, ShouldEqual, "field3")
+		So(fields[2].Width, ShouldEqual, -1)
+
+		tab, err := parseDataLines([]string{
+			"data1     data2   data three",
+			"data four data_5  data six ok",
+			"data7     dataochodata niner",
+		}, fields)
+		So(err, ShouldBeNil)
+		So(tab, ShouldNotBeNil)
+		So(len(tab), ShouldEqual, 3)
+		So(len(tab[0].Data), ShouldEqual, 3)
+		So(tab.Width(), ShouldEqual, 3)
+		So(tab.Height(), ShouldEqual, 3)
+		So(tab.Item(1, 1), ShouldEqual, "data_5")
+		So(tab.Item(2, 1), ShouldEqual, "dataocho")
+
+		//		fmt.Println("Columns")
+		//		for _, c := range tab {
+		//			fmt.Println(c.String())
+		//		}
+
+		//		fmt.Println("Table")
+		//		fmt.Println(tab.String())
+
+	})
+
+	Convey("Test public API", t, func() {
+
+		tabstr := "field1    f2 field3  field4\n" +
+			"data1     d2 data3   data four\n" +
+			"data fivesd6 datasevsdata 8888 888 88\n" +
+			"data9     d10data_eledata 12"
+
+		tab, err := NewFixedWidthTable(bytes.NewBufferString(tabstr).Bytes())
+
+		So(err, ShouldBeNil)
+		So(tab, ShouldNotBeNil)
+		So(tab.Width(), ShouldEqual, 4)
+		So(tab.Height(), ShouldEqual, 3)
+		So(tab.Item(-1, -2), ShouldEqual, "")
+		So(tab.Item(0, 0), ShouldEqual, "data1")
+		So(tab.Item(2, 1), ShouldEqual, "d10")
+		So(tab.Item(1, 3), ShouldEqual, "data 8888 888 88")
+		So(tab.Item(2, 3), ShouldEqual, "data 12")
+		So(tab.Item(3, 3), ShouldEqual, "")
+
+		m := tab.RowMap(1)
+		So(len(m), ShouldEqual, 4)
+		So(m["field1"], ShouldEqual, "data fives")
+		So(m["f2"], ShouldEqual, "d6")
+
+		m = tab.RowMap(3)
+		So(len(m), ShouldEqual, 0)
+		
+		So(tab.String(), ShouldEqual, tabstr)
+	})
+
+}
diff --git a/lib/catnodeinfo.go b/lib/catnodeinfo.go
index bce84cf8..9454cfc9 100644
--- a/lib/catnodeinfo.go
+++ b/lib/catnodeinfo.go
@@ -4,195 +4,189 @@ import (
 	"fmt"
 	"strconv"
 	"strings"
+	"github.com/mattbaird/elastigo/fixedwidth"
 )
 
 // newCatNodeInfo returns an instance of CatNodeInfo populated with the
-// the information in the cat output indexLine which contains the
-// specified fields. An err is returned if a field is not known.
-func newCatNodeInfo(fields []string, indexLine string) (catNode *CatNodeInfo, err error) {
+// the information in the cat output node line, which is passed in as a
+// map of field name to value. An err is returned if a field is not known.
+func newCatNodeInfo(data map[string]string) (catNode *CatNodeInfo, err error) {
 
-	split := strings.Fields(indexLine)
 	catNode = &CatNodeInfo{}
 
-	// Check the fields length compared to the number of stats
-	lf, ls := len(fields), len(split)
-	if lf > ls {
-		return nil, fmt.Errorf("Number of fields (%d) greater than number of stats (%d)", lf, ls)
-	}
-
 	// Populate the apropriate field in CatNodeInfo
-	for i, field := range fields {
+	for field, value := range data {
 		switch field {
 		case "id", "nodeId":
-			catNode.Id = split[i]
+			catNode.Id = value
 		case "pid", "p":
-			i, _ := strconv.Atoi(split[i])
+			i, _ := strconv.Atoi(value)
 			catNode.PID = int32(i)
 		case "host", "h":
-			catNode.Host = split[i]
+			catNode.Host = value
 		case "ip", "i":
-			catNode.IP = split[i]
+			catNode.IP = value
 		case "port", "po":
-			i, _ := strconv.Atoi(split[i])
+			i, _ := strconv.Atoi(value)
 			catNode.Port = int16(i)
 		case "version", "v":
-			catNode.Version = split[i]
+			catNode.Version = value
 		case "build", "b":
-			catNode.Build = split[i]
+			catNode.Build = value
 		case "jdk", "j":
-			catNode.JDK = split[i]
+			catNode.JDK = value
 		case "disk.avail", "d", "disk", "diskAvail":
-			catNode.DiskAvail = split[i]
+			catNode.DiskAvail = value
 		case "heap.current", "hc", "heapCurrent":
-			catNode.HeapCur = split[i]
+			catNode.HeapCur = value
 		case "heap.percent", "hp", "heapPercent":
-			i, _ := strconv.Atoi(split[i])
+			i, _ := strconv.Atoi(value)
 			catNode.HeapPerc = int16(i)
 		case "heap.max", "hm", "heapMax":
-			catNode.HeapMax = split[i]
+			catNode.HeapMax = value
 		case "ram.current", "rc", "ramCurrent":
-			catNode.RamCur = split[i]
+			catNode.RamCur = value
 		case "ram.percent", "rp", "ramPercent":
-			i, _ := strconv.Atoi(split[i])
+			i, _ := strconv.Atoi(value)
 			catNode.RamPerc = int16(i)
 		case "ram.max", "rm", "ramMax":
-			catNode.RamMax = split[i]
+			catNode.RamMax = value
 		case "file_desc.current", "fdc", "fileDescriptorCurrent":
-			catNode.FileDescCur = split[i]
+			catNode.FileDescCur = value
 		case "file_desc.percent", "fdp", "fileDescriptorPercent":
-			i, _ := strconv.Atoi(split[i])
+			i, _ := strconv.Atoi(value)
 			catNode.FileDescPerc = int16(i)
 		case "file_desc.max", "fdm", "fileDescriptorMax":
-			catNode.FileDescMax = split[i]
+			catNode.FileDescMax = value
 		case "load", "l":
-			catNode.Load = split[i]
+			catNode.Load = value
 		case "uptime", "u":
-			catNode.UpTime = split[i]
+			catNode.UpTime = value
 		case "node.role", "r", "role", "dc", "nodeRole":
-			catNode.NodeRole = split[i]
+			catNode.NodeRole = value
 		case "master", "m":
-			catNode.Master = split[i]
+			catNode.Master = value
 		case "name", "n":
-			catNode.Name = split[i]
+			catNode.Name = value
 		case "completion.size", "cs", "completionSize":
-			catNode.CmpltSize = split[i]
+			catNode.CmpltSize = value
 		case "fielddata.memory_size", "fm", "fielddataMemory":
-			catNode.FieldMem = split[i]
+			catNode.FieldMem = value
 		case "fielddata.evictions", "fe", "fieldataEvictions":
-			i, _ := strconv.Atoi(split[i])
+			i, _ := strconv.Atoi(value)
 			catNode.FieldEvict = int32(i)
 		case "filter_cache.memory_size", "fcm", "filterCacheMemory":
-			catNode.FiltMem = split[i]
+			catNode.FiltMem = value
 		case "filter_cache.evictions", "fce", "filterCacheEvictions":
-			i, _ := strconv.Atoi(split[i])
+			i, _ := strconv.Atoi(value)
 			catNode.FiltEvict = int32(i)
 		case "flush.total", "ft", "flushTotal":
-			i, _ := strconv.Atoi(split[i])
+			i, _ := strconv.Atoi(value)
 			catNode.FlushTotal = int32(i)
 		case "flush.total_time", "ftt", "flushTotalTime":
-			catNode.FlushTotalTime = split[i]
+			catNode.FlushTotalTime = value
 		case "get.current", "gc", "getCurrent":
-			i, _ := strconv.Atoi(split[i])
+			i, _ := strconv.Atoi(value)
 			catNode.GetCur = int32(i)
 		case "get.time", "gti", "getTime":
-			catNode.GetTime = split[i]
+			catNode.GetTime = value
 		case "get.total", "gto", "getTotal":
-			i, _ := strconv.Atoi(split[i])
+			i, _ := strconv.Atoi(value)
 			catNode.GetTotal = int32(i)
 		case "get.exists_time", "geti", "getExistsTime":
-			catNode.GetExistsTime = split[i]
+			catNode.GetExistsTime = value
 		case "get.exists_total", "geto", "getExistsTotal":
-			i, _ := strconv.Atoi(split[i])
+			i, _ := strconv.Atoi(value)
 			catNode.GetExistsTotal = int32(i)
 		case "get.missing_time", "gmti", "getMissingTime":
-			catNode.GetMissingTime = split[i]
+			catNode.GetMissingTime = value
 		case "get.missing_total", "gmto", "getMissingTotal":
-			i, _ := strconv.Atoi(split[i])
+			i, _ := strconv.Atoi(value)
 			catNode.GetMissingTotal = int32(i)
 		case "id_cache.memory_size", "im", "idCacheMemory":
-			catNode.IDCacheMemory = split[i]
+			catNode.IDCacheMemory = value
 		case "indexing.delete_current", "idc", "indexingDeleteCurrent":
-			i, _ := strconv.Atoi(split[i])
+			i, _ := strconv.Atoi(value)
 			catNode.IdxDelCur = int32(i)
 		case "indexing.delete_time", "idti", "indexingDeleteime":
-			catNode.IdxDelTime = split[i]
+			catNode.IdxDelTime = value
 		case "indexing.delete_total", "idto", "indexingDeleteTotal":
-			i, _ := strconv.Atoi(split[i])
+			i, _ := strconv.Atoi(value)
 			catNode.IdxDelTotal = int32(i)
 		case "indexing.index_current", "iic", "indexingIndexCurrent":
-			i, _ := strconv.Atoi(split[i])
+			i, _ := strconv.Atoi(value)
 			catNode.IdxIdxCur = int32(i)
 		case "indexing.index_time", "iiti", "indexingIndexTime":
-			catNode.IdxIdxTime = split[i]
+			catNode.IdxIdxTime = value
 		case "indexing.index_total", "iito", "indexingIndexTotal":
-			i, _ := strconv.Atoi(split[i])
+			i, _ := strconv.Atoi(value)
 			catNode.IdxIdxTotal = int32(i)
 		case "merges.current", "mc", "mergesCurrent":
-			i, _ := strconv.Atoi(split[i])
+			i, _ := strconv.Atoi(value)
 			catNode.MergCur = int32(i)
 		case "merges.current_docs", "mcd", "mergesCurrentDocs":
-			i, _ := strconv.Atoi(split[i])
+			i, _ := strconv.Atoi(value)
 			catNode.MergCurDocs = int32(i)
 		case "merges.current_size", "mcs", "mergesCurrentSize":
-			catNode.MergCurSize = split[i]
+			catNode.MergCurSize = value
 		case "merges.total", "mt", "mergesTotal":
-			i, _ := strconv.Atoi(split[i])
+			i, _ := strconv.Atoi(value)
 			catNode.MergTotal = int32(i)
 		case "merges.total_docs", "mtd", "mergesTotalDocs":
-			i, _ := strconv.Atoi(split[i])
+			i, _ := strconv.Atoi(value)
 			catNode.MergTotalDocs = int32(i)
 		case "merges.total_size", "mts", "mergesTotalSize":
-			catNode.MergTotalSize = split[i]
+			catNode.MergTotalSize = value
 		case "merges.total_time", "mtt", "mergesTotalTime":
-			catNode.MergTotalTime = split[i]
+			catNode.MergTotalTime = value
 		case "percolate.current", "pc", "percolateCurrent":
-			i, _ := strconv.Atoi(split[i])
+			i, _ := strconv.Atoi(value)
 			catNode.PercCur = int32(i)
 		case "percolate.memory_size", "pm", "percolateMemory":
-			catNode.PercMem = split[i]
+			catNode.PercMem = value
 		case "percolate.queries", "pq", "percolateQueries":
-			i, _ := strconv.Atoi(split[i])
+			i, _ := strconv.Atoi(value)
 			catNode.PercQueries = int32(i)
 		case "percolate.time", "pti", "percolateTime":
-			catNode.PercTime = split[i]
+			catNode.PercTime = value
 		case "percolate.total", "pto", "percolateTotal":
-			i, _ := strconv.Atoi(split[i])
+			i, _ := strconv.Atoi(value)
 			catNode.PercTotal = int32(i)
 		case "refesh.total", "rto", "refreshTotal":
-			i, _ := strconv.Atoi(split[i])
+			i, _ := strconv.Atoi(value)
 			catNode.RefreshTotal = int32(i)
 		case "refresh.time", "rti", "refreshTime":
-			catNode.RefreshTime = split[i]
+			catNode.RefreshTime = value
 		case "search.fetch_current", "sfc", "searchFetchCurrent":
-			i, _ := strconv.Atoi(split[i])
+			i, _ := strconv.Atoi(value)
 			catNode.SearchFetchCur = int32(i)
 		case "search.fetch_time", "sfti", "searchFetchTime":
-			catNode.SearchFetchTime = split[i]
+			catNode.SearchFetchTime = value
 		case "search.fetch_total", "sfto", "searchFetchTotal":
-			i, _ := strconv.Atoi(split[i])
+			i, _ := strconv.Atoi(value)
 			catNode.SearchFetchTotal = int32(i)
 		case "search.open_contexts", "so", "searchOpenContexts":
-			i, _ := strconv.Atoi(split[i])
+			i, _ := strconv.Atoi(value)
 			catNode.SearchOpenContexts = int32(i)
 		case "search.query_current", "sqc", "searchQueryCurrent":
-			i, _ := strconv.Atoi(split[i])
+			i, _ := strconv.Atoi(value)
 			catNode.SearchQueryCur = int32(i)
 		case "search.query_time", "sqti", "searchQueryTime":
-			catNode.SearchQueryTime = split[i]
+			catNode.SearchQueryTime = value
 		case "search.query_total", "sqto", "searchQueryTotal":
-			i, _ := strconv.Atoi(split[i])
+			i, _ := strconv.Atoi(value)
 			catNode.SearchQueryTotal = int32(i)
 		case "segments.count", "sc", "segmentsCount":
-			i, _ := strconv.Atoi(split[i])
+			i, _ := strconv.Atoi(value)
 			catNode.SegCount = int32(i)
 		case "segments.memory", "sm", "segmentsMemory":
-			catNode.SegMem = split[i]
+			catNode.SegMem = value
 		case "segments.index_writer_memory", "siwm", "segmentsIndexWriterMemory":
-			catNode.SegIdxWriterMem = split[i]
+			catNode.SegIdxWriterMem = value
 		case "segments.index_writer_max_memory", "siwmx", "segmentsIndexWriterMaxMemory":
-			catNode.SegIdxWriterMax = split[i]
+			catNode.SegIdxWriterMax = value
 		case "segments.version_map_memory", "svmm", "segmentsVersionMapMemory":
-			catNode.SegVerMapMem = split[i]
+			catNode.SegVerMapMem = value
 		default:
 			return nil, fmt.Errorf("Invalid cat nodes field: %s", field)
 		}
@@ -205,6 +199,8 @@ func newCatNodeInfo(fields []string, indexLine string) (catNode *CatNodeInfo, er
 // fields and returns a list of CatNodeInfos, one for each node, whose requested
 // members are populated with statistics. If fields is nil or empty, the default
 // cat output is used.
+// NOTE: if you include the name field, make sure it is the last field in the
+// list, because name values can contain spaces which screw up the parsing
 func (c *Conn) GetCatNodeInfo(fields []string) (catNodes []CatNodeInfo, err error) {
 
 	catNodes = make([]CatNodeInfo, 0)
@@ -217,6 +213,7 @@ func (c *Conn) GetCatNodeInfo(fields []string) (catNodes []CatNodeInfo, err erro
 
 	// Issue a request for stats on the requested fields
 	args := map[string]interface{}{
+		"v": "",
 		"bytes": "b",
 		"h":     strings.Join(fields, ","),
 	}
@@ -225,17 +222,14 @@ func (c *Conn) GetCatNodeInfo(fields []string) (catNodes []CatNodeInfo, err erro
 		return catNodes, err
 	}
 
-	// Create a CatIndexInfo for each line in the response
-	indexLines := strings.Split(string(indices[:]), "\n")
-	for _, index := range indexLines {
+	// Create a table of response data
+	tab, err := fixedwidth.NewFixedWidthTable(indices)
+	for row := 0; row < tab.Height(); row++ {
 
-		// Ignore empty output lines
-		if len(index) < 1 {
-			continue
-		}
+		data := tab.RowMap(row)
 
 		// Create a CatNodeInfo and append it to the result
-		ci, err := newCatNodeInfo(fields, index)
+		ci, err := newCatNodeInfo(data)
 		if ci != nil {
 			catNodes = append(catNodes, *ci)
 		} else if err != nil {
diff --git a/lib/catnodeinfo_test.go b/lib/catnodeinfo_test.go
index ab0893ae..d284d74d 100644
--- a/lib/catnodeinfo_test.go
+++ b/lib/catnodeinfo_test.go
@@ -74,9 +74,12 @@ func TestCatNode(t *testing.T) {
 	Convey("Invalid field error behavior", t, func() {
 
 		fields := []string{"fm", "bogus"}
-		_, err := c.GetCatNodeInfo(fields)
+		catNodes, err := c.GetCatNodeInfo(fields)
+		
+		So(err, ShouldBeNil)
 
-		So(err, ShouldNotBeNil)
-		So(err.Error(), ShouldEqual, "Number of fields (2) greater than number of stats (1)")
+		for _, catNode := range catNodes {
+			So(catNode.FieldMem, ShouldNotBeEmpty)
+		}
 	})
 }

From 99516d1b6ae48fc6f3f2dff64209d96c22179581 Mon Sep 17 00:00:00 2001
From: Paul Oldenburg <paul.oldenburg@logrhythm.com>
Date: Wed, 2 Sep 2015 14:31:22 -0600
Subject: [PATCH 55/81] Refactoring to fix stat types and accomodate spaces in
 the Name field

---
 lib/catnodeinfo.go      | 235 +++++++++++++++++++++-------------------
 lib/catnodeinfo_test.go |   6 +-
 lib/catresponses.go     |  64 +++++------
 3 files changed, 158 insertions(+), 147 deletions(-)

diff --git a/lib/catnodeinfo.go b/lib/catnodeinfo.go
index 9454cfc9..cee73774 100644
--- a/lib/catnodeinfo.go
+++ b/lib/catnodeinfo.go
@@ -4,189 +4,196 @@ import (
 	"fmt"
 	"strconv"
 	"strings"
-	"github.com/mattbaird/elastigo/fixedwidth"
 )
 
+
+
 // newCatNodeInfo returns an instance of CatNodeInfo populated with the
-// the information in the cat output node line, which is passed in as a
-// map of field name to value. An err is returned if a field is not known.
-func newCatNodeInfo(data map[string]string) (catNode *CatNodeInfo, err error) {
+// the information in the cat output indexLine which contains the
+// specified fields. An err is returned if a field is not known.
+func newCatNodeInfo(fields []string, indexLine string) (catNode *CatNodeInfo, err error) {
 
+	split := strings.Fields(indexLine)
 	catNode = &CatNodeInfo{}
 
+	// Check the fields length compared to the number of stats
+	lf, ls := len(fields), len(split)
+	if lf > ls {
+		return nil, fmt.Errorf("Number of fields (%d) greater than number of stats (%d)", lf, ls)
+	}
+
 	// Populate the apropriate field in CatNodeInfo
-	for field, value := range data {
+	for i, field := range fields {
+
 		switch field {
 		case "id", "nodeId":
-			catNode.Id = value
+			catNode.Id = split[i]
 		case "pid", "p":
-			i, _ := strconv.Atoi(value)
-			catNode.PID = int32(i)
+			catNode.PID = split[i]
 		case "host", "h":
-			catNode.Host = value
+			catNode.Host = split[i]
 		case "ip", "i":
-			catNode.IP = value
+			catNode.IP = split[i]
 		case "port", "po":
-			i, _ := strconv.Atoi(value)
-			catNode.Port = int16(i)
+			catNode.Port = split[i]
 		case "version", "v":
-			catNode.Version = value
+			catNode.Version = split[i]
 		case "build", "b":
-			catNode.Build = value
+			catNode.Build = split[i]
 		case "jdk", "j":
-			catNode.JDK = value
+			catNode.JDK = split[i]
 		case "disk.avail", "d", "disk", "diskAvail":
-			catNode.DiskAvail = value
+			catNode.DiskAvail = split[i]
 		case "heap.current", "hc", "heapCurrent":
-			catNode.HeapCur = value
+			catNode.HeapCur = split[i]
 		case "heap.percent", "hp", "heapPercent":
-			i, _ := strconv.Atoi(value)
-			catNode.HeapPerc = int16(i)
+			catNode.HeapPerc = split[i]
 		case "heap.max", "hm", "heapMax":
-			catNode.HeapMax = value
+			catNode.HeapMax = split[i]
 		case "ram.current", "rc", "ramCurrent":
-			catNode.RamCur = value
+			catNode.RamCur = split[i]
 		case "ram.percent", "rp", "ramPercent":
-			i, _ := strconv.Atoi(value)
-			catNode.RamPerc = int16(i)
+			val, err := strconv.Atoi(split[i])
+			if err != nil {
+				return nil, err
+			}
+			catNode.RamPerc = int16(val)
 		case "ram.max", "rm", "ramMax":
-			catNode.RamMax = value
+			catNode.RamMax = split[i]
 		case "file_desc.current", "fdc", "fileDescriptorCurrent":
-			catNode.FileDescCur = value
+			catNode.FileDescCur = split[i]
 		case "file_desc.percent", "fdp", "fileDescriptorPercent":
-			i, _ := strconv.Atoi(value)
-			catNode.FileDescPerc = int16(i)
+			catNode.FileDescPerc = split[i]
 		case "file_desc.max", "fdm", "fileDescriptorMax":
-			catNode.FileDescMax = value
+			catNode.FileDescMax = split[i]
 		case "load", "l":
-			catNode.Load = value
+			catNode.Load = split[i]
 		case "uptime", "u":
-			catNode.UpTime = value
+			catNode.UpTime = split[i]
 		case "node.role", "r", "role", "dc", "nodeRole":
-			catNode.NodeRole = value
+			catNode.NodeRole = split[i]
 		case "master", "m":
-			catNode.Master = value
+			catNode.Master = split[i]
 		case "name", "n":
-			catNode.Name = value
+			catNode.Name = strings.Join(split[i:], " ")
 		case "completion.size", "cs", "completionSize":
-			catNode.CmpltSize = value
+			catNode.CmpltSize = split[i]
 		case "fielddata.memory_size", "fm", "fielddataMemory":
-			catNode.FieldMem = value
+			val, err := strconv.Atoi(split[i])
+			if err != nil {
+				return nil, err
+			}
+			catNode.FieldMem = val
 		case "fielddata.evictions", "fe", "fieldataEvictions":
-			i, _ := strconv.Atoi(value)
-			catNode.FieldEvict = int32(i)
+			val, err := strconv.Atoi(split[i])
+			if err != nil {
+				return nil, err
+			}
+			catNode.FieldEvict = val
 		case "filter_cache.memory_size", "fcm", "filterCacheMemory":
-			catNode.FiltMem = value
+			val, err := strconv.Atoi(split[i])
+			if err != nil {
+				return nil, err
+			}
+			catNode.FiltMem = val
 		case "filter_cache.evictions", "fce", "filterCacheEvictions":
-			i, _ := strconv.Atoi(value)
-			catNode.FiltEvict = int32(i)
+			val, err := strconv.Atoi(split[i])
+			if err != nil {
+				return nil, err
+			}
+			catNode.FiltEvict = val
 		case "flush.total", "ft", "flushTotal":
-			i, _ := strconv.Atoi(value)
-			catNode.FlushTotal = int32(i)
+			val, err := strconv.Atoi(split[i])
+			if err != nil {
+				return nil, err
+			}
+			catNode.FlushTotal = val
 		case "flush.total_time", "ftt", "flushTotalTime":
-			catNode.FlushTotalTime = value
+			catNode.FlushTotalTime = split[i]
 		case "get.current", "gc", "getCurrent":
-			i, _ := strconv.Atoi(value)
-			catNode.GetCur = int32(i)
+			catNode.GetCur = split[i]
 		case "get.time", "gti", "getTime":
-			catNode.GetTime = value
+			catNode.GetTime = split[i]
 		case "get.total", "gto", "getTotal":
-			i, _ := strconv.Atoi(value)
-			catNode.GetTotal = int32(i)
+			catNode.GetTotal = split[i]
 		case "get.exists_time", "geti", "getExistsTime":
-			catNode.GetExistsTime = value
+			catNode.GetExistsTime = split[i]
 		case "get.exists_total", "geto", "getExistsTotal":
-			i, _ := strconv.Atoi(value)
-			catNode.GetExistsTotal = int32(i)
+			catNode.GetExistsTotal = split[i]
 		case "get.missing_time", "gmti", "getMissingTime":
-			catNode.GetMissingTime = value
+			catNode.GetMissingTime = split[i]
 		case "get.missing_total", "gmto", "getMissingTotal":
-			i, _ := strconv.Atoi(value)
-			catNode.GetMissingTotal = int32(i)
+			catNode.GetMissingTotal = split[i]
 		case "id_cache.memory_size", "im", "idCacheMemory":
-			catNode.IDCacheMemory = value
+			val, err := strconv.Atoi(split[i])
+			if err != nil {
+				return nil, err
+			}
+			catNode.IDCacheMemory = val
 		case "indexing.delete_current", "idc", "indexingDeleteCurrent":
-			i, _ := strconv.Atoi(value)
-			catNode.IdxDelCur = int32(i)
+			catNode.IdxDelCur = split[i]
 		case "indexing.delete_time", "idti", "indexingDeleteime":
-			catNode.IdxDelTime = value
+			catNode.IdxDelTime = split[i]
 		case "indexing.delete_total", "idto", "indexingDeleteTotal":
-			i, _ := strconv.Atoi(value)
-			catNode.IdxDelTotal = int32(i)
+			catNode.IdxDelTotal = split[i]
 		case "indexing.index_current", "iic", "indexingIndexCurrent":
-			i, _ := strconv.Atoi(value)
-			catNode.IdxIdxCur = int32(i)
+			catNode.IdxIdxCur = split[i]
 		case "indexing.index_time", "iiti", "indexingIndexTime":
-			catNode.IdxIdxTime = value
+			catNode.IdxIdxTime = split[i]
 		case "indexing.index_total", "iito", "indexingIndexTotal":
-			i, _ := strconv.Atoi(value)
-			catNode.IdxIdxTotal = int32(i)
+			catNode.IdxIdxTotal = split[i]
 		case "merges.current", "mc", "mergesCurrent":
-			i, _ := strconv.Atoi(value)
-			catNode.MergCur = int32(i)
+			catNode.MergCur = split[i]
 		case "merges.current_docs", "mcd", "mergesCurrentDocs":
-			i, _ := strconv.Atoi(value)
-			catNode.MergCurDocs = int32(i)
+			catNode.MergCurDocs = split[i]
 		case "merges.current_size", "mcs", "mergesCurrentSize":
-			catNode.MergCurSize = value
+			catNode.MergCurSize = split[i]
 		case "merges.total", "mt", "mergesTotal":
-			i, _ := strconv.Atoi(value)
-			catNode.MergTotal = int32(i)
+			catNode.MergTotal = split[i]
 		case "merges.total_docs", "mtd", "mergesTotalDocs":
-			i, _ := strconv.Atoi(value)
-			catNode.MergTotalDocs = int32(i)
+			catNode.MergTotalDocs = split[i]
 		case "merges.total_size", "mts", "mergesTotalSize":
-			catNode.MergTotalSize = value
+			catNode.MergTotalSize = split[i]
 		case "merges.total_time", "mtt", "mergesTotalTime":
-			catNode.MergTotalTime = value
+			catNode.MergTotalTime = split[i]
 		case "percolate.current", "pc", "percolateCurrent":
-			i, _ := strconv.Atoi(value)
-			catNode.PercCur = int32(i)
+			catNode.PercCur = split[i]
 		case "percolate.memory_size", "pm", "percolateMemory":
-			catNode.PercMem = value
+			catNode.PercMem = split[i]
 		case "percolate.queries", "pq", "percolateQueries":
-			i, _ := strconv.Atoi(value)
-			catNode.PercQueries = int32(i)
+			catNode.PercQueries = split[i]
 		case "percolate.time", "pti", "percolateTime":
-			catNode.PercTime = value
+			catNode.PercTime = split[i]
 		case "percolate.total", "pto", "percolateTotal":
-			i, _ := strconv.Atoi(value)
-			catNode.PercTotal = int32(i)
+			catNode.PercTotal = split[i]
 		case "refesh.total", "rto", "refreshTotal":
-			i, _ := strconv.Atoi(value)
-			catNode.RefreshTotal = int32(i)
+			catNode.RefreshTotal = split[i]
 		case "refresh.time", "rti", "refreshTime":
-			catNode.RefreshTime = value
+			catNode.RefreshTime = split[i]
 		case "search.fetch_current", "sfc", "searchFetchCurrent":
-			i, _ := strconv.Atoi(value)
-			catNode.SearchFetchCur = int32(i)
+			catNode.SearchFetchCur = split[i]
 		case "search.fetch_time", "sfti", "searchFetchTime":
-			catNode.SearchFetchTime = value
+			catNode.SearchFetchTime = split[i]
 		case "search.fetch_total", "sfto", "searchFetchTotal":
-			i, _ := strconv.Atoi(value)
-			catNode.SearchFetchTotal = int32(i)
+			catNode.SearchFetchTotal = split[i]
 		case "search.open_contexts", "so", "searchOpenContexts":
-			i, _ := strconv.Atoi(value)
-			catNode.SearchOpenContexts = int32(i)
+			catNode.SearchOpenContexts = split[i]
 		case "search.query_current", "sqc", "searchQueryCurrent":
-			i, _ := strconv.Atoi(value)
-			catNode.SearchQueryCur = int32(i)
+			catNode.SearchQueryCur = split[i]
 		case "search.query_time", "sqti", "searchQueryTime":
-			catNode.SearchQueryTime = value
+			catNode.SearchQueryTime = split[i]
 		case "search.query_total", "sqto", "searchQueryTotal":
-			i, _ := strconv.Atoi(value)
-			catNode.SearchQueryTotal = int32(i)
+			catNode.SearchQueryTotal = split[i]
 		case "segments.count", "sc", "segmentsCount":
-			i, _ := strconv.Atoi(value)
-			catNode.SegCount = int32(i)
+			catNode.SegCount = split[i]
 		case "segments.memory", "sm", "segmentsMemory":
-			catNode.SegMem = value
+			catNode.SegMem = split[i]
 		case "segments.index_writer_memory", "siwm", "segmentsIndexWriterMemory":
-			catNode.SegIdxWriterMem = value
+			catNode.SegIdxWriterMem = split[i]
 		case "segments.index_writer_max_memory", "siwmx", "segmentsIndexWriterMaxMemory":
-			catNode.SegIdxWriterMax = value
+			catNode.SegIdxWriterMax = split[i]
 		case "segments.version_map_memory", "svmm", "segmentsVersionMapMemory":
-			catNode.SegVerMapMem = value
+			catNode.SegVerMapMem = split[i]
 		default:
 			return nil, fmt.Errorf("Invalid cat nodes field: %s", field)
 		}
@@ -213,7 +220,6 @@ func (c *Conn) GetCatNodeInfo(fields []string) (catNodes []CatNodeInfo, err erro
 
 	// Issue a request for stats on the requested fields
 	args := map[string]interface{}{
-		"v": "",
 		"bytes": "b",
 		"h":     strings.Join(fields, ","),
 	}
@@ -222,16 +228,19 @@ func (c *Conn) GetCatNodeInfo(fields []string) (catNodes []CatNodeInfo, err erro
 		return catNodes, err
 	}
 
-	// Create a table of response data
-	tab, err := fixedwidth.NewFixedWidthTable(indices)
-	for row := 0; row < tab.Height(); row++ {
+	// Create a CatIndexInfo for each line in the response
+	indexLines := strings.Split(string(indices[:]), "\n")
+	for _, index := range indexLines {
 
-		data := tab.RowMap(row)
+		// Ignore empty output lines
+		if len(index) < 1 {
+			continue
+		}
 
 		// Create a CatNodeInfo and append it to the result
-		ci, err := newCatNodeInfo(data)
-		if ci != nil {
-			catNodes = append(catNodes, *ci)
+		info, err := newCatNodeInfo(fields, index)
+		if info != nil {
+			catNodes = append(catNodes, *info)
 		} else if err != nil {
 			return catNodes, err
 		}
diff --git a/lib/catnodeinfo_test.go b/lib/catnodeinfo_test.go
index d284d74d..2b64889a 100644
--- a/lib/catnodeinfo_test.go
+++ b/lib/catnodeinfo_test.go
@@ -11,18 +11,20 @@ func TestCatNode(t *testing.T) {
 
 	Convey("Basic cat nodes", t, func() {
 
-		fields := []string{"fm", "fe", "fcm", "fce", "ft", "ftt", "im", "rp"}
+		fields := []string{"fm", "fe", "fcm", "fce", "ft", "ftt", "im", "rp", "n"}
 		catNodes, err := c.GetCatNodeInfo(fields)
 
 		So(err, ShouldBeNil)
 		So(catNodes, ShouldNotBeNil)
 		So(len(catNodes), ShouldBeGreaterThan, 0)
 
+
 		for _, catNode := range catNodes {
 			So(catNode.FieldMem, ShouldNotBeEmpty)
 			So(catNode.FiltMem, ShouldNotBeEmpty)
 			So(catNode.IDCacheMemory, ShouldNotBeEmpty)
 			So(catNode.RamPerc, ShouldNotBeEmpty)
+			So(catNode.Name, ShouldNotBeEmpty)
 		}
 	})
 
@@ -76,7 +78,7 @@ func TestCatNode(t *testing.T) {
 		fields := []string{"fm", "bogus"}
 		catNodes, err := c.GetCatNodeInfo(fields)
 		
-		So(err, ShouldBeNil)
+		So(err, ShouldNotBeNil)
 
 		for _, catNode := range catNodes {
 			So(catNode.FieldMem, ShouldNotBeEmpty)
diff --git a/lib/catresponses.go b/lib/catresponses.go
index 7ce2e28d..17129549 100644
--- a/lib/catresponses.go
+++ b/lib/catresponses.go
@@ -33,22 +33,22 @@ type CatShardInfo struct {
 
 type CatNodeInfo struct {
 	Id                 string
-	PID                int32
+	PID                string
 	Host               string
 	IP                 string
-	Port               int16
+	Port               string
 	Version            string
 	Build              string
 	JDK                string
 	DiskAvail          string
 	HeapCur            string
-	HeapPerc           int16
+	HeapPerc           string
 	HeapMax            string
 	RamCur             string
 	RamPerc            int16
 	RamMax             string
 	FileDescCur        string
-	FileDescPerc       int16
+	FileDescPerc       string
 	FileDescMax        string
 	Load               string
 	UpTime             string
@@ -56,48 +56,48 @@ type CatNodeInfo struct {
 	Master             string
 	Name               string
 	CmpltSize          string
-	FieldMem           string
-	FieldEvict         int32
-	FiltMem            string
-	FiltEvict          int32
-	FlushTotal         int32
+	FieldMem           int
+	FieldEvict         int
+	FiltMem            int
+	FiltEvict          int
+	FlushTotal         int
 	FlushTotalTime     string
-	GetCur             int32
+	GetCur             string
 	GetTime            string
-	GetTotal           int32
+	GetTotal           string
 	GetExistsTime      string
-	GetExistsTotal     int32
+	GetExistsTotal     string
 	GetMissingTime     string
-	GetMissingTotal    int32
-	IDCacheMemory      string
-	IdxDelCur          int32
+	GetMissingTotal    string
+	IDCacheMemory      int
+	IdxDelCur          string
 	IdxDelTime         string
-	IdxDelTotal        int32
-	IdxIdxCur          int32
+	IdxDelTotal        string
+	IdxIdxCur          string
 	IdxIdxTime         string
-	IdxIdxTotal        int32
-	MergCur            int32
-	MergCurDocs        int32
+	IdxIdxTotal        string
+	MergCur            string
+	MergCurDocs        string
 	MergCurSize        string
-	MergTotal          int32
-	MergTotalDocs      int32
+	MergTotal          string
+	MergTotalDocs      string
 	MergTotalSize      string
 	MergTotalTime      string
-	PercCur            int32
+	PercCur            string
 	PercMem            string
-	PercQueries        int32
+	PercQueries        string
 	PercTime           string
-	PercTotal          int32
-	RefreshTotal       int32
+	PercTotal          string
+	RefreshTotal       string
 	RefreshTime        string
-	SearchFetchCur     int32
+	SearchFetchCur     string
 	SearchFetchTime    string
-	SearchFetchTotal   int32
-	SearchOpenContexts int32
-	SearchQueryCur     int32
+	SearchFetchTotal   string
+	SearchOpenContexts string
+	SearchQueryCur     string
 	SearchQueryTime    string
-	SearchQueryTotal   int32
-	SegCount           int32
+	SearchQueryTotal   string
+	SegCount           string
 	SegMem             string
 	SegIdxWriterMem    string
 	SegIdxWriterMax    string

From ec15ec1bed1f8cb3f3d8d55535517748dffb299b Mon Sep 17 00:00:00 2001
From: Paul Oldenburg <oldenbur@gmail.com>
Date: Wed, 2 Sep 2015 15:12:03 -0600
Subject: [PATCH 56/81] Refactoring test setup and tests to ensure more
 explicit mapping

---
 lib/searchfacet_test.go  |  4 +---
 lib/searchfilter_test.go | 12 +++++-------
 lib/searchsearch_test.go | 17 ++++++++---------
 lib/testsetup.go         | 15 +++++++++++----
 4 files changed, 25 insertions(+), 23 deletions(-)

diff --git a/lib/searchfacet_test.go b/lib/searchfacet_test.go
index e7001431..11e2664a 100644
--- a/lib/searchfacet_test.go
+++ b/lib/searchfacet_test.go
@@ -21,9 +21,7 @@ func TestFacetRegex(t *testing.T) {
 
 	c := NewTestConn()
 	PopulateTestDB(t, c)
-	defer func() {
-		TearDownTestDB(c)
-	}()
+	defer TearDownTestDB(c)
 
 	Convey("Facted regex query", t, func() {
 
diff --git a/lib/searchfilter_test.go b/lib/searchfilter_test.go
index defc9c1e..3039fb4e 100644
--- a/lib/searchfilter_test.go
+++ b/lib/searchfilter_test.go
@@ -20,9 +20,7 @@ func TestFilters(t *testing.T) {
 
 	c := NewTestConn()
 	PopulateTestDB(t, c)
-	defer func() {
-		TearDownTestDB(c)
-	}()
+	defer TearDownTestDB(c)
 
 	Convey("Exists filter", t, func() {
 		qry := Search("oilers").Filter(
@@ -49,7 +47,7 @@ func TestFilters(t *testing.T) {
 
 	Convey("Terms filter", t, func() {
 		qry := Search("oilers").Filter(
-			Filter().Terms("pos", "rw", "lw"),
+			Filter().Terms("pos", "RW", "LW"),
 		)
 		out, err := qry.Result(c)
 		So(err, ShouldBeNil)
@@ -60,7 +58,7 @@ func TestFilters(t *testing.T) {
 
 	Convey("Filter involving an AND", t, func() {
 		qry := Search("oilers").Filter(
-			Filter().Terms("pos", "lw"),
+			Filter().Terms("pos", "LW"),
 			Filter().Exists("PIM"),
 		)
 		out, err := qry.Result(c)
@@ -72,7 +70,7 @@ func TestFilters(t *testing.T) {
 
 	Convey("Filterng filter results", t, func() {
 		qry := Search("oilers").Filter(
-			Filter().Terms("pos", "lw"),
+			Filter().Terms("pos", "LW"),
 		)
 		qry.Filter(
 			Filter().Exists("PIM"),
@@ -87,7 +85,7 @@ func TestFilters(t *testing.T) {
 	Convey("Filter involving OR", t, func() {
 		qry := Search("oilers").Filter(
 			"or",
-			Filter().Terms("pos", "g"),
+			Filter().Terms("pos", "G"),
 			Range().Field("goals").Gt(80),
 		)
 		out, err := qry.Result(c)
diff --git a/lib/searchsearch_test.go b/lib/searchsearch_test.go
index 688c9ccf..4ad72afc 100644
--- a/lib/searchsearch_test.go
+++ b/lib/searchsearch_test.go
@@ -21,9 +21,7 @@ func TestSearch(t *testing.T) {
 
 	c := NewTestConn()
 	PopulateTestDB(t, c)
-	defer func() {
-		//TearDownTestDB(c)
-	}()
+	defer TearDownTestDB(c)
 
 	Convey("Wildcard request query", t, func() {
 
@@ -63,14 +61,15 @@ func TestSearch(t *testing.T) {
 
 	Convey("URL Request query string", t, func() {
 
-		out, err := c.SearchUri("oilers", "", map[string]interface{}{"q": "pos:*w"})
+		out, err := c.SearchUri("oilers", "", map[string]interface{}{"q": "pos:LW"})
 
 		So(err, ShouldBeNil)
 		So(out, ShouldNotBeNil)
 		So(out.Hits, ShouldNotBeNil)
-		So(out.Hits.Total, ShouldEqual, 6)
+		So(out.Hits.Total, ShouldEqual, 3)
 	})
 
+
 	//	A faceted search for what "type" of events there are
 	//	- since we are not specifying an elasticsearch type it searches all ()
 	//
@@ -133,6 +132,7 @@ func TestSearch(t *testing.T) {
 		So(len(h.List("teams.terms")), ShouldEqual, 4)
 	})
 
+
 	Convey("Facet search with wildcard", t, func() {
 
 		qry := Search("oilers").Pretty().Facet(
@@ -170,7 +170,7 @@ func TestSearch(t *testing.T) {
 	Convey("Search query with terms", t, func() {
 
 		qry := Search("oilers").Query(
-			Query().Term("teams", "nyr"),
+			Query().Term("teams", "NYR"),
 		)
 		out, err := qry.Result(c)
 		So(err, ShouldBeNil)
@@ -182,7 +182,7 @@ func TestSearch(t *testing.T) {
 	Convey("Search query with fields", t, func() {
 
 		qry := Search("oilers").Query(
-			Query().Fields("teams", "nyr", "", ""),
+			Query().Fields("teams", "NYR", "", ""),
 		)
 		out, err := qry.Result(c)
 		So(err, ShouldBeNil)
@@ -217,7 +217,7 @@ func TestSearch(t *testing.T) {
 		out, err := Search("oilers").Size("25").Query(
 			Query().Fields("name", "*d*", "", ""),
 		).Filter(
-			Filter().Terms("teams", "stl"),
+			Filter().Terms("teams", "STL"),
 		).Result(c)
 		So(err, ShouldBeNil)
 		So(out, ShouldNotBeNil)
@@ -288,5 +288,4 @@ func TestSearch(t *testing.T) {
 		h1 := gou.NewJsonHelper(b)
 		So(h1.String("name"), ShouldEqual, "Wayne Gretzky")
 	})
-
 }
diff --git a/lib/testsetup.go b/lib/testsetup.go
index 19f83c8a..026f2dc5 100644
--- a/lib/testsetup.go
+++ b/lib/testsetup.go
@@ -28,13 +28,20 @@ func newIndexWorker(c *Conn, t *testing.T) func(interface{}) {
 func PopulateTestDB(t *testing.T, c *Conn) {
 
 	// it is not technically necessary to create an index here
-	c.CreateIndex("oilers")
+	_, err := c.CreateIndex("oilers")
+	if err != nil {
+		t.Fatal("Error in CreateIndex", err)
+	}
 
 	// set the mapping for dob to be a date so it can be used for range searches
-	_, err := c.DoCommand("PUT", "/oilers/heyday/_mapping?ignore_conflicts", nil,
-		string(`{"heyday": {"properties": {"dob": {"type": "date"}}}}`))
+	_, err = c.DoCommand("PUT", "/oilers/heyday/_mapping?ignore_conflicts", nil,
+		string(`{"heyday": {"properties": {
+					"dob": {"type": "date", "format": "basic_date"},
+					"pos": {"type": "string", "index": "not_analyzed"},
+					"teams": {"type": "string", "index": "not_analyzed"}
+				}}}`))
 	if err != nil {
-		t.Fatal("Error setting dob mapping")
+		t.Fatal("Error setting dob mapping", err)
 	}
 
 	idx := newIndexWorker(c, t)

From 8a0d8c4095bfa7832b4ee35bc5444e8a48aed13f Mon Sep 17 00:00:00 2001
From: Paul Oldenburg <paul.oldenburg@logrhythm.com>
Date: Wed, 2 Sep 2015 15:20:56 -0600
Subject: [PATCH 57/81] Removing unused fixedwidth module

---
 fixedwidth/fixedwidth.go      | 217 ----------------------------------
 fixedwidth/fixedwidth_test.go |  77 ------------
 2 files changed, 294 deletions(-)
 delete mode 100644 fixedwidth/fixedwidth.go
 delete mode 100644 fixedwidth/fixedwidth_test.go

diff --git a/fixedwidth/fixedwidth.go b/fixedwidth/fixedwidth.go
deleted file mode 100644
index 9a514777..00000000
--- a/fixedwidth/fixedwidth.go
+++ /dev/null
@@ -1,217 +0,0 @@
-package fixedwidth
-
-import (
-	"bytes"
-	"fmt"
-	"log"
-	"regexp"
-	"strings"
-)
-
-type Field struct {
-	Header string
-	Width  int
-}
-
-func (f *Field) String() string {
-	return fmt.Sprintf("Field{Header:\"%s\", Width:%d}", f.Header, f.Width)
-}
-
-type Column struct {
-	FieldInfo Field
-	Data      []string
-}
-
-func (c *Column) String() string {
-	d := ""
-	if len(c.Data) > 0 {
-		d = fmt.Sprint("'", strings.Join(c.Data, "', '"), "'")
-	}
-	return fmt.Sprintf("Column{FieldInfo:%s, Data:[%s]}", c.FieldInfo.String(), d)
-}
-
-type FixedWidthTable []Column
-
-var tokenRegexp *regexp.Regexp
-
-// getTokenRegexp returns the compiled Regexp object corresponding
-// to a single field header with trailing whitespace. This method
-// caches the compiled Regexp instance in the field tokenRegexp.
-func getTokenRegexp() *regexp.Regexp {
-
-	if tokenRegexp == nil {
-		t, err := regexp.Compile("(\\S+(?:\\s+)?)")
-		if err != nil {
-			log.Fatal(err)
-		}
-		tokenRegexp = t
-	}
-	return tokenRegexp
-}
-
-// parseHeaderLine assumes that the input is the header line of a fixed
-// width table, whose header names do not contain whitespace. It
-// returns the parsed header structure with the width of the last
-// field being -1.
-func parseHeaderLine(line string) (fields []Field) {
-
-	fields = []Field{}
-
-	for _, token := range getTokenRegexp().FindAllString(line, -1) {
-		fields = append(fields, Field{strings.Trim(token, " "), len(token)})
-	}
-	if len(fields) > 0 {
-		fields[len(fields)-1].Width = -1
-	}
-
-	return
-}
-
-// parseDataLines treats the specified lines as the data and the specified
-// fields as the headers for the data. One column is built and returned for
-// each field whose data vector is populated with width-parsed values from
-// the corresponding position in each line.
-func parseDataLines(lines []string, fields []Field) (t FixedWidthTable, err error) {
-
-	// Initialize a new Column slice
-	t = []Column{}
-	for _, f := range fields {
-		t = append(t, Column{FieldInfo: f, Data: []string{}})
-	}
-
-	// Parse columns, removing the values from each line
-	for c, f := range fields {
-
-		w := f.Width
-		for r := 0; r < len(lines); r++ {
-			
-			if len(lines[r]) < 1 {
-				continue
-			}
-			
-			line := lines[r]
-			if w > 0 && len(line) > w {
-				t[c].Data = append(t[c].Data, strings.Trim(line[0:w], " "))
-				lines[r] = line[w:]
-			} else {
-				t[c].Data = append(t[c].Data, line)
-			}
-		}
-	}
-
-	return t, nil
-}
-
-func (t *FixedWidthTable) Width() int {
-
-	w := 0
-	if t != nil {
-		w = len(*t)
-	}
-	return w
-}
-
-func (tp *FixedWidthTable) Height() int {
-
-	t := *tp
-	h := 0
-	if t != nil && len(t) > 0 {
-		h = len(t[0].Data)
-	}
-
-	return h
-}
-
-// Item returns the data item at the specified row and column from the
-// given FixedWidthTable using zero-based indexing. If the specified
-// indices are out of bounds, an empty string is returned.
-func (tp *FixedWidthTable) Item(row int, col int) string {
-
-	t := *tp
-
-	switch {
-	case t == nil || len(t) < 1 || len(t[0].Data) < 1:
-		//log.Fatalf("Input c is nil, empty or has no Data")
-		return ""
-	case col < 0 || col >= len(t):
-		//log.Fatalf("Index col (%d) is out of range [0,%d]", col, len(t)-1)
-		return ""
-	case row < 0 || row >= len(t[0].Data):
-		//log.Fatalf("Index row (%d) is out of range [0,%d]", row, len(t[0].Data)-1)
-		return ""
-	}
-
-	return t[col].Data[row]
-}
-
-// RowMap returns a map containing all of the rows data, indexed by
-// the header name for each column. If the table is empty, or the
-// row is out of bounds, an empty map is returned.
-func (tp *FixedWidthTable) RowMap(row int) map[string]string {
-
-	t := *tp
-	m := map[string]string{}
-	if row < 0 || row >= t.Height() || t.Width() < 1 {
-		return m
-	}
-
-	for col, c := range t {
-		m[c.FieldInfo.Header] = t.Item(row, col)
-	}
-
-	return m
-}
-
-// String returns the string representation of the fixed width table,
-// basically the inverse of NewFixedWidthTable().
-func (tp *FixedWidthTable) String() string {
-
-	t := *tp
-	var buffer bytes.Buffer
-	var pat = make(map[int]string)
-
-	for col, c := range t {
-		pat[col] = fmt.Sprint("%-", c.FieldInfo.Width, "s")
-		buffer.WriteString(fmt.Sprintf(pat[col], c.FieldInfo.Header))
-	}
-	buffer.WriteString("\n")
-
-	for r := 0; r < t.Height(); r++ {
-
-		for c, col := range t {
-			buffer.WriteString(fmt.Sprintf(pat[c], col.Data[r]))
-		}
-		if r < t.Height()-1 {
-			buffer.WriteString("\n")
-		}
-	}
-
-	return buffer.String()
-}
-
-// NewFixedWidthTable assumes that the input is a newline-delimited
-// set of lines, the first of which is a space-delimited fixed width
-// header line. The trailing spaces in each header field are assumed to
-// be included in the width of that field. The remaining lines should 
-// contain data for each field in columns of the same width. For
-// example:
-//  
-//   field1    f2 field3  field4
-//   data1     d2 data3   data four
-//   data fivesd6 datasevsdata 8888 888 88
-//
-func NewFixedWidthTable(data []byte) (*FixedWidthTable, error) {
-
-	lines := strings.Split(string(data[:]), "\n")
-	if len(lines) < 2 {
-		return nil, fmt.Errorf("At least two input lines are required, recieved %d", len(lines))
-	}
-	
-	f := parseHeaderLine(lines[0])
-	t, err := parseDataLines(lines[1:], f)
-	if err != nil {
-		return nil, err
-	}
-
-	return &t, nil
-}
diff --git a/fixedwidth/fixedwidth_test.go b/fixedwidth/fixedwidth_test.go
deleted file mode 100644
index 9589c23e..00000000
--- a/fixedwidth/fixedwidth_test.go
+++ /dev/null
@@ -1,77 +0,0 @@
-package fixedwidth
-
-import (
-	"bytes"
-	. "github.com/smartystreets/goconvey/convey"
-	"testing"
-)
-
-func TestFixedWidth(t *testing.T) {
-
-	Convey("Analyze basic header line", t, func() {
-		// 0----*----1----*----2----*----3----*----4----*----5----*----6
-		fields := parseHeaderLine("field1    field2  field3")
-		So(len(fields), ShouldEqual, 3)
-		So(fields[0].Header, ShouldEqual, "field1")
-		So(fields[0].Width, ShouldEqual, 10)
-		So(fields[1].Header, ShouldEqual, "field2")
-		So(fields[1].Width, ShouldEqual, 8)
-		So(fields[2].Header, ShouldEqual, "field3")
-		So(fields[2].Width, ShouldEqual, -1)
-
-		tab, err := parseDataLines([]string{
-			"data1     data2   data three",
-			"data four data_5  data six ok",
-			"data7     dataochodata niner",
-		}, fields)
-		So(err, ShouldBeNil)
-		So(tab, ShouldNotBeNil)
-		So(len(tab), ShouldEqual, 3)
-		So(len(tab[0].Data), ShouldEqual, 3)
-		So(tab.Width(), ShouldEqual, 3)
-		So(tab.Height(), ShouldEqual, 3)
-		So(tab.Item(1, 1), ShouldEqual, "data_5")
-		So(tab.Item(2, 1), ShouldEqual, "dataocho")
-
-		//		fmt.Println("Columns")
-		//		for _, c := range tab {
-		//			fmt.Println(c.String())
-		//		}
-
-		//		fmt.Println("Table")
-		//		fmt.Println(tab.String())
-
-	})
-
-	Convey("Test public API", t, func() {
-
-		tabstr := "field1    f2 field3  field4\n" +
-			"data1     d2 data3   data four\n" +
-			"data fivesd6 datasevsdata 8888 888 88\n" +
-			"data9     d10data_eledata 12"
-
-		tab, err := NewFixedWidthTable(bytes.NewBufferString(tabstr).Bytes())
-
-		So(err, ShouldBeNil)
-		So(tab, ShouldNotBeNil)
-		So(tab.Width(), ShouldEqual, 4)
-		So(tab.Height(), ShouldEqual, 3)
-		So(tab.Item(-1, -2), ShouldEqual, "")
-		So(tab.Item(0, 0), ShouldEqual, "data1")
-		So(tab.Item(2, 1), ShouldEqual, "d10")
-		So(tab.Item(1, 3), ShouldEqual, "data 8888 888 88")
-		So(tab.Item(2, 3), ShouldEqual, "data 12")
-		So(tab.Item(3, 3), ShouldEqual, "")
-
-		m := tab.RowMap(1)
-		So(len(m), ShouldEqual, 4)
-		So(m["field1"], ShouldEqual, "data fives")
-		So(m["f2"], ShouldEqual, "d6")
-
-		m = tab.RowMap(3)
-		So(len(m), ShouldEqual, 0)
-		
-		So(tab.String(), ShouldEqual, tabstr)
-	})
-
-}

From fa4edb2811c0f6b5b250f6880fa88b4a2a0777c1 Mon Sep 17 00:00:00 2001
From: Ben Aldrich <vrecan@gmail.com>
Date: Wed, 2 Sep 2015 15:36:13 -0600
Subject: [PATCH 58/81] Remove broken link for total views

---
 README.md | 1 -
 1 file changed, 1 deletion(-)

diff --git a/README.md b/README.md
index 1a94145a..04ab1cd2 100644
--- a/README.md
+++ b/README.md
@@ -2,7 +2,6 @@ elastigo v2.0
 -------------
 
 [![Build Status](https://drone.io/github.com/mattbaird/elastigo/status.png)](https://drone.io/github.com/mattbaird/elastigo/latest)
-[![Total views](https://sourcegraph.com/api/repos/github.com/mattbaird/elastigo/counters/views.png)](https://sourcegraph.com/github.com/mattbaird/elastigo)
 
 Big thanks to @alicebob for helping to get the drone.io CI working (note: the badge is being cached, known issue).
 

From 7a476d962d59c40656ea90bc2b498f0ec01a5afe Mon Sep 17 00:00:00 2001
From: Paul Oldenburg <paul.oldenburg@logrhythm.com>
Date: Wed, 2 Sep 2015 15:37:36 -0600
Subject: [PATCH 59/81] Removed test which was dependent on elasticsearch
 version

---
 lib/catnodeinfo_test.go | 31 +------------------------------
 1 file changed, 1 insertion(+), 30 deletions(-)

diff --git a/lib/catnodeinfo_test.go b/lib/catnodeinfo_test.go
index 2b64889a..441de244 100644
--- a/lib/catnodeinfo_test.go
+++ b/lib/catnodeinfo_test.go
@@ -18,7 +18,6 @@ func TestCatNode(t *testing.T) {
 		So(catNodes, ShouldNotBeNil)
 		So(len(catNodes), ShouldBeGreaterThan, 0)
 
-
 		for _, catNode := range catNodes {
 			So(catNode.FieldMem, ShouldNotBeEmpty)
 			So(catNode.FiltMem, ShouldNotBeEmpty)
@@ -45,39 +44,11 @@ func TestCatNode(t *testing.T) {
 		}
 	})
 
-	Convey("Cat nodes with all output fields", t, func() {
-
-		fields := []string{
-			"id", "pid", "h", "i", "po", "v", "b", "j", "d", "hc", "hp", "hm",
-			"rc", "rp", "rm", "fdc", "fdp", "fdm", "l", "u", "r", "m", "n",
-			"cs", "fm", "fe", "fcm", "fce", "ft", "ftt", "gc", "gti", "gto",
-			"geti", "geto", "gmti", "gmto", "im", "idc", "idti", "idto", "iic",
-			"iiti", "iito", "mc", "mcd", "mcs", "mt", "mtd", "mts", "mtt",
-			"pc", "pm", "pq", "pti", "pto", "rto", "rti", "sfc", "sfti", "sfto",
-			"so", "sqc", "sqti", "sqto", "sc", "sm", "siwm", "siwmx", "svmm",
-		}
-		catNodes, err := c.GetCatNodeInfo(fields)
-
-		So(err, ShouldBeNil)
-		So(catNodes, ShouldNotBeNil)
-		So(len(catNodes), ShouldBeGreaterThan, 0)
-
-		for _, catNode := range catNodes {
-			So(catNode.Host, ShouldNotBeEmpty)
-			So(catNode.IP, ShouldNotBeEmpty)
-			So(catNode.NodeRole, ShouldNotBeEmpty)
-			So(catNode.Name, ShouldNotBeEmpty)
-			So(catNode.MergTotalSize, ShouldNotBeEmpty)
-			So(catNode.GetMissingTime, ShouldNotBeEmpty)
-			So(catNode.SegIdxWriterMem, ShouldNotBeEmpty)
-		}
-	})
-
 	Convey("Invalid field error behavior", t, func() {
 
 		fields := []string{"fm", "bogus"}
 		catNodes, err := c.GetCatNodeInfo(fields)
-		
+
 		So(err, ShouldNotBeNil)
 
 		for _, catNode := range catNodes {

From 25ac375631b93f52f242e4e884fd43897d430f9f Mon Sep 17 00:00:00 2001
From: Ben Aldrich <vrecan@gmail.com>
Date: Wed, 2 Sep 2015 15:48:44 -0600
Subject: [PATCH 60/81] update link in readme for drone.io

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 04ab1cd2..26d0625d 100644
--- a/README.md
+++ b/README.md
@@ -1,7 +1,7 @@
 elastigo v2.0 
 -------------
 
-[![Build Status](https://drone.io/github.com/mattbaird/elastigo/status.png)](https://drone.io/github.com/mattbaird/elastigo/latest)
+[![Build Status](https://drone.io/github.com/mattbaird/elastigo/status.png)](https://drone.io/github.com/mattbaird/elastigo)
 
 Big thanks to @alicebob for helping to get the drone.io CI working (note: the badge is being cached, known issue).
 

From c698cdaa601e80e722feffba2a0b5284624160ef Mon Sep 17 00:00:00 2001
From: woodsaj <anthony@monkey.id.au>
Date: Thu, 3 Sep 2015 16:16:27 +0800
Subject: [PATCH 61/81] remove refresh from action/meta. fixes #215

This commit moves the refresh property to be
an attribute of the BulkIndexer struct.
The refresh attribute is included as a query param
when the request is sent to the Elasticsearch API.
---
 lib/corebulk.go         | 33 ++++++++++++++++++---------------
 lib/corebulk_test.go    | 28 ++++++++++++++--------------
 lib/coreexample_test.go |  4 ++--
 lib/coretest_test.go    |  2 +-
 4 files changed, 35 insertions(+), 32 deletions(-)

diff --git a/lib/corebulk.go b/lib/corebulk.go
index 763ea97b..bc38c9e2 100644
--- a/lib/corebulk.go
+++ b/lib/corebulk.go
@@ -47,6 +47,11 @@ type BulkIndexer struct {
 	// to allow a mock sendor for test purposes
 	Sender func(*bytes.Buffer) error
 
+	// The refresh parameter can be set to true in order to refresh the
+	// relevant primary and replica shards immediately after the bulk
+	// operation has occurred
+	Refresh bool
+
 	// If we encounter an error in sending, we are going to retry for this long
 	// before returning an error
 	// if 0 it will not retry
@@ -281,9 +286,9 @@ func (b *BulkIndexer) shutdown() {
 // The index bulk API adds or updates a typed JSON document to a specific index, making it searchable.
 // it operates by buffering requests, and ocassionally flushing to elasticsearch
 // http://www.elasticsearch.org/guide/reference/api/bulk.html
-func (b *BulkIndexer) Index(index string, _type string, id, parent, ttl string, date *time.Time, data interface{}, refresh bool) error {
+func (b *BulkIndexer) Index(index string, _type string, id, parent, ttl string, date *time.Time, data interface{}) error {
 	//{ "index" : { "_index" : "test", "_type" : "type1", "_id" : "1" } }
-	by, err := WriteBulkBytes("index", index, _type, id, parent, ttl, date, data, refresh)
+	by, err := WriteBulkBytes("index", index, _type, id, parent, ttl, date, data)
 	if err != nil {
 		return err
 	}
@@ -291,9 +296,9 @@ func (b *BulkIndexer) Index(index string, _type string, id, parent, ttl string,
 	return nil
 }
 
-func (b *BulkIndexer) Update(index string, _type string, id, parent, ttl string, date *time.Time, data interface{}, refresh bool) error {
+func (b *BulkIndexer) Update(index string, _type string, id, parent, ttl string, date *time.Time, data interface{}) error {
 	//{ "index" : { "_index" : "test", "_type" : "type1", "_id" : "1" } }
-	by, err := WriteBulkBytes("update", index, _type, id, parent, ttl, date, data, refresh)
+	by, err := WriteBulkBytes("update", index, _type, id, parent, ttl, date, data)
 	if err != nil {
 		return err
 	}
@@ -301,20 +306,20 @@ func (b *BulkIndexer) Update(index string, _type string, id, parent, ttl string,
 	return nil
 }
 
-func (b *BulkIndexer) Delete(index, _type, id string, refresh bool) {
-	queryLine := fmt.Sprintf("{\"delete\":{\"_index\":%q,\"_type\":%q,\"_id\":%q,\"refresh\":%t}}\n", index, _type, id, refresh)
+func (b *BulkIndexer) Delete(index, _type, id string) {
+	queryLine := fmt.Sprintf("{\"delete\":{\"_index\":%q,\"_type\":%q,\"_id\":%q}}\n", index, _type, id)
 	b.bulkChannel <- []byte(queryLine)
 	return
 }
 
-func (b *BulkIndexer) UpdateWithWithScript(index string, _type string, id, parent, ttl string, date *time.Time, script string, refresh bool) error {
+func (b *BulkIndexer) UpdateWithWithScript(index string, _type string, id, parent, ttl string, date *time.Time, script string) error {
 
 	var data map[string]interface{} = make(map[string]interface{})
 	data["script"] = script
-	return b.Update(index, _type, id, parent, ttl, date, data, refresh)
+	return b.Update(index, _type, id, parent, ttl, date, data)
 }
 
-func (b *BulkIndexer) UpdateWithPartialDoc(index string, _type string, id, parent, ttl string, date *time.Time, partialDoc interface{}, upsert bool, refresh bool) error {
+func (b *BulkIndexer) UpdateWithPartialDoc(index string, _type string, id, parent, ttl string, date *time.Time, partialDoc interface{}, upsert bool) error {
 
 	var data map[string]interface{} = make(map[string]interface{})
 
@@ -322,7 +327,7 @@ func (b *BulkIndexer) UpdateWithPartialDoc(index string, _type string, id, paren
 	if upsert {
 		data["doc_as_upsert"] = true
 	}
-	return b.Update(index, _type, id, parent, ttl, date, data, refresh)
+	return b.Update(index, _type, id, parent, ttl, date, data)
 }
 
 // This does the actual send of a buffer, which has already been formatted
@@ -336,7 +341,7 @@ func (b *BulkIndexer) Send(buf *bytes.Buffer) error {
 
 	response := responseStruct{}
 
-	body, err := b.conn.DoCommand("POST", "/_bulk", nil, buf)
+	body, err := b.conn.DoCommand("POST", fmt.Sprintf("/_bulk?refresh=%t", b.Refresh), nil, buf)
 
 	if err != nil {
 		b.numErrors += 1
@@ -355,7 +360,7 @@ func (b *BulkIndexer) Send(buf *bytes.Buffer) error {
 
 // Given a set of arguments for index, type, id, data create a set of bytes that is formatted for bulkd index
 // http://www.elasticsearch.org/guide/reference/api/bulk.html
-func WriteBulkBytes(op string, index string, _type string, id, parent, ttl string, date *time.Time, data interface{}, refresh bool) ([]byte, error) {
+func WriteBulkBytes(op string, index string, _type string, id, parent, ttl string, date *time.Time, data interface{}) ([]byte, error) {
 	// only index and update are currently supported
 	if op != "index" && op != "update" {
 		return nil, errors.New(fmt.Sprintf("Operation '%s' is not yet supported", op))
@@ -394,9 +399,7 @@ func WriteBulkBytes(op string, index string, _type string, id, parent, ttl strin
 		buf.WriteString(strconv.FormatInt(date.UnixNano()/1e6, 10))
 		buf.WriteString(`"`)
 	}
-	if refresh {
-		buf.WriteString(`,"refresh":true`)
-	}
+
 	buf.WriteString(`}}`)
 	buf.WriteRune('\n')
 	//buf.WriteByte('\n')
diff --git a/lib/corebulk_test.go b/lib/corebulk_test.go
index d8486503..4358c6c6 100644
--- a/lib/corebulk_test.go
+++ b/lib/corebulk_test.go
@@ -63,7 +63,7 @@ func TestBulkIndexerBasic(t *testing.T) {
 		messageSets += 1
 		totalBytesSent += buf.Len()
 		buffers = append(buffers, buf)
-		// log.Printf("buffer:%s", string(buf.Bytes()))
+		//log.Printf("buffer:%s", string(buf.Bytes()))
 		return indexer.Send(buf)
 	}
 	indexer.Start()
@@ -75,7 +75,7 @@ func TestBulkIndexerBasic(t *testing.T) {
 		"date": "yesterday",
 	}
 
-	err := indexer.Index(testIndex, "user", "1", "", "", &date, data, true)
+	err := indexer.Index(testIndex, "user", "1", "", "", &date, data)
 	waitFor(func() bool {
 		return len(buffers) > 0
 	}, 5)
@@ -84,10 +84,10 @@ func TestBulkIndexerBasic(t *testing.T) {
 	//totalBytesSent = totalBytesSent - len(*eshost)
 	assert.T(t, len(buffers) == 1, fmt.Sprintf("Should have sent one operation but was %d", len(buffers)))
 	assert.T(t, indexer.NumErrors() == 0 && err == nil, fmt.Sprintf("Should not have any errors. NumErrors: %v, err: %v", indexer.NumErrors(), err))
-	expectedBytes := 144
+	expectedBytes := 129
 	assert.T(t, totalBytesSent == expectedBytes, fmt.Sprintf("Should have sent %v bytes but was %v", expectedBytes, totalBytesSent))
 
-	err = indexer.Index(testIndex, "user", "2", "", "", nil, data, true)
+	err = indexer.Index(testIndex, "user", "2", "", "", nil, data)
 	waitFor(func() bool {
 		return len(buffers) > 1
 	}, 5)
@@ -99,7 +99,7 @@ func TestBulkIndexerBasic(t *testing.T) {
 	assert.T(t, len(buffers) == 2, fmt.Sprintf("Should have another buffer ct=%d", len(buffers)))
 
 	assert.T(t, indexer.NumErrors() == 0, fmt.Sprintf("Should not have any errors %d", indexer.NumErrors()))
-	expectedBytes = 250 // with refresh
+	expectedBytes = 220
 	assert.T(t, closeInt(totalBytesSent, expectedBytes), fmt.Sprintf("Should have sent %v bytes but was %v", expectedBytes, totalBytesSent))
 
 	indexer.Stop()
@@ -137,7 +137,7 @@ func XXXTestBulkUpdate(t *testing.T) {
 	data := map[string]interface{}{
 		"script": "ctx._source.count += 2",
 	}
-	err = indexer.Update("users", "user", "5", "", "", &date, data, true)
+	err = indexer.Update("users", "user", "5", "", "", &date, data)
 	// So here's the deal. Flushing does seem to work, you just have to give the
 	// channel a moment to recieve the message ...
 	//	<- time.After(time.Millisecond * 20)
@@ -183,9 +183,9 @@ func TestBulkSmallBatch(t *testing.T) {
 	indexer.Start()
 	<-time.After(time.Millisecond * 20)
 
-	indexer.Index("users", "user", "2", "", "", &date, data, true)
-	indexer.Index("users", "user", "3", "", "", &date, data, true)
-	indexer.Index("users", "user", "4", "", "", &date, data, true)
+	indexer.Index("users", "user", "2", "", "", &date, data)
+	indexer.Index("users", "user", "3", "", "", &date, data)
+	indexer.Index("users", "user", "4", "", "", &date, data)
 	<-time.After(time.Millisecond * 200)
 	//	indexer.Flush()
 	indexer.Stop()
@@ -207,14 +207,14 @@ func TestBulkDelete(t *testing.T) {
 
 	indexer.Start()
 
-	indexer.Delete("fake", "fake_type", "1", true)
+	indexer.Delete("fake", "fake_type", "1")
 
 	indexer.Flush()
 	indexer.Stop()
 
 	sent := string(sentBytes)
 
-	expected := `{"delete":{"_index":"fake","_type":"fake_type","_id":"1","refresh":true}}
+	expected := `{"delete":{"_index":"fake","_type":"fake_type","_id":"1"}}
 `
 	asExpected := sent == expected
 	assert.T(t, asExpected, fmt.Sprintf("Should have sent '%s' but actually sent '%s'", expected, sent))
@@ -234,7 +234,7 @@ func XXXTestBulkErrors(t *testing.T) {
 		for i := 0; i < 20; i++ {
 			date := time.Unix(1257894000, 0)
 			data := map[string]interface{}{"name": "smurfs", "age": 22, "date": date}
-			indexer.Index("users", "user", strconv.Itoa(i), "", "", &date, data, true)
+			indexer.Index("users", "user", strconv.Itoa(i), "", "", &date, data)
 		}
 	}()
 	var errBuf *ErrorBuffer
@@ -274,7 +274,7 @@ func BenchmarkSend(b *testing.B) {
 		about := make([]byte, 1000)
 		rand.Read(about)
 		data := map[string]interface{}{"name": "smurfs", "age": 22, "date": time.Unix(1257894000, 0), "about": about}
-		indexer.Index("users", "user", strconv.Itoa(i), "", "", nil, data, true)
+		indexer.Index("users", "user", strconv.Itoa(i), "", "", nil, data)
 	}
 	log.Printf("Sent %d messages in %d sets totaling %d bytes \n", b.N, sets, totalBytes)
 	if indexer.NumErrors() != 0 {
@@ -308,7 +308,7 @@ func BenchmarkSendBytes(b *testing.B) {
 		return indexer.Send(buf)
 	}
 	for i := 0; i < b.N; i++ {
-		indexer.Index("users", "user", strconv.Itoa(i), "", "", nil, body, true)
+		indexer.Index("users", "user", strconv.Itoa(i), "", "", nil, body)
 	}
 	log.Printf("Sent %d messages in %d sets totaling %d bytes \n", b.N, sets, totalBytes)
 	if indexer.NumErrors() != 0 {
diff --git a/lib/coreexample_test.go b/lib/coreexample_test.go
index 608ee4a7..fed31dfe 100644
--- a/lib/coreexample_test.go
+++ b/lib/coreexample_test.go
@@ -25,7 +25,7 @@ func ExampleBulkIndexer_simple() {
 
 	indexer := c.NewBulkIndexerErrors(10, 60)
 	indexer.Start()
-	indexer.Index("twitter", "user", "1", "", "", nil, `{"name":"bob"}`, true)
+	indexer.Index("twitter", "user", "1", "", "", nil, `{"name":"bob"}`)
 	indexer.Stop()
 }
 
@@ -46,7 +46,7 @@ func ExampleBulkIndexer_responses() {
 	}
 	indexer.Start()
 	for i := 0; i < 20; i++ {
-		indexer.Index("twitter", "user", strconv.Itoa(i), "", "", nil, `{"name":"bob"}`, true)
+		indexer.Index("twitter", "user", strconv.Itoa(i), "", "", nil, `{"name":"bob"}`)
 	}
 	indexer.Stop()
 }
diff --git a/lib/coretest_test.go b/lib/coretest_test.go
index 6d2acf72..37aa3fc9 100644
--- a/lib/coretest_test.go
+++ b/lib/coretest_test.go
@@ -179,7 +179,7 @@ func LoadTestData() {
 				log.Println("HM, already exists? ", ge.Url)
 			}
 			docsm[id] = true
-			indexer.Index(testIndex, ge.Type, id, "", "", &ge.Created, line, true)
+			indexer.Index(testIndex, ge.Type, id, "", "", &ge.Created, line)
 			docCt++
 		} else {
 			log.Println("ERROR? ", string(line))

From 658877a111f891bb604dce1c0f8d63d8dcb70009 Mon Sep 17 00:00:00 2001
From: flow <flowbehappy@gmail.com>
Date: Sun, 6 Sep 2015 12:02:38 +0800
Subject: [PATCH 62/81] also show the illegal json content in request error

---
 lib/request.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/request.go b/lib/request.go
index b4ac4b34..4e42dda1 100644
--- a/lib/request.go
+++ b/lib/request.go
@@ -101,7 +101,7 @@ func (r *Request) DoResponse(v interface{}) (*http.Response, []byte, error) {
 	if res.StatusCode > 304 && v != nil {
 		jsonErr := json.Unmarshal(bodyBytes, v)
 		if jsonErr != nil {
-			return nil, nil, jsonErr
+			return nil, nil, fmt.Errorf("Json response unmarshal error: [%s], response content: [%s]", jsonErr.Error(), string(bodyBytes))
 		}
 	}
 	return res, bodyBytes, err

From 2e1c4bcdfadf60f5e248e657ce8914ef7feeb8a5 Mon Sep 17 00:00:00 2001
From: woodsaj <anthony@monkey.id.au>
Date: Wed, 9 Sep 2015 15:09:27 +0800
Subject: [PATCH 63/81] add unit tests for refresh param in BulkAPI request

---
 lib/corebulk_test.go | 57 +++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 54 insertions(+), 3 deletions(-)

diff --git a/lib/corebulk_test.go b/lib/corebulk_test.go
index 4358c6c6..1fd7dc40 100644
--- a/lib/corebulk_test.go
+++ b/lib/corebulk_test.go
@@ -17,13 +17,13 @@ import (
 	"encoding/json"
 	"flag"
 	"fmt"
+	"github.com/araddon/gou"
+	"github.com/bmizerany/assert"
 	"log"
+	"net/url"
 	"strconv"
 	"testing"
 	"time"
-
-	"github.com/araddon/gou"
-	"github.com/bmizerany/assert"
 )
 
 //  go test -bench=".*"
@@ -105,6 +105,57 @@ func TestBulkIndexerBasic(t *testing.T) {
 	indexer.Stop()
 }
 
+func TestRefreshParam(t *testing.T) {
+	var requrl *url.URL
+	InitTests(true)
+	c := NewTestConn()
+	c.RequestTracer = func(method, urlStr, body string) {
+		requrl, _ = url.Parse(urlStr)
+	}
+	date := time.Unix(1257894000, 0)
+	data := map[string]interface{}{"name": "smurfs", "age": 22, "date": date}
+
+	// Now tests small batches
+	indexer := c.NewBulkIndexer(1)
+	indexer.Refresh = true
+
+	indexer.Start()
+	<-time.After(time.Millisecond * 20)
+
+	indexer.Index("users", "user", "2", "", "", &date, data)
+
+	<-time.After(time.Millisecond * 200)
+	//	indexer.Flush()
+	indexer.Stop()
+
+	assert.T(t, requrl.Query().Get("refresh") == "true", "Should have set refresh query param to true")
+}
+
+func TestWithoutRefreshParam(t *testing.T) {
+	var requrl *url.URL
+	InitTests(true)
+	c := NewTestConn()
+	c.RequestTracer = func(method, urlStr, body string) {
+		requrl, _ = url.Parse(urlStr)
+	}
+	date := time.Unix(1257894000, 0)
+	data := map[string]interface{}{"name": "smurfs", "age": 22, "date": date}
+
+	// Now tests small batches
+	indexer := c.NewBulkIndexer(1)
+
+	indexer.Start()
+	<-time.After(time.Millisecond * 20)
+
+	indexer.Index("users", "user", "2", "", "", &date, data)
+
+	<-time.After(time.Millisecond * 200)
+	//	indexer.Flush()
+	indexer.Stop()
+
+	assert.T(t, requrl.Query().Get("refresh") == "false", "Should have set refresh query param to false")
+}
+
 // currently broken in drone.io
 func XXXTestBulkUpdate(t *testing.T) {
 	var (

From 9d2596c6a958a6395056862917d009f673d7493c Mon Sep 17 00:00:00 2001
From: woodsaj <anthony@monkey.id.au>
Date: Wed, 9 Sep 2015 15:11:46 +0800
Subject: [PATCH 64/81] undo auto ording of imports.

---
 lib/corebulk_test.go | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/lib/corebulk_test.go b/lib/corebulk_test.go
index 1fd7dc40..da853c96 100644
--- a/lib/corebulk_test.go
+++ b/lib/corebulk_test.go
@@ -17,13 +17,14 @@ import (
 	"encoding/json"
 	"flag"
 	"fmt"
-	"github.com/araddon/gou"
-	"github.com/bmizerany/assert"
 	"log"
 	"net/url"
 	"strconv"
 	"testing"
 	"time"
+
+	"github.com/araddon/gou"
+	"github.com/bmizerany/assert"
 )
 
 //  go test -bench=".*"

From 1c4fcf09aaba687938ad1579b432b05865e3480d Mon Sep 17 00:00:00 2001
From: Paul Oldenburg <paul.oldenburg@logrhythm.com>
Date: Wed, 9 Sep 2015 14:50:44 -0600
Subject: [PATCH 65/81] Renaming test setup file

---
 lib/{testsetup.go => setup_test.go} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename lib/{testsetup.go => setup_test.go} (100%)

diff --git a/lib/testsetup.go b/lib/setup_test.go
similarity index 100%
rename from lib/testsetup.go
rename to lib/setup_test.go

From 9bb50d92e77a65f14bf9d5783bcb15782f7d3da3 Mon Sep 17 00:00:00 2001
From: Mike Yoon <mike@yaki.org>
Date: Wed, 9 Sep 2015 22:15:01 -0700
Subject: [PATCH 66/81] Fix tests

---
 lib/searchfilter_test.go | 6 +++---
 lib/searchsearch_test.go | 6 +++---
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/lib/searchfilter_test.go b/lib/searchfilter_test.go
index 4f0d0102..57671bf7 100644
--- a/lib/searchfilter_test.go
+++ b/lib/searchfilter_test.go
@@ -232,7 +232,7 @@ func TestFilters(t *testing.T) {
 
 	Convey("Terms filter", t, func() {
 		qry := Search("oilers").Filter(
-			Filter().Terms("pos", "RW", "LW"),
+			Filter().Terms("pos", TEM_DEFAULT, "RW", "LW"),
 		)
 		out, err := qry.Result(c)
 		So(err, ShouldBeNil)
@@ -244,7 +244,7 @@ func TestFilters(t *testing.T) {
 	Convey("Filter involving an AND", t, func() {
 		qry := Search("oilers").Filter(
 			Filter().And(
-				Filter().Terms("pos", "LW"),
+				Filter().Terms("pos", TEM_DEFAULT, "LW"),
 				Filter().Exists("PIM"),
 			),
 		)
@@ -259,7 +259,7 @@ func TestFilters(t *testing.T) {
 
 	Convey("Filterng filter results", t, func() {
 		qry := Search("oilers").Filter(
-			Filter().Terms("pos", "LW"),
+			Filter().Terms("pos", TEM_DEFAULT, "LW"),
 		)
 		qry.Filter(
 			Filter().Exists("PIM"),
diff --git a/lib/searchsearch_test.go b/lib/searchsearch_test.go
index 4843cbe8..b09420f0 100644
--- a/lib/searchsearch_test.go
+++ b/lib/searchsearch_test.go
@@ -155,7 +155,7 @@ func TestSearch(t *testing.T) {
 			Facet().Fields("teams").Size("20"),
 		).Query(
 			Query().Range(
-				Filter().Range("dob", 19600101, nil, 19621231, nil, ""),
+				Filter().Range("dob", "19600101", nil, "19621231", nil, ""),
 			).Search("*w*"),
 		)
 		out, err := qry.Result(c)
@@ -217,7 +217,7 @@ func TestSearch(t *testing.T) {
 		out, err := Search("oilers").Size("25").Query(
 			Query().Fields("name", "*d*", "", ""),
 		).Filter(
-			Filter().Terms("teams", "STL"),
+			Filter().Terms("teams", TEM_DEFAULT, "STL"),
 		).Result(c)
 		So(err, ShouldBeNil)
 		So(out, ShouldNotBeNil)
@@ -229,7 +229,7 @@ func TestSearch(t *testing.T) {
 
 		out, err := Search("oilers").Size("25").Query(
 			Query().Range(
-				Filter().Range("dob", 19600101, nil, 19621231, nil, ""),
+				Filter().Range("dob", "19600101", nil, "19621231", nil, ""),
 			).Search("*w*"),
 		).Result(c)
 		So(err, ShouldBeNil)

From debade9cfe9a48edb581ecbcfd17710df63a5b42 Mon Sep 17 00:00:00 2001
From: woodsaj <anthony@monkey.id.au>
Date: Tue, 29 Sep 2015 14:58:22 +0800
Subject: [PATCH 67/81] sync access to shared buffer int corebulk_test.

issue #241
The buffer used for tracking bytes written is shared by
many goroutines.  This commit uses a sync.Mutex to sychronize
reads and writes to the buffer to elimitate race conditions.
---
 lib/corebulk_test.go | 47 +++++++++++++++++++++++++++++++++-----------
 1 file changed, 35 insertions(+), 12 deletions(-)

diff --git a/lib/corebulk_test.go b/lib/corebulk_test.go
index da853c96..7226623f 100644
--- a/lib/corebulk_test.go
+++ b/lib/corebulk_test.go
@@ -17,19 +17,42 @@ import (
 	"encoding/json"
 	"flag"
 	"fmt"
+	"github.com/araddon/gou"
+	"github.com/bmizerany/assert"
 	"log"
 	"net/url"
 	"strconv"
+	"sync"
 	"testing"
 	"time"
-
-	"github.com/araddon/gou"
-	"github.com/bmizerany/assert"
 )
 
 //  go test -bench=".*"
 //  go test -bench="Bulk"
 
+type sharedBuffer struct {
+	mu     sync.Mutex
+	Buffer []*bytes.Buffer
+}
+
+func NewSharedBuffer() *sharedBuffer {
+	return &sharedBuffer{
+		Buffer: make([]*bytes.Buffer, 0),
+	}
+}
+
+func (b *sharedBuffer) Append(buf *bytes.Buffer) {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	b.Buffer = append(b.Buffer, buf)
+}
+
+func (b *sharedBuffer) Length() int {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	return len(b.Buffer)
+}
+
 func init() {
 	flag.Parse()
 	if testing.Verbose() {
@@ -49,7 +72,7 @@ func closeInt(a, b int) bool {
 func TestBulkIndexerBasic(t *testing.T) {
 	testIndex := "users"
 	var (
-		buffers        = make([]*bytes.Buffer, 0)
+		buffers        = NewSharedBuffer()
 		totalBytesSent int
 		messageSets    int
 	)
@@ -63,7 +86,7 @@ func TestBulkIndexerBasic(t *testing.T) {
 	indexer.Sender = func(buf *bytes.Buffer) error {
 		messageSets += 1
 		totalBytesSent += buf.Len()
-		buffers = append(buffers, buf)
+		buffers.Append(buf)
 		//log.Printf("buffer:%s", string(buf.Bytes()))
 		return indexer.Send(buf)
 	}
@@ -78,26 +101,26 @@ func TestBulkIndexerBasic(t *testing.T) {
 
 	err := indexer.Index(testIndex, "user", "1", "", "", &date, data)
 	waitFor(func() bool {
-		return len(buffers) > 0
+		return buffers.Length() > 0
 	}, 5)
 
 	// part of request is url, so lets factor that in
 	//totalBytesSent = totalBytesSent - len(*eshost)
-	assert.T(t, len(buffers) == 1, fmt.Sprintf("Should have sent one operation but was %d", len(buffers)))
+	assert.T(t, buffers.Length() == 1, fmt.Sprintf("Should have sent one operation but was %d", buffers.Length()))
 	assert.T(t, indexer.NumErrors() == 0 && err == nil, fmt.Sprintf("Should not have any errors. NumErrors: %v, err: %v", indexer.NumErrors(), err))
 	expectedBytes := 129
 	assert.T(t, totalBytesSent == expectedBytes, fmt.Sprintf("Should have sent %v bytes but was %v", expectedBytes, totalBytesSent))
 
 	err = indexer.Index(testIndex, "user", "2", "", "", nil, data)
 	waitFor(func() bool {
-		return len(buffers) > 1
+		return buffers.Length() > 1
 	}, 5)
 
 	// this will test to ensure that Flush actually catches a doc
 	indexer.Flush()
 	totalBytesSent = totalBytesSent - len(*eshost)
 	assert.T(t, err == nil, fmt.Sprintf("Should have nil error  =%v", err))
-	assert.T(t, len(buffers) == 2, fmt.Sprintf("Should have another buffer ct=%d", len(buffers)))
+	assert.T(t, buffers.Length() == 2, fmt.Sprintf("Should have another buffer ct=%d", buffers.Length()))
 
 	assert.T(t, indexer.NumErrors() == 0, fmt.Sprintf("Should not have any errors %d", indexer.NumErrors()))
 	expectedBytes = 220
@@ -160,7 +183,7 @@ func TestWithoutRefreshParam(t *testing.T) {
 // currently broken in drone.io
 func XXXTestBulkUpdate(t *testing.T) {
 	var (
-		buffers        = make([]*bytes.Buffer, 0)
+		buffers        = NewSharedBuffer()
 		totalBytesSent int
 		messageSets    int
 	)
@@ -172,7 +195,7 @@ func XXXTestBulkUpdate(t *testing.T) {
 	indexer.Sender = func(buf *bytes.Buffer) error {
 		messageSets += 1
 		totalBytesSent += buf.Len()
-		buffers = append(buffers, buf)
+		buffers.Append(buf)
 		return indexer.Send(buf)
 	}
 	indexer.Start()
@@ -196,7 +219,7 @@ func XXXTestBulkUpdate(t *testing.T) {
 	//	indexer.Flush()
 
 	waitFor(func() bool {
-		return len(buffers) > 0
+		return buffers.Length() > 0
 	}, 5)
 
 	indexer.Stop()

From e554b05eb1d18256b6522140a23ce3b9b21df37f Mon Sep 17 00:00:00 2001
From: woodsaj <anthony@monkey.id.au>
Date: Tue, 29 Sep 2015 15:56:15 +0800
Subject: [PATCH 68/81] sync access to sender bytes payload tracker.

issue #241
adds a sync.Mutex to prevent race conditions when reading and
writting to the the sentBytes var shared between goroutines.
---
 lib/corebulk_test.go | 20 +++++++++++++-------
 1 file changed, 13 insertions(+), 7 deletions(-)

diff --git a/lib/corebulk_test.go b/lib/corebulk_test.go
index 7226623f..ddee0f9d 100644
--- a/lib/corebulk_test.go
+++ b/lib/corebulk_test.go
@@ -130,11 +130,12 @@ func TestBulkIndexerBasic(t *testing.T) {
 }
 
 func TestRefreshParam(t *testing.T) {
-	var requrl *url.URL
+	requrlChan := make(chan *url.URL, 1)
 	InitTests(true)
 	c := NewTestConn()
 	c.RequestTracer = func(method, urlStr, body string) {
-		requrl, _ = url.Parse(urlStr)
+		requrl, _ := url.Parse(urlStr)
+		requrlChan <- requrl
 	}
 	date := time.Unix(1257894000, 0)
 	data := map[string]interface{}{"name": "smurfs", "age": 22, "date": date}
@@ -151,16 +152,17 @@ func TestRefreshParam(t *testing.T) {
 	<-time.After(time.Millisecond * 200)
 	//	indexer.Flush()
 	indexer.Stop()
-
+	requrl := <-requrlChan
 	assert.T(t, requrl.Query().Get("refresh") == "true", "Should have set refresh query param to true")
 }
 
 func TestWithoutRefreshParam(t *testing.T) {
-	var requrl *url.URL
+	requrlChan := make(chan *url.URL, 1)
 	InitTests(true)
 	c := NewTestConn()
 	c.RequestTracer = func(method, urlStr, body string) {
-		requrl, _ = url.Parse(urlStr)
+		requrl, _ := url.Parse(urlStr)
+		requrlChan <- requrl
 	}
 	date := time.Unix(1257894000, 0)
 	data := map[string]interface{}{"name": "smurfs", "age": 22, "date": date}
@@ -176,7 +178,7 @@ func TestWithoutRefreshParam(t *testing.T) {
 	<-time.After(time.Millisecond * 200)
 	//	indexer.Flush()
 	indexer.Stop()
-
+	requrl := <-requrlChan
 	assert.T(t, requrl.Query().Get("refresh") == "false", "Should have set refresh query param to false")
 }
 
@@ -270,13 +272,15 @@ func TestBulkSmallBatch(t *testing.T) {
 
 func TestBulkDelete(t *testing.T) {
 	InitTests(true)
-
+	var lock sync.Mutex
 	c := NewTestConn()
 	indexer := c.NewBulkIndexer(1)
 	sentBytes := []byte{}
 
 	indexer.Sender = func(buf *bytes.Buffer) error {
+		lock.Lock()
 		sentBytes = append(sentBytes, buf.Bytes()...)
+		lock.Unlock()
 		return nil
 	}
 
@@ -287,7 +291,9 @@ func TestBulkDelete(t *testing.T) {
 	indexer.Flush()
 	indexer.Stop()
 
+	lock.Lock()
 	sent := string(sentBytes)
+	lock.Unlock()
 
 	expected := `{"delete":{"_index":"fake","_type":"fake_type","_id":"1"}}
 `

From b5e5aca7f5480af0bbcd1ef1e69048ca22c514ea Mon Sep 17 00:00:00 2001
From: woodsaj <anthony@monkey.id.au>
Date: Tue, 29 Sep 2015 15:58:03 +0800
Subject: [PATCH 69/81] address race condition waiting for goroutines to
 complete.

issue #241
Replaces the abstracted approach of using channels to notify
when a waitgroup is done. Instead we just call wg.Wait() and
block until it is complete.
---
 lib/corebulk.go | 12 +-----------
 1 file changed, 1 insertion(+), 11 deletions(-)

diff --git a/lib/corebulk.go b/lib/corebulk.go
index bc38c9e2..72153dea 100644
--- a/lib/corebulk.go
+++ b/lib/corebulk.go
@@ -160,16 +160,6 @@ func (b *BulkIndexer) Stop() {
 	}
 }
 
-// Make a channel that will close when the given WaitGroup is done.
-func wgChan(wg *sync.WaitGroup) <-chan interface{} {
-	ch := make(chan interface{})
-	go func() {
-		wg.Wait()
-		close(ch)
-	}()
-	return ch
-}
-
 func (b *BulkIndexer) PendingDocuments() int {
 	return b.docCt
 }
@@ -280,7 +270,7 @@ func (b *BulkIndexer) shutdown() {
 	close(b.timerDoneChan)
 	close(b.sendBuf)
 	close(b.bulkChannel)
-	<-wgChan(b.sendWg)
+	b.sendWg.Wait()
 }
 
 // The index bulk API adds or updates a typed JSON document to a specific index, making it searchable.

From 08c6449d641cd36a90acbc6df160565841e692fc Mon Sep 17 00:00:00 2001
From: woodsaj <anthony@monkey.id.au>
Date: Tue, 29 Sep 2015 16:05:40 +0800
Subject: [PATCH 70/81] re-order imports

---
 lib/corebulk_test.go | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/lib/corebulk_test.go b/lib/corebulk_test.go
index ddee0f9d..e352ae33 100644
--- a/lib/corebulk_test.go
+++ b/lib/corebulk_test.go
@@ -17,14 +17,15 @@ import (
 	"encoding/json"
 	"flag"
 	"fmt"
-	"github.com/araddon/gou"
-	"github.com/bmizerany/assert"
 	"log"
 	"net/url"
 	"strconv"
 	"sync"
 	"testing"
 	"time"
+
+	"github.com/araddon/gou"
+	"github.com/bmizerany/assert"
 )
 
 //  go test -bench=".*"

From 4fe38a52a65866f69cd8780ba5a9880e6d4a76d1 Mon Sep 17 00:00:00 2001
From: Levi Corcoran <levi.corcoran@concur.com>
Date: Tue, 29 Sep 2015 17:25:05 -0500
Subject: [PATCH 71/81] refs elastigo #241 / PR #242 - additional fixes to
 WaitGroup usage and making numErrors tracking atomic to prevent races

---
 lib/corebulk.go | 24 +++++++-----------------
 1 file changed, 7 insertions(+), 17 deletions(-)

diff --git a/lib/corebulk.go b/lib/corebulk.go
index bc38c9e2..da4872ae 100644
--- a/lib/corebulk.go
+++ b/lib/corebulk.go
@@ -19,6 +19,7 @@ import (
 	"io"
 	"strconv"
 	"sync"
+	"sync/atomic"
 	"time"
 )
 
@@ -96,7 +97,7 @@ type BulkIndexer struct {
 }
 
 func (b *BulkIndexer) NumErrors() uint64 {
-	return b.numErrors
+	return atomic.LoadUint64(&b.numErrors)
 }
 
 func (c *Conn) NewBulkIndexer(maxConns int) *BulkIndexer {
@@ -160,16 +161,6 @@ func (b *BulkIndexer) Stop() {
 	}
 }
 
-// Make a channel that will close when the given WaitGroup is done.
-func wgChan(wg *sync.WaitGroup) <-chan interface{} {
-	ch := make(chan interface{})
-	go func() {
-		wg.Wait()
-		close(ch)
-	}()
-	return ch
-}
-
 func (b *BulkIndexer) PendingDocuments() int {
 	return b.docCt
 }
@@ -190,9 +181,9 @@ func (b *BulkIndexer) startHttpSender() {
 	// in theory, the whole set will cause a backup all the way to IndexBulk if
 	// we have consumed all maxConns
 	for i := 0; i < b.maxConns; i++ {
+		b.sendWg.Add(1)
 		go func() {
 			for buf := range b.sendBuf {
-				b.sendWg.Add(1)
 				// Copy for the potential re-send.
 				bufCopy := bytes.NewBuffer(buf.Bytes())
 				err := b.Sender(buf)
@@ -207,7 +198,6 @@ func (b *BulkIndexer) startHttpSender() {
 						err = b.Sender(bufCopy)
 						if err == nil {
 							// Successfully re-sent with no error
-							b.sendWg.Done()
 							continue
 						}
 					}
@@ -215,8 +205,8 @@ func (b *BulkIndexer) startHttpSender() {
 						b.ErrorChannel <- &ErrorBuffer{err, buf}
 					}
 				}
-				b.sendWg.Done()
 			}
+			b.sendWg.Done()
 		}()
 	}
 }
@@ -280,7 +270,7 @@ func (b *BulkIndexer) shutdown() {
 	close(b.timerDoneChan)
 	close(b.sendBuf)
 	close(b.bulkChannel)
-	<-wgChan(b.sendWg)
+	b.sendWg.Wait()
 }
 
 // The index bulk API adds or updates a typed JSON document to a specific index, making it searchable.
@@ -344,14 +334,14 @@ func (b *BulkIndexer) Send(buf *bytes.Buffer) error {
 	body, err := b.conn.DoCommand("POST", fmt.Sprintf("/_bulk?refresh=%t", b.Refresh), nil, buf)
 
 	if err != nil {
-		b.numErrors += 1
+		atomic.AddUint64(&b.numErrors, 1)
 		return err
 	}
 	// check for response errors, bulk insert will give 200 OK but then include errors in response
 	jsonErr := json.Unmarshal(body, &response)
 	if jsonErr == nil {
 		if response.Errors {
-			b.numErrors += uint64(len(response.Items))
+			atomic.AddUint64(&b.numErrors, uint64(len(response.Items)))
 			return fmt.Errorf("Bulk Insertion Error. Failed item count [%d]", len(response.Items))
 		}
 	}

From f25e9e77adbf116a525b431b842b17bd4c602ba8 Mon Sep 17 00:00:00 2001
From: Chris Vanderschuere <chris@offlinelabs.com>
Date: Tue, 6 Oct 2015 00:01:02 -0700
Subject: [PATCH 72/81] Fixed escape to properly handle int32, int64, and
 float32; add corresponding tests

---
 lib/request.go      | 10 +++++++---
 lib/request_test.go | 20 +++++++++++++++++---
 2 files changed, 24 insertions(+), 6 deletions(-)

diff --git a/lib/request.go b/lib/request.go
index c74481d1..4f72955e 100644
--- a/lib/request.go
+++ b/lib/request.go
@@ -15,13 +15,15 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
-	hostpool "github.com/bitly/go-hostpool"
 	"io"
 	"io/ioutil"
 	"net/http"
 	"net/url"
+	"reflect"
 	"strconv"
 	"strings"
+
+	hostpool "github.com/bitly/go-hostpool"
 )
 
 type Request struct {
@@ -107,9 +109,11 @@ func Escape(args map[string]interface{}) (s string, err error) {
 		case bool:
 			vals.Add(key, strconv.FormatBool(v))
 		case int, int32, int64:
-			vals.Add(key, strconv.Itoa(v.(int)))
+			vInt := reflect.ValueOf(v).Int()
+			vals.Add(key, strconv.FormatInt(vInt, 10))
 		case float32, float64:
-			vals.Add(key, strconv.FormatFloat(v.(float64), 'f', -1, 64))
+			vFloat := reflect.ValueOf(v).Float()
+			vals.Add(key, strconv.FormatFloat(vFloat, 'f', -1, 32))
 		case []string:
 			vals.Add(key, strings.Join(v, ","))
 		default:
diff --git a/lib/request_test.go b/lib/request_test.go
index 754d11ca..0f2a5909 100644
--- a/lib/request_test.go
+++ b/lib/request_test.go
@@ -28,12 +28,26 @@ func TestQueryString(t *testing.T) {
 	assert.T(t, s == exp && err == nil, fmt.Sprintf("Expected %s, got: %s", exp, s))
 
 	// Test single int argument
-	s, err = Escape(map[string]interface{}{"foo": 1})
+	s, err = Escape(map[string]interface{}{"foo": int(1)})
 	exp = "foo=1"
 	assert.T(t, s == exp && err == nil, fmt.Sprintf("Expected %s, got: %s", exp, s))
 
-	// Test single float argument
-	s, err = Escape(map[string]interface{}{"foo": 3.141592})
+	// Test single int64 argument
+	s, err = Escape(map[string]interface{}{"foo": int64(1)})
+	exp = "foo=1"
+	assert.T(t, s == exp && err == nil, fmt.Sprintf("Expected %s, got: %s", exp, s))
+
+	// Test single int64 argument
+	s, err = Escape(map[string]interface{}{"foo": int32(1)})
+	exp = "foo=1"
+	assert.T(t, s == exp && err == nil, fmt.Sprintf("Expected %s, got: %s", exp, s))
+
+	// Test single float64 argument
+	s, err = Escape(map[string]interface{}{"foo": float64(3.141592)})
+	exp = "foo=3.141592"
+	assert.T(t, s == exp && err == nil, fmt.Sprintf("Expected %s, got: %s", exp, s))
+
+	s, err = Escape(map[string]interface{}{"foo": float32(3.141592)})
 	exp = "foo=3.141592"
 	assert.T(t, s == exp && err == nil, fmt.Sprintf("Expected %s, got: %s", exp, s))
 

From 6f948380b5dc800a441e2cfce74efc2171cd9242 Mon Sep 17 00:00:00 2001
From: Chris Vanderschuere <chris@offlinelabs.com>
Date: Tue, 6 Oct 2015 00:05:42 -0700
Subject: [PATCH 73/81] Fixed comments

---
 lib/request_test.go | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/request_test.go b/lib/request_test.go
index 0f2a5909..fa173d6e 100644
--- a/lib/request_test.go
+++ b/lib/request_test.go
@@ -37,7 +37,7 @@ func TestQueryString(t *testing.T) {
 	exp = "foo=1"
 	assert.T(t, s == exp && err == nil, fmt.Sprintf("Expected %s, got: %s", exp, s))
 
-	// Test single int64 argument
+	// Test single int32 argument
 	s, err = Escape(map[string]interface{}{"foo": int32(1)})
 	exp = "foo=1"
 	assert.T(t, s == exp && err == nil, fmt.Sprintf("Expected %s, got: %s", exp, s))
@@ -47,6 +47,7 @@ func TestQueryString(t *testing.T) {
 	exp = "foo=3.141592"
 	assert.T(t, s == exp && err == nil, fmt.Sprintf("Expected %s, got: %s", exp, s))
 
+	// Test single float32 argument
 	s, err = Escape(map[string]interface{}{"foo": float32(3.141592)})
 	exp = "foo=3.141592"
 	assert.T(t, s == exp && err == nil, fmt.Sprintf("Expected %s, got: %s", exp, s))

From ee761c2a85dc232a67c766b218192309bee632ca Mon Sep 17 00:00:00 2001
From: Marcus Barczak <mbarczak@gmail.com>
Date: Sun, 25 Oct 2015 20:18:45 -0700
Subject: [PATCH 74/81] explicitly stop ticker to avoid memory leaks

---
 lib/corebulk.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/corebulk.go b/lib/corebulk.go
index 5010f843..f1f0df29 100644
--- a/lib/corebulk.go
+++ b/lib/corebulk.go
@@ -237,6 +237,7 @@ func (b *BulkIndexer) startTimer() {
 				b.mu.Unlock()
 			case <-b.timerDoneChan:
 				// shutdown this go routine
+				ticker.Stop()
 				return
 			}
 

From e9e50197ea6bc63892b7af60ff483769f3feceb3 Mon Sep 17 00:00:00 2001
From: Mariano Cano <mariano@betable.com>
Date: Mon, 16 Nov 2015 18:08:32 -0800
Subject: [PATCH 75/81] Make sure to send a valid buffer to the ErrorChannel.

---
 lib/corebulk.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/corebulk.go b/lib/corebulk.go
index 650e7331..35a194fb 100644
--- a/lib/corebulk.go
+++ b/lib/corebulk.go
@@ -193,6 +193,7 @@ func (b *BulkIndexer) startHttpSender() {
 				//  2.  Retry then return error and let runner decide
 				//  3.  Retry, then log to disk?   retry later?
 				if err != nil {
+					buf = bytes.NewBuffer(bufCopy.Bytes())
 					if b.RetryForSeconds > 0 {
 						time.Sleep(time.Second * time.Duration(b.RetryForSeconds))
 						err = b.Sender(bufCopy)

From 41d3e4073476d775e06bf118a03c4473b31ed135 Mon Sep 17 00:00:00 2001
From: Mike Yoon <mike@yaki.org>
Date: Tue, 24 Nov 2015 23:16:25 -0800
Subject: [PATCH 76/81] Revert vagrantfile change

---
 Vagrantfile | 37 ++++++++++++++-----------------------
 1 file changed, 14 insertions(+), 23 deletions(-)

diff --git a/Vagrantfile b/Vagrantfile
index ad8fc4cb..cbd1f563 100644
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -2,37 +2,28 @@
 # vi: set ft=ruby :
 
 Vagrant.configure("2") do |config|
-  config.vm.box = "ubuntu/trusty64"
-  config.vm.network :forwarded_port, guest: 9200, host: 9200, auto_correct: true
+  config.vm.box = "lucid64"
+  config.vm.box_url = "http://files.vagrantup.com/lucid64.box"
   config.vm.network :forwarded_port, guest: 9300, host: 9300, auto_correct: true
-  #config.vm.provision :shell, :inline => "curl -L https://www.chef.io/chef/install.sh | sudo bash"
-
-  config.vm.provision "shell", inline: <<-SHELL
-    wget -qO - https://packages.elasticsearch.org/GPG-KEY-elasticsearch | sudo apt-key add -
-    sudo apt-add-repository 'deb http://packages.elasticsearch.org/elasticsearch/1.4/debian stable main'
-    sudo apt-get update
-    sudo apt-get install -y openjdk-7-jre
-    sudo apt-get install -y elasticsearch
-    sudo /etc/init.d/elasticsearch start
-  SHELL
+  config.vm.provision :shell, :inline => "gem install chef --version 10.26.0 --no-rdoc --no-ri --conservative"
 
   config.vm.provider :virtualbox do |vb|
     vb.gui = false
     vb.customize ["modifyvm", :id, "--memory", "1024"]
     vb.customize ["modifyvm", :id, "--cpus", "1"]
-    # This allows symlinks to be created within the /vagrant root directory, 
+    # This allows symlinks to be created within the /vagrant root directory,
     # which is something librarian-puppet needs to be able to do. This might
     # be enabled by default depending on what version of VirtualBox is used.
     vb.customize ["setextradata", :id, "VBoxInternal2/SharedFoldersEnableSymlinksCreate/v-root", "1"]
   end
-  #config.vm.provision :chef_solo do |chef|
-  #  chef.cookbooks_path = "cookbooks"
-  #  chef.add_recipe("apt")
-  #  chef.add_recipe("java")
-  #  chef.add_recipe("elasticsearch")
-  #  chef.add_recipe("git")
-  #  chef.add_recipe("mercurial")
-  #  chef.add_recipe("build-essential")
-  #  chef.add_recipe("golang")
-  #end
+  config.vm.provision :chef_solo do |chef|
+    chef.cookbooks_path = "cookbooks"
+    chef.add_recipe("apt")
+    chef.add_recipe("java")
+    chef.add_recipe("elasticsearch")
+    chef.add_recipe("git")
+    chef.add_recipe("mercurial")
+    chef.add_recipe("build-essential")
+    chef.add_recipe("golang")
+  end
 end
\ No newline at end of file

From 54b3449f77b9861aa7cf5ddcbc21d2bc5abca42d Mon Sep 17 00:00:00 2001
From: Mike Yoon <mike@yaki.org>
Date: Wed, 25 Nov 2015 00:22:37 -0800
Subject: [PATCH 77/81] Lint and add godoc to searchfilter

---
 lib/searchfilter.go      | 107 ++++++++++++++++++++++++++++++---------
 lib/searchfilter_test.go |  10 ++--
 lib/searchquery.go       |  10 ++--
 lib/searchsearch.go      |   1 +
 lib/searchsearch_test.go |   2 +-
 5 files changed, 97 insertions(+), 33 deletions(-)

diff --git a/lib/searchfilter.go b/lib/searchfilter.go
index b4323553..3d10ab57 100644
--- a/lib/searchfilter.go
+++ b/lib/searchfilter.go
@@ -14,38 +14,50 @@ package elastigo
 import (
 	"encoding/json"
 	"fmt"
-	. "github.com/araddon/gou"
+	"github.com/araddon/gou"
 )
 
 var (
-	_ = DEBUG
+	_ = gou.DEBUG
 )
 
-// A bool (and/or) clause
+// BoolClause represents aa bool (and/or) clause for use with FilterWrap
+// Legacy, use new FilterOp functions instead
 type BoolClause string
 
+// TermExecutionMode refers to how a terms (not term) filter should behave
+// The acceptable options are all prefixed with TEM
+// See https://www.elastic.co/guide/en/elasticsearch/reference/1.5/query-dsl-terms-filter.html
 type TermExecutionMode string
 
 const (
-	TEM_DEFAULT TermExecutionMode = ""
-	TEM_PLAIN                     = "plain"
-	TEM_FIELD                     = "field_data"
-	TEM_BOOL                      = "bool"
-	TEM_AND                       = "and"
-	TEM_OR                        = "or"
+	// TEMDefault default ES term filter behavior (plain)
+	TEMDefault TermExecutionMode = ""
+	// TEMPlain default ES term filter behavior
+	TEMPlain TermExecutionMode = "plain"
+	// TEMField field_data execution mode
+	TEMField TermExecutionMode = "field_data"
+	// TEMBool bool execution mode
+	TEMBool TermExecutionMode = "bool"
+	// TEMAnd and execution mode
+	TEMAnd TermExecutionMode = "and"
+	// TEMOr or execution mode
+	TEMOr TermExecutionMode = "or"
 )
 
-// Filter clause is either a boolClause or FilterOp
+// FilterClause is either a boolClause or FilterOp for use with FilterWrap
 type FilterClause interface {
 	String() string
 }
 
-// A wrapper to allow for custom serialization
+// FilterWrap is the legacy struct for chaining multiple filters with a bool
+// Legacy, use new FilterOp functions instead
 type FilterWrap struct {
 	boolClause string
 	filters    []interface{}
 }
 
+// NewFilterWrap creates a new FilterWrap struct
 func NewFilterWrap() *FilterWrap {
 	return &FilterWrap{filters: make([]interface{}, 0), boolClause: "and"}
 }
@@ -56,6 +68,7 @@ func (f *FilterWrap) String() string {
 
 // Bool sets the type of boolean filter to use.
 // Accepted values are "and" and "or".
+// Legacy, use new FilterOp functions instead
 func (f *FilterWrap) Bool(s string) {
 	f.boolClause = s
 }
@@ -73,7 +86,7 @@ func (f *FilterWrap) addFilters(fl []interface{}) {
 	f.filters = append(f.filters, fl...)
 }
 
-// Custom marshalling to support the query dsl
+// MarshalJSON override for FilterWrap to match the expected ES syntax with the bool at the root
 func (f *FilterWrap) MarshalJSON() ([]byte, error) {
 	var root interface{}
 	if len(f.filters) > 1 {
@@ -129,7 +142,10 @@ func (f *FilterWrap) MarshalJSON() ([]byte, error) {
 
 */
 
-// Filter Operation
+
+// Filter creates a blank FilterOp that can be customized with further function calls
+// This is the starting point for constructing any filter query
+// Examples:
 //
 //   Filter().Term("user","kimchy")
 //
@@ -137,23 +153,26 @@ func (f *FilterWrap) MarshalJSON() ([]byte, error) {
 //   Filter().Terms("user", "kimchy", "elasticsearch")
 //
 //   Filter().Exists("repository.name")
-//
 func Filter() *FilterOp {
 	return &FilterOp{}
 }
 
+// CompoundFilter creates a complete FilterWrap given multiple filters
+// Legacy, use new FilterOp functions instead
 func CompoundFilter(fl ...interface{}) *FilterWrap {
 	FilterVal := NewFilterWrap()
 	FilterVal.addFilters(fl)
 	return FilterVal
 }
 
+// FilterOp holds all the information for a filter query
+// Properties should not be set directly, but instead via the fluent-style API.
 type FilterOp struct {
 	TermsMap        map[string]interface{} `json:"terms,omitempty"`
 	TermMap         map[string]interface{} `json:"term,omitempty"`
 	RangeMap        map[string]RangeFilter `json:"range,omitempty"`
-	ExistsProp      *PropertyPathMarker    `json:"exists,omitempty"`
-	MissingProp     *PropertyPathMarker    `json:"missing,omitempty"`
+	ExistsProp      *propertyPathMarker    `json:"exists,omitempty"`
+	MissingProp     *propertyPathMarker    `json:"missing,omitempty"`
 	AndFilters      []*FilterOp            `json:"and,omitempty"`
 	OrFilters       []*FilterOp            `json:"or,omitempty"`
 	NotFilters      []*FilterOp            `json:"not,omitempty"`
@@ -165,29 +184,47 @@ type FilterOp struct {
 	GeoDistRangeMap map[string]interface{} `json:"geo_distance_range,omitempty"`
 }
 
-type PropertyPathMarker struct {
+type propertyPathMarker struct {
 	Field string `json:"field"`
 }
 
+// LimitFilter holds the Limit filter information
+// Value: number of documents to limit
 type LimitFilter struct {
 	Value int `json:"value"`
 }
 
+// TypeFilter filters on the document type
+// Value: the document type to filter
 type TypeFilter struct {
 	Value string `json:"value"`
 }
 
+// IdsFilter holds the type and ids (on the _id field) to filter
+// Type: a string or an array of string types. Optional.
+// Values: Array of ids to match
 type IdsFilter struct {
 	Type   []string      `json:"type,omitempty"`
 	Values []interface{} `json:"values,omitempty"`
 }
 
+// ScriptFilter will filter using a custom javascript function
+// Script: the javascript to run
+// Params: map of custom parameters to pass into the function (JSON), if any
+// IsCached: whether to cache the results of the filter
 type ScriptFilter struct {
 	Script   string                 `json:"script"`
 	Params   map[string]interface{} `json:"params,omitempty"`
 	IsCached bool                   `json:"_cache,omitempty"`
 }
 
+// RangeFilter filters given a range. Parameters need to be comparable for ES to accept.
+// Only a minimum of one comparison parameter is required. You probably shouldn't mix GT and GTE parameters.
+// Gte: the greater-than-or-equal to value. Should be a number or date.
+// Lte: the less-than-or-equal to value. Should be a number or date.
+// Gt: the greater-than value. Should be a number or date.
+// Lt: the less-than value. Should be a number or date.
+// TimeZone: the timezone to use (+|-h:mm format), if the other parameters are dates
 type RangeFilter struct {
 	Gte      interface{} `json:"gte,omitempty"`
 	Lte      interface{} `json:"lte,omitempty"`
@@ -196,11 +233,14 @@ type RangeFilter struct {
 	TimeZone string      `json:"time_zone,omitempty"` //Ideally this would be an int
 }
 
+// GeoLocation holds the coordinates for a geo query. Currently hashes are not supported.
 type GeoLocation struct {
 	Latitude  float32 `json:"lat"`
 	Longitude float32 `json:"lon"`
 }
 
+// GeoField holds a GeoLocation and a field to match to.
+// This exists so the struct will match the ES schema.
 type GeoField struct {
 	GeoLocation
 	Field string
@@ -208,6 +248,7 @@ type GeoField struct {
 
 // Term will add a term to the filter.
 // Multiple Term filters can be added, and ES will OR them.
+// If the term already exists in the FilterOp, the value will be overridden.
 func (f *FilterOp) Term(field string, value interface{}) *FilterOp {
 	if len(f.TermMap) == 0 {
 		f.TermMap = make(map[string]interface{})
@@ -217,6 +258,7 @@ func (f *FilterOp) Term(field string, value interface{}) *FilterOp {
 	return f
 }
 
+// And will add an AND op to the filter. One or more FilterOps can be passed in.
 func (f *FilterOp) And(filters ...*FilterOp) *FilterOp {
 	if len(f.AndFilters) == 0 {
 		f.AndFilters = filters[:]
@@ -227,6 +269,7 @@ func (f *FilterOp) And(filters ...*FilterOp) *FilterOp {
 	return f
 }
 
+// Or will add an OR op to the filter. One or more FilterOps can be passed in.
 func (f *FilterOp) Or(filters ...*FilterOp) *FilterOp {
 	if len(f.OrFilters) == 0 {
 		f.OrFilters = filters[:]
@@ -237,6 +280,7 @@ func (f *FilterOp) Or(filters ...*FilterOp) *FilterOp {
 	return f
 }
 
+// Not will add a NOT op to the filter. One or more FilterOps can be passed in.
 func (f *FilterOp) Not(filters ...*FilterOp) *FilterOp {
 	if len(f.NotFilters) == 0 {
 		f.NotFilters = filters[:]
@@ -248,6 +292,9 @@ func (f *FilterOp) Not(filters ...*FilterOp) *FilterOp {
 	return f
 }
 
+// GeoDistance will add a GEO DISTANCE op to the filter.
+// distance: distance in ES distance format, i.e. "100km" or "100mi".
+// fields: an array of GeoField origin coordinates. Only one coordinate needs to match.
 func (f *FilterOp) GeoDistance(distance string, fields ...GeoField) *FilterOp {
 	f.GeoDistMap = make(map[string]interface{})
 	f.GeoDistMap["distance"] = distance
@@ -258,6 +305,10 @@ func (f *FilterOp) GeoDistance(distance string, fields ...GeoField) *FilterOp {
 	return f
 }
 
+// GeoDistanceRange will add a GEO DISTANCE RANGE op to the filter.
+// from: minimum distance in ES distance format, i.e. "100km" or "100mi".
+// to: maximum distance in ES distance format, i.e. "100km" or "100mi".
+// fields: an array of GeoField origin coordinates. Only one coor
 func (f *FilterOp) GeoDistanceRange(from string, to string, fields ...GeoField) *FilterOp {
 	f.GeoDistRangeMap = make(map[string]interface{})
 	f.GeoDistRangeMap["from"] = from
@@ -270,17 +321,18 @@ func (f *FilterOp) GeoDistanceRange(from string, to string, fields ...GeoField)
 	return f
 }
 
-// Helper to create values for the GeoDistance filters
+// NewGeoField is a helper function to create values for the GeoDistance filters
 func NewGeoField(field string, latitude float32, longitude float32) GeoField {
 	return GeoField{
 		GeoLocation: GeoLocation{Latitude: latitude, Longitude: longitude},
 		Field:       field}
 }
 
-// Filter Terms
-//
-//   Filter().Terms("user","kimchy","stuff")
-//	 Note: you can only have one terms clause in a filter. Use a bool filter to combine
+// Terms adds a TERMS op to the filter.
+// field: the document field
+// executionMode Term execution mode, starts with TEM
+// values: array of values to match
+// Note: you can only have one terms clause in a filter. Use a bool filter to combine multiple.
 func (f *FilterOp) Terms(field string, executionMode TermExecutionMode, values ...interface{}) *FilterOp {
 	//You can only have one terms in a filter
 	f.TermsMap = make(map[string]interface{})
@@ -295,6 +347,7 @@ func (f *FilterOp) Terms(field string, executionMode TermExecutionMode, values .
 }
 
 // Range adds a range filter for the given field.
+// See the RangeFilter struct documentation for information about the parameters.
 func (f *FilterOp) Range(field string, gte interface{},
 	gt interface{}, lte interface{}, lt interface{}, timeZone string) *FilterOp {
 
@@ -312,31 +365,37 @@ func (f *FilterOp) Range(field string, gte interface{},
 	return f
 }
 
+// Type adds a TYPE op to the filter.
 func (f *FilterOp) Type(fieldType string) *FilterOp {
 	f.TypeProp = &TypeFilter{Value: fieldType}
 	return f
 }
 
+// Ids adds a IDS op to the filter.
 func (f *FilterOp) Ids(ids ...interface{}) *FilterOp {
 	f.IdsProp = &IdsFilter{Values: ids}
 	return f
 }
 
+// IdsByTypes adds a IDS op to the filter, but also allows passing in an array of types for the query.
 func (f *FilterOp) IdsByTypes(types []string, ids ...interface{}) *FilterOp {
 	f.IdsProp = &IdsFilter{Type: types, Values: ids}
 	return f
 }
 
+// Exists adds an EXISTS op to the filter.
 func (f *FilterOp) Exists(field string) *FilterOp {
-	f.ExistsProp = &PropertyPathMarker{Field: field}
+	f.ExistsProp = &propertyPathMarker{Field: field}
 	return f
 }
 
+// Missing adds an MISSING op to the filter.
 func (f *FilterOp) Missing(field string) *FilterOp {
-	f.MissingProp = &PropertyPathMarker{Field: field}
+	f.MissingProp = &propertyPathMarker{Field: field}
 	return f
 }
 
+// Limit adds an LIMIT op to the filter.
 func (f *FilterOp) Limit(maxResults int) *FilterOp {
 	f.LimitProp = &LimitFilter{Value: maxResults}
 	return f
diff --git a/lib/searchfilter_test.go b/lib/searchfilter_test.go
index 57671bf7..a4931ddc 100644
--- a/lib/searchfilter_test.go
+++ b/lib/searchfilter_test.go
@@ -59,7 +59,7 @@ func TestFilterDsl(t *testing.T) {
 	})
 
 	Convey("Terms filter", t, func() {
-		filter := Filter().Terms("Sample", TEM_AND, "asdf", 123, true)
+		filter := Filter().Terms("Sample", TEMAnd, "asdf", 123, true)
 		actual, err  := GetJson(filter)
 
 		actualTerms := actual["terms"].(map[string]interface{})
@@ -232,7 +232,7 @@ func TestFilters(t *testing.T) {
 
 	Convey("Terms filter", t, func() {
 		qry := Search("oilers").Filter(
-			Filter().Terms("pos", TEM_DEFAULT, "RW", "LW"),
+			Filter().Terms("pos", TEMDefault, "RW", "LW"),
 		)
 		out, err := qry.Result(c)
 		So(err, ShouldBeNil)
@@ -244,7 +244,7 @@ func TestFilters(t *testing.T) {
 	Convey("Filter involving an AND", t, func() {
 		qry := Search("oilers").Filter(
 			Filter().And(
-				Filter().Terms("pos", TEM_DEFAULT, "LW"),
+				Filter().Terms("pos", TEMDefault, "LW"),
 				Filter().Exists("PIM"),
 			),
 		)
@@ -259,7 +259,7 @@ func TestFilters(t *testing.T) {
 
 	Convey("Filterng filter results", t, func() {
 		qry := Search("oilers").Filter(
-			Filter().Terms("pos", TEM_DEFAULT, "LW"),
+			Filter().Terms("pos", TEMDefault, "LW"),
 		)
 		qry.Filter(
 			Filter().Exists("PIM"),
@@ -274,7 +274,7 @@ func TestFilters(t *testing.T) {
 	Convey("Filter involving OR", t, func() {
 		qry := Search("oilers").Filter(
 			Filter().Or(
-				Filter().Terms("pos", TEM_DEFAULT, "G"),
+				Filter().Terms("pos", TEMDefault, "G"),
 				Filter().Range("goals", nil, 80, nil, nil, ""),
 			),
 		)
diff --git a/lib/searchquery.go b/lib/searchquery.go
index 8a8630d2..dd01ed71 100644
--- a/lib/searchquery.go
+++ b/lib/searchquery.go
@@ -98,14 +98,14 @@ func (q *QueryDsl) All() *QueryDsl {
 	return q
 }
 
-// Limit the query to this range
+// Range adds a RANGE FilterOp to the search query
+// Legacy. Use the Filter() function instead
 func (q *QueryDsl) Range(fop *FilterOp) *QueryDsl {
 	if q.FilterVal == nil {
 		q.FilterVal = fop
 		return q
 	}
-	// TODO:  this is not valid, refactor
-	//q.FilterVal.Add(fop)
+
 	return q
 }
 
@@ -144,6 +144,8 @@ func (q *QueryDsl) Qs(qs *QueryString) *QueryDsl {
 	return q
 }
 
+// SetLenient sets whether the query should ignore format based failures,
+// such as passing in text to a number field.
 func (q *QueryDsl) SetLenient(lenient bool) *QueryDsl {
 	q.QueryEmbed.Qs.Lenient = lenient
 	return q
@@ -213,6 +215,8 @@ type QueryString struct {
 	//_missing_:field1,
 }
 
+//I don't know how any of the Term stuff below is supposed to work. -mikeyoon
+
 // Generic Term based (used in query, facet, filter)
 type Term struct {
 	Terms     Terms       `json:"terms,omitempty"`
diff --git a/lib/searchsearch.go b/lib/searchsearch.go
index 4b76e4f1..c921ae5a 100644
--- a/lib/searchsearch.go
+++ b/lib/searchsearch.go
@@ -174,6 +174,7 @@ func (s *SearchDsl) Query(q *QueryDsl) *SearchDsl {
 //         Filter().Exists("repository.name"),
 //         Filter().Terms("repository.has_wiki", true)
 //     )
+
 func (s *SearchDsl) Filter(fl *FilterOp) *SearchDsl {
 	s.FilterVal = fl
 	return s
diff --git a/lib/searchsearch_test.go b/lib/searchsearch_test.go
index b09420f0..81f11b3a 100644
--- a/lib/searchsearch_test.go
+++ b/lib/searchsearch_test.go
@@ -217,7 +217,7 @@ func TestSearch(t *testing.T) {
 		out, err := Search("oilers").Size("25").Query(
 			Query().Fields("name", "*d*", "", ""),
 		).Filter(
-			Filter().Terms("teams", TEM_DEFAULT, "STL"),
+			Filter().Terms("teams", TEMDefault, "STL"),
 		).Result(c)
 		So(err, ShouldBeNil)
 		So(out, ShouldNotBeNil)

From ba181e4e2cbeecf0bf2772fde79a6226c11059e2 Mon Sep 17 00:00:00 2001
From: Gregory Eremin <g@erem.in>
Date: Thu, 10 Dec 2015 16:22:01 +0300
Subject: [PATCH 78/81] Parse `sort` field in search results

---
 lib/coresearch.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/coresearch.go b/lib/coresearch.go
index 71eb2a01..0d83c9c9 100644
--- a/lib/coresearch.go
+++ b/lib/coresearch.go
@@ -209,6 +209,7 @@ type Hit struct {
 	Fields      *json.RawMessage `json:"fields"`           // when a field arg is passed to ES, instead of _source it returns fields
 	Explanation *Explanation     `json:"_explanation,omitempty"`
 	Highlight   *Highlight       `json:"highlight,omitempty"`
+	Sort        []interface{}    `json:"sort,omitempty"`
 }
 
 func (e *Explanation) String(indent string) string {

From b9e3398ca47c934930bdd91fb3496e0a50c98c37 Mon Sep 17 00:00:00 2001
From: Shawn Smith <shawnpsmith@gmail.com>
Date: Wed, 3 Feb 2016 16:08:36 +0900
Subject: [PATCH 79/81] fix typo

---
 lib/coreget.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/coreget.go b/lib/coreget.go
index 8a04eedd..b0d69958 100644
--- a/lib/coreget.go
+++ b/lib/coreget.go
@@ -102,7 +102,7 @@ func (c *Conn) ExistsBool(index string, _type string, id string, args map[string
 	return httpStatusCode == http.StatusOK, err
 }
 
-// ExistsIndex allows caller to check for the existance of an index or a type using HEAD
+// ExistsIndex allows caller to check for the existence of an index or a type using HEAD
 func (c *Conn) ExistsIndex(index string, _type string, args map[string]interface{}) (bool, error) {
 	var url string
 

From d43e429ce1615d6bab9dc280063c8b2802f10832 Mon Sep 17 00:00:00 2001
From: Shawn Smith <shawnpsmith@gmail.com>
Date: Wed, 3 Feb 2016 16:10:41 +0900
Subject: [PATCH 80/81] fix typo

---
 lib/indicesindicesexists.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/indicesindicesexists.go b/lib/indicesindicesexists.go
index 5946769f..b1a7ea30 100644
--- a/lib/indicesindicesexists.go
+++ b/lib/indicesindicesexists.go
@@ -16,7 +16,7 @@ import (
 	"strings"
 )
 
-// IndicesExists checks for the existance of indices. uses RecordNotFound message if it doesn't exist and
+// IndicesExists checks for the existence of indices. uses RecordNotFound message if it doesn't exist and
 // "no error" situation if it exists. If there is some other error, gives the error and says it exists
 // just in case
 // see http://www.elasticsearch.org/guide/reference/api/admin-indices-indices-exists/

From ac1a489552f844fc0612818eeafe3dda69215058 Mon Sep 17 00:00:00 2001
From: Shawn Smith <shawnpsmith@gmail.com>
Date: Wed, 3 Feb 2016 16:11:37 +0900
Subject: [PATCH 81/81] fix typo

---
 lib/baserequest.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/baserequest.go b/lib/baserequest.go
index 27809ae2..f9a9dcbf 100644
--- a/lib/baserequest.go
+++ b/lib/baserequest.go
@@ -100,7 +100,7 @@ func (e ESError) Error() string {
 	return fmt.Sprintf("%v: %v [%v]", e.When, e.What, e.Code)
 }
 
-// Exists allows the caller to check for the existance of a document using HEAD
+// Exists allows the caller to check for the existence of a document using HEAD
 // This appears to be broken in the current version of elasticsearch 0.19.10, currently
 // returning nothing
 func (c *Conn) Exists(index string, _type string, id string, args map[string]interface{}) (BaseResponse, error) {