Strip diacritics from sort keys if query is ASCII

Author: deanishe

doc.go

@@ -95,6 +95,8 @@ on your own structs and customising the sort settings.
 The algorithm is based on Forrest Smith's reverse engineering of Sublime
 Text's search: https://blog.forrestthewoods.com/reverse-engineering-sublime-text-s-fuzzy-match-4cffeed33fdb
 
+It additionally strips diacritics from sort keys if the query is ASCII.
+
 
 Sending results to Alfred
 

fuzzy.go

@@ -9,8 +9,13 @@
 package aw
 
 import (
+	"log"
 	"sort"
 	"strings"
+	"unicode"
+
+	"golang.org/x/text/transform"
+	"golang.org/x/text/unicode/norm"
 )
 
 // Default bonuses and penalties for fuzzy sorting. To customise
@@ -24,8 +29,15 @@ const (
 	DefaultLeadingLetterPenalty    = -3.0 // Penalty applied for every letter in string before first match
 	DefaultMaxLeadingLetterPenalty = -9.0 // Maximum penalty for leading letters
 	DefaultUnmatchedLetterPenalty  = -1.0 // Penalty for every letter that doesn't match
+	DefaultStripDiacritics         = true // Strip diacritics from sort keys if query is plain ASCII
 )
 
+var stripper transform.Transformer
+
+func init() {
+	stripper = transform.Chain(norm.NFD, transform.RemoveFunc(isMn), norm.NFC)
+}
+
 // Sortable makes the implementer fuzzy-sortable. It is a superset
 // of sort.Interface (i.e. your struct must also implement sort.Interface).
 type Sortable interface {
@@ -57,6 +69,7 @@ type SortOptions struct {
 	LeadingLetterPenalty    float64 // Penalty applied for every letter in string before first match
 	MaxLeadingLetterPenalty float64 // Maximum penalty for leading letters
 	UnmatchedLetterPenalty  float64 // Penalty for every letter that doesn't match
+	StripDiacritics         bool    //  Strip diacritics from sort keys if query is plain ASCII
 }
 
 // NewSortOptions creates a SortOptions object with the default values.
@@ -68,6 +81,7 @@ func NewSortOptions() *SortOptions {
 		LeadingLetterPenalty:    DefaultLeadingLetterPenalty,
 		MaxLeadingLetterPenalty: DefaultMaxLeadingLetterPenalty,
 		UnmatchedLetterPenalty:  DefaultUnmatchedLetterPenalty,
+		StripDiacritics:         DefaultStripDiacritics,
 	}
 }
 
@@ -77,8 +91,10 @@ type Sorter struct {
 	Data Sortable
 	// Options contains the bonuses and penalties
 	Options *SortOptions
-	// results stores the results of the fuzzy sort
-	results []*Result
+
+	query           string    // Search query
+	stripDiacritics bool      // Whether sort keys need folding
+	results         []*Result // Results of the fuzzy sort
 }
 
 // NewSorter returns a new Sorter. If opts is nil, Sorter is initialised
@@ -116,57 +132,31 @@ func (s *Sorter) Swap(i, j int) {
 
 // Sort sorts data against query.
 func (s *Sorter) Sort(query string) []*Result {
-	if s.results == nil {
-		s.results = make([]*Result, s.Data.Len())
+	s.results = make([]*Result, s.Data.Len())
+	s.query = query
+	if isASCII(query) && s.Options.StripDiacritics {
+		s.stripDiacritics = true
 	}
-
+	// Generate matches for Data, then call sort.Sort()
 	for i := 0; i < s.Data.Len(); i++ {
 		key := s.Data.SortKey(i)
-		// s.matches[i] = match
-		// s.scores[i] = score
-		s.results[i] = Match(key, query, s.Options)
+		s.results[i] = s.Match(key)
 	}
 	sort.Sort(s)
 	return s.results
 }
 
-// Sort sorts data against query. Convenience that creates and
-// uses a Sorter with the default settings.
-func Sort(data Sortable, query string) []*Result {
-	s := NewSorter(data, nil)
-	return s.Sort(query)
-}
-
-// stringSlice implements sort.Interface for []string.
-// It is a helper for SortStrings.
-type stringSlice struct {
-	data []string
-}
-
-// Len etc. implement sort.Interface.
-func (s stringSlice) Len() int           { return len(s.data) }
-func (s stringSlice) Less(i, j int) bool { return s.data[i] < s.data[j] }
-func (s stringSlice) Swap(i, j int)      { s.data[i], s.data[j] = s.data[j], s.data[i] }
-
-// SortKey implements Sortable.
-func (s stringSlice) SortKey(i int) string { return s.data[i] }
-
-// Sort is a convenience method.
-func (s stringSlice) Sort(query string) []*Result { return Sort(s, query) }
-
-// SortStrings is a convenience function.
-func SortStrings(data []string, query string) []*Result {
-	s := stringSlice{data}
-	return s.Sort(query)
-}
-
 // Match scores str against query using fuzzy matching and the specified sort options.
-func Match(str, query string, o *SortOptions) *Result {
+func (s *Sorter) Match(str string) *Result {
+	if s.stripDiacritics {
+		str = stripDiacritics(str)
+	}
+
 	var (
 		match    = false
 		score    = 0.0
 		uStr     = []rune(str)
-		uQuery   = []rune(query)
+		uQuery   = []rune(s.query)
 		strLen   = len(uStr)
 		queryLen = len(uQuery)
 	)
@@ -232,26 +222,26 @@ func Match(str, query string, o *SortOptions) *Result {
 
 			// Apply penalty for letters before first match
 			if queryIdx == 0 {
-				penalty = float64(strIdx) * o.LeadingLetterPenalty
-				if penalty <= o.MaxLeadingLetterPenalty {
-					penalty = o.MaxLeadingLetterPenalty
+				penalty = float64(strIdx) * s.Options.LeadingLetterPenalty
+				if penalty <= s.Options.MaxLeadingLetterPenalty {
+					penalty = s.Options.MaxLeadingLetterPenalty
 				}
 				score += penalty
 			}
 
 			// Apply bonus for consecutive matches
 			if prevMatched {
-				newScore += o.AdjacencyBonus
+				newScore += s.Options.AdjacencyBonus
 			}
 
 			// Apply bonus for match after separator
 			if prevSeparator {
-				newScore += o.SeparatorBonus
+				newScore += s.Options.SeparatorBonus
 			}
 
 			// Apply bonus across camel case boundaries
 			if prevLower && strChar == strUpper && strLower != strUpper {
-				newScore += o.CamelBonus
+				newScore += s.Options.CamelBonus
 			}
 
 			// Update query index if next query letter was matched
@@ -264,7 +254,7 @@ func Match(str, query string, o *SortOptions) *Result {
 			if newScore >= bestLetterScore {
 
 				if bestLetter != "" {
-					score += o.UnmatchedLetterPenalty
+					score += s.Options.UnmatchedLetterPenalty
 				}
 
 				bestLetter = strChar
@@ -274,7 +264,7 @@ func Match(str, query string, o *SortOptions) *Result {
 
 			prevMatched = true
 		} else {
-			score += o.UnmatchedLetterPenalty
+			score += s.Options.UnmatchedLetterPenalty
 			prevMatched = false
 		}
 
@@ -303,5 +293,61 @@ func Match(str, query string, o *SortOptions) *Result {
 	}
 
 	// log.Printf("query=%#v, str=%#v", match=%v, score=%v, query, str, match, score)
-	return &Result{match, query, score, str}
+	return &Result{match, s.query, score, str}
+}
+
+// Sort sorts data against query. Convenience that creates and
+// uses a Sorter with the default settings.
+func Sort(data Sortable, query string) []*Result {
+	s := NewSorter(data, nil)
+	return s.Sort(query)
+}
+
+// stringSlice implements sort.Interface for []string.
+// It is a helper for SortStrings.
+type stringSlice struct {
+	data []string
+}
+
+// Len etc. implement sort.Interface.
+func (s stringSlice) Len() int           { return len(s.data) }
+func (s stringSlice) Less(i, j int) bool { return s.data[i] < s.data[j] }
+func (s stringSlice) Swap(i, j int)      { s.data[i], s.data[j] = s.data[j], s.data[i] }
+
+// SortKey implements Sortable.
+func (s stringSlice) SortKey(i int) string { return s.data[i] }
+
+// Sort is a convenience method.
+func (s stringSlice) Sort(query string) []*Result { return Sort(s, query) }
+
+// SortStrings is a convenience function.
+func SortStrings(data []string, query string) []*Result {
+	s := stringSlice{data}
+	return s.Sort(query)
+}
+
+// Match scores str against query using fuzzy matching and the specified sort options.
+// WARNING: Match creates a new Sorter for every call. Don't use this on
+// large datasets.
+func Match(str, query string, o *SortOptions) *Result {
+	data := stringSlice{[]string{str}}
+	s := NewSorter(data, o)
+	return s.Sort(query)[0]
+}
+
+func isMn(r rune) bool {
+	return unicode.Is(unicode.Mn, r) // Mn: non-spacing mark
+}
+
+func stripDiacritics(s string) string {
+	stripped, _, err := transform.String(stripper, s)
+	if err != nil {
+		log.Printf("Couldn't strip diacritics from `%s`: %s", s, err)
+		return s
+	}
+	return stripped
+}
+
+func isASCII(s string) bool {
+	return stripDiacritics(s) == s
 }

fuzzy_test.go

@@ -191,3 +191,24 @@ func TestFilterFeedback(t *testing.T) {
 		}
 	}
 }
+
+// TestStripDiacritics
+func TestStripDiacritics(t *testing.T) {
+	o := NewSortOptions()
+	// Non-ASCII query and data
+	if r := Match("fün", "fün", o); r.Match == false {
+		t.Fatalf("fün != fün (diacritic stripping on): %v", r)
+	}
+	// Non-ASCII data
+	if r := Match("fün", "fun", o); r.Match == false {
+		t.Fatalf("fun != fün (diacritic stripping on): %v", r)
+	}
+	// No diacritic stripping
+	o.StripDiacritics = false
+	if r := Match("fün", "fün", o); r.Match == false {
+		t.Fatalf("fün != fün (diacritic stripping off): %s", r)
+	}
+	if r := Match("fün", "fun", o); r.Match == true {
+		t.Fatalf("fun != fün (diacritic stripping off): %s", r)
+	}
+}