Remove maxLineSize limit

Some valid text files contain long lines, but are of interest to search. This
CR just removes IsText returning false due to the long line. This does mean
search results will include long lines, which may not be of interest. That will
be fixed in another CR.

Change-Id: I11fd84dc3e377a477a77c2218f6623bc6c26dba0

indexbuilder.go

@@ -262,7 +262,6 @@ func (b *IndexBuilder) AddFile(name string, content []byte) error {
 }
 
 const maxTrigramCount = 20000
-const maxLineSize = 1000
 
 // IsText returns false if the given contents are probably not source texts.
 func IsText(content []byte) bool {
@@ -272,21 +271,11 @@ func IsText(content []byte) bool {
 
 	trigrams := map[ngram]struct{}{}
 
-	lineSize := 0
-
 	var cur [3]rune
 	for len(content) > 0 {
 		if content[0] == 0 {
 			return false
 		}
-		if content[0] == '\n' {
-			lineSize = 0
-		} else {
-			lineSize++
-		}
-		if lineSize > maxLineSize {
-			return false
-		}
 
 		r, sz := utf8.DecodeRune(content)
 		if r == utf8.RuneError {

toc.go

@@ -34,7 +34,8 @@ const IndexFormatVersion = 15
 // 2: Rank field for shards.
 // 3: Rank documents within shards
 // 4: Dedup file bugfix
-const FeatureVersion = 4
+// 5: Remove max line size limit
+const FeatureVersion = 5
 
 type indexTOC struct {
 	fileContents compoundSection