Modify parsing functions and config structs to accept hcl changes

Signed-off-by: Patrick Van Stee <patrick@vanstee.me>
This commit is contained in:
Patrick Van Stee
2020-04-15 21:00:18 -04:00
parent 87c4bf1df9
commit 4121ae50b5
131 changed files with 45185 additions and 110 deletions

View File

@@ -0,0 +1,19 @@
package textseg
import "unicode/utf8"
// ScanGraphemeClusters is a split function for bufio.Scanner that splits
// on UTF8 sequence boundaries.
//
// This is included largely for completeness, since this behavior is already
// built in to Go when ranging over a string.
func ScanUTF8Sequences(data []byte, atEOF bool) (int, []byte, error) {
if len(data) == 0 {
return 0, nil, nil
}
r, seqLen := utf8.DecodeRune(data)
if r == utf8.RuneError && !atEOF {
return 0, nil, nil
}
return seqLen, data[:seqLen], nil
}