Merge pull request #11 from jf-tech/fixscanner3

switch delim/esc to []byte, reimplement (Byte)IndexWithEsc, Unescape, add ByteUnescape, and more benches

Author: jf-tech

ios/scanners.go

@@ -29,28 +29,30 @@ const (
 )
 
 // NewScannerByDelim creates a scanner that returns tokens from the source reader separated by a delimiter.
-func NewScannerByDelim(r io.Reader, delim string, flags ScannerByDelimFlag) *bufio.Scanner {
+func NewScannerByDelim(r io.Reader, delim []byte, flags ScannerByDelimFlag) *bufio.Scanner {
 	return NewScannerByDelim2(r, delim, nil, flags)
 }
 
 // NewScannerByDelim2 creates a scanner that returns tokens from the source reader separated by a delimiter, with
 // consideration of potential presence of escaping sequence.
 // Note: the token returned from the scanner will **NOT** do any unescaping, thus keeping the original value.
-func NewScannerByDelim2(r io.Reader, delim string, escape *rune, flags ScannerByDelimFlag) *bufio.Scanner {
+func NewScannerByDelim2(r io.Reader, delim, escape []byte, flags ScannerByDelimFlag) *bufio.Scanner {
+	if escape == nil {
+		return NewScannerByDelim3(r, delim, nil, flags, nil)
+	}
 	return NewScannerByDelim3(r, delim, escape, flags, nil)
 }
 
 // NewScannerByDelim3 creates a scanner that utilizes given buf to avoid/minimize allocation and returns tokens
 // from the source reader separated by a delimiter, with consideration of potential presence of escaping sequence.
 // Note: the token returned from the scanner will **NOT** do any unescaping, thus keeping the original value.
-func NewScannerByDelim3(r io.Reader, delim string, escape *rune, flags ScannerByDelimFlag, buf []byte) *bufio.Scanner {
+func NewScannerByDelim3(r io.Reader, delim, escape []byte, flags ScannerByDelimFlag, buf []byte) *bufio.Scanner {
 	flags &= scannerByDelimValidFlags
 
 	includeDelimLenInToken := len(delim)
 	if flags&ScannerByDelimFlagDropDelimInReturn != 0 {
 		includeDelimLenInToken = 0
 	}
-
 	eofAsDelim := flags&ScannerByDelimFlagEofAsDelim != 0
 
 	scanner := bufio.NewScanner(r)
@@ -60,7 +62,7 @@ func NewScannerByDelim3(r io.Reader, delim string, escape *rune, flags ScannerBy
 			if atEof && len(data) == 0 {
 				return 0, nil, nil
 			}
-			if index := strs.IndexWithEsc(string(data), delim, escape); index >= 0 {
+			if index := strs.ByteIndexWithEsc(data, delim, escape); index >= 0 {
 				return index + len(delim), data[:index+includeDelimLenInToken], nil
 			}
 			if atEof && eofAsDelim {

ios/scanners_test.go

@@ -6,50 +6,48 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
-
-	"github.com/jf-tech/go-corelib/strs"
 )
 
 func TestNewScannerByDelim(t *testing.T) {
 	for _, test := range []struct {
 		name           string
 		input          io.Reader
-		delim          string
+		delim          []byte
 		flags          ScannerByDelimFlag
 		expectedTokens []string
 	}{
 		{
 			name:           "multi-char delim | eof as delim | drop delim",
 			input:          strings.NewReader("abc#123##efg####???##xyz##"),
-			delim:          "##",
+			delim:          []byte("##"),
 			flags:          ScannerByDelimFlagEofAsDelim | ScannerByDelimFlagDropDelimInReturn,
 			expectedTokens: []string{"abc#123", "efg", "", "???", "xyz"},
 		},
 		{
 			name:           "CR LF delim | eof as delim | include delim",
 			input:          strings.NewReader("\r\n\rabc\r"),
-			delim:          "\r\n",
+			delim:          []byte("\r\n"),
 			flags:          ScannerByDelimFlagEofAsDelim | ScannerByDelimFlagIncludeDelimInReturn,
 			expectedTokens: []string{"\r\n", "\rabc\r"},
 		},
 		{
 			name:           "empty reader",
 			input:          strings.NewReader(""),
-			delim:          "*",
+			delim:          []byte("*"),
 			flags:          ScannerByDelimFlagDefault,
 			expectedTokens: []string{},
 		},
 		{
 			name:           "empty token",
 			input:          strings.NewReader("*"),
-			delim:          "*",
+			delim:          []byte("*"),
 			flags:          ScannerByDelimFlagEofNotAsDelim | ScannerByDelimFlagDropDelimInReturn,
 			expectedTokens: []string{""},
 		},
 		{
 			name:           "trailing newlines",
 			input:          strings.NewReader("*\n"),
-			delim:          "*",
+			delim:          []byte("*"),
 			flags:          ScannerByDelimFlagEofAsDelim | ScannerByDelimFlagIncludeDelimInReturn,
 			expectedTokens: []string{"*", "\n"},
 		},
@@ -69,8 +67,8 @@ func TestNewScannerByDelim(t *testing.T) {
 func TestNewScannerByDelim2(t *testing.T) {
 	s := NewScannerByDelim2(
 		strings.NewReader("abc#123##efg####???##xyz##"),
-		"##",
-		strs.RunePtr('?'),
+		[]byte("##"),
+		[]byte("?"),
 		ScannerByDelimFlagEofAsDelim|ScannerByDelimFlagDropDelimInReturn)
 	var tokens []string
 	for s.Scan() {
@@ -84,8 +82,8 @@ func TestNewScannerByDelim3(t *testing.T) {
 	buf := make([]byte, 0, 100)
 	s := NewScannerByDelim3(
 		strings.NewReader("abc#123##efg####???##xyz##"),
-		"##",
-		strs.RunePtr('?'),
+		[]byte("##"),
+		[]byte("?"),
 		ScannerByDelimFlagEofAsDelim|ScannerByDelimFlagDropDelimInReturn,
 		buf)
 	var tokens []string
@@ -96,17 +94,16 @@ func TestNewScannerByDelim3(t *testing.T) {
 	assert.Equal(t, []string{"abc#123", "efg", "", "???##xyz"}, tokens)
 }
 
-// Benchmark shows the benefit of using NewScannerByDelim3 with pre-allocated buf.
-// BenchmarkNewScannerByDelim2-8                            	    5000	    299188 ns/op	 2141712 B/op	     996 allocs/op
-// BenchmarkNewScannerByDelim3-8                            	   30000	     48893 ns/op	     208 B/op	       3 allocs/op
-
-var benchmarkInput = strings.Repeat("abc#", 1000)
+var benchmarkInput = strings.Repeat("abc#", 100000)
+var benchmarkDelim = []byte("#")
+var benchmarkBuf = make([]byte, 1024)
 
+// BenchmarkNewScannerByDelim2-8                            	     500	   2597188 ns/op	    4344 B/op	       5 allocs/op
 func BenchmarkNewScannerByDelim2(b *testing.B) {
 	for i := 0; i < b.N; i++ {
 		s := NewScannerByDelim2(
 			strings.NewReader(benchmarkInput),
-			"#",
+			benchmarkDelim,
 			nil,
 			ScannerByDelimFlagEofAsDelim|ScannerByDelimFlagDropDelimInReturn)
 		for s.Scan() {
@@ -121,15 +118,15 @@ func BenchmarkNewScannerByDelim2(b *testing.B) {
 	}
 }
 
+// BenchmarkNewScannerByDelim3-8                            	     500	   2585825 ns/op	     242 B/op	       3 allocs/op
 func BenchmarkNewScannerByDelim3(b *testing.B) {
-	buf := make([]byte, 0, 10)
 	for i := 0; i < b.N; i++ {
 		s := NewScannerByDelim3(
 			strings.NewReader(benchmarkInput),
-			"#",
+			benchmarkDelim,
 			nil,
 			ScannerByDelimFlagEofAsDelim|ScannerByDelimFlagDropDelimInReturn,
-			buf)
+			benchmarkBuf)
 		for s.Scan() {
 			token := s.Bytes()
 			if len(token) != 3 || token[0] != 'a' || token[1] != 'b' || token[2] != 'c' {