GoLang bufio.NewScanner 详解

上代码

func NewScanner(r io.Reader) *Scanner {
    return &Scanner{
        r:            r,
        split:        ScanLines,
        maxTokenSize: MaxScanTokenSize,
    }
}

函数NewScanner返回一个Scanner,这个返回值来自于函数参数r。他的类型是io.Reader。

type Scanner struct {
    r            io.Reader // The reader provided by the client.
    split        SplitFunc // The function to split the tokens.
    maxTokenSize int       // Maximum size of a token; modified by tests.
    token        []byte    // Last token returned by split.
    buf          []byte    // Buffer used as argument to split.
    start        int       // First non-processed byte in buf.
    end          int       // End of data in buf.
    err          error     // Sticky error.
    empties      int       // Count of successive empty tokens.
    scanCalled   bool      // Scan has been called; buffer is in use.
    done         bool      // Scan has finished.
}

Scanner是一个结构体，他包含了一些参数。当然他也有自己的方法。

func (s *Scanner) Scan() bool {
    if s.done {
        return false
    }
    s.scanCalled = true
    // Loop until we have a token.
    for {
        // See if we can get a token with what we already have.
        // If we've run out of data but have an error, give the split function
        // a chance to recover any remaining, possibly empty token.
        if s.end > s.start || s.err != nil {
            advance, token, err := s.split(s.buf[s.start:s.end], s.err != nil)
            if err != nil {
                if err == ErrFinalToken {
                    s.token = token
                    s.done = true
                    return true
                }
                s.setErr(err)
                return false
            }
            if !s.advance(advance) {
                return false
            }
            s.token = token
            if token != nil {
                if s.err == nil || advance > 0 {
                    s.empties = 0
                } else {
                    // Returning tokens not advancing input at EOF.
                    s.empties++
                    if s.empties > maxConsecutiveEmptyReads {
                        panic("bufio.Scan: too many empty tokens without progressing")
                    }
                }
                return true
            }
        }
        // We cannot generate a token with what we are holding.
        // If we've already hit EOF or an I/O error, we are done.
        if s.err != nil {
            // Shut it down.
            s.start = 0
            s.end = 0
            return false
        }
        // Must read more data.
        // First, shift data to beginning of buffer if there's lots of empty space
        // or space is needed.
        if s.start > 0 && (s.end == len(s.buf) || s.start > len(s.buf)/2) {
            copy(s.buf, s.buf[s.start:s.end])
            s.end -= s.start
            s.start = 0
        }
        // Is the buffer full? If so, resize.
        if s.end == len(s.buf) {
            // Guarantee no overflow in the multiplication below.
            const maxInt = int(^uint(0) >> 1)
            if len(s.buf) >= s.maxTokenSize || len(s.buf) > maxInt/2 {
                s.setErr(ErrTooLong)
                return false
            }
            newSize := len(s.buf) * 2
            if newSize == 0 {
                newSize = startBufSize
            }
            if newSize > s.maxTokenSize {
                newSize = s.maxTokenSize
            }
            newBuf := make([]byte, newSize)
            copy(newBuf, s.buf[s.start:s.end])
            s.buf = newBuf
            s.end -= s.start
            s.start = 0
        }
        // Finally we can read some input. Make sure we don't get stuck with
        // a misbehaving Reader. Officially we don't need to do this, but let's
        // be extra careful: Scanner is for safe, simple jobs.
        for loop := 0; ; {
            n, err := s.r.Read(s.buf[s.end:len(s.buf)])
            s.end += n
            if err != nil {
                s.setErr(err)
                break
            }
            if n > 0 {
                s.empties = 0
                break
            }
            loop++
            if loop > maxConsecutiveEmptyReads {
                s.setErr(io.ErrNoProgress)
                break
            }
        }
    }