// scanUnixLine is a bufio.Scanner SplitFunc. It tokenizes on lines, but
// only returns unix-style lines. So even if the line is "one\r\n", the
// token returned will be "one\n".
556949
func scanUnixLine(data []byte, atEOF bool) (advance int, token []byte, err error) {
	advance, token, err = bufio.ScanLines(data, atEOF)
	if advance == 0 {
		// If we reached the end of a line without a newline, then
		// just return as it is. Otherwise the Scanner will keep trying
		// to scan, blocking forever.
		return
	}

	return advance, append(token, '\n'), err
}
209211
func (p *unvisParser) Peek() (rune, error) {
	if p.idx >= len(p.tokens) {
		return unicode.ReplacementChar, fmt.Errorf("tried to read past end of token list")
	}
	return p.tokens[p.idx], nil
}
181201
func (uuid *UUID) Scan(src interface{}) error {
	switch src.(type) {
	case string:
		// if an empty UUID comes from a table, we return a null UUID
		if src.(string) == "" {
			return nil
		}

		// see uuid.Parse for required string format
		parsed := Parse(src.(string))

		if parsed == nil {
			return errors.New("Scan: invalid UUID format")
		}

		*uuid = parsed
	case []byte:
		b := src.([]byte)

		// if an empty UUID comes from a table, we return a null UUID
		if len(b) == 0 {
			return nil
		}

		// assumes a simple slice of bytes if 16 bytes
		// otherwise attempts to parse
		if len(b) == 16 {
			parsed := make([]byte, 16)
			copy(parsed, b)
			*uuid = UUID(parsed)
		} else {
			u := Parse(string(b))

			if u == nil {
				return errors.New("Scan: invalid UUID format")
			}

			*uuid = u
		}

	default:
		return fmt.Errorf("Scan: unable to scan type %T into UUID", src)
	}

	return nil
}
1106020
func (t *Tokenizer) scanStream() (*Token, error) {
	state := STATE_START
	var tokenType TokenType
	value := make([]int32, 0, INITIAL_TOKEN_CAPACITY)
	var (
		nextRune     int32
		nextRuneType RuneTokenType
		err          error
	)
SCAN:
	for {
		nextRune, _, err = t.input.ReadRune()
		nextRuneType = t.classifier.ClassifyRune(nextRune)
		if err != nil {
			if err == io.EOF {
				nextRuneType = RUNETOKEN_EOF
				err = nil
			} else {
				return nil, err
			}
		}
		switch state {
		case STATE_START: // no runes read yet
			{
				switch nextRuneType {
				case RUNETOKEN_EOF:
					{
						return nil, io.EOF
					}
				case RUNETOKEN_CHAR:
					{
						tokenType = TOKEN_WORD
						value = append(value, nextRune)
						state = STATE_INWORD
					}
				case RUNETOKEN_SPACE:
					{
					}
				case RUNETOKEN_ESCAPING_QUOTE:
					{
						tokenType = TOKEN_WORD
						state = STATE_QUOTED_ESCAPING
					}
				case RUNETOKEN_NONESCAPING_QUOTE:
					{
						tokenType = TOKEN_WORD
						state = STATE_QUOTED
					}
				case RUNETOKEN_ESCAPE:
					{
						tokenType = TOKEN_WORD
						state = STATE_ESCAPING
					}
				case RUNETOKEN_COMMENT:
					{
						tokenType = TOKEN_COMMENT
						state = STATE_COMMENT
					}
				default:
					{
						return nil, errors.New(fmt.Sprintf("Unknown rune: %v", nextRune))
					}
				}
			}
		case STATE_INWORD: // in a regular word
			{
				switch nextRuneType {
				case RUNETOKEN_EOF:
					{
						break SCAN
					}
				case RUNETOKEN_CHAR, RUNETOKEN_COMMENT:
					{
						value = append(value, nextRune)
					}
				case RUNETOKEN_SPACE:
					{
						t.input.UnreadRune()
						break SCAN
					}
				case RUNETOKEN_ESCAPING_QUOTE:
					{
						state = STATE_QUOTED_ESCAPING
					}
				case RUNETOKEN_NONESCAPING_QUOTE:
					{
						state = STATE_QUOTED
					}
				case RUNETOKEN_ESCAPE:
					{
						state = STATE_ESCAPING
					}
				default:
					{
						return nil, errors.New(fmt.Sprintf("Uknown rune: %v", nextRune))
					}
				}
			}
		case STATE_ESCAPING: // the next rune after an escape character
			{
				switch nextRuneType {
				case RUNETOKEN_EOF:
					{
						err = errors.New("EOF found after escape character")
						break SCAN
					}
				case RUNETOKEN_CHAR, RUNETOKEN_SPACE, RUNETOKEN_ESCAPING_QUOTE, RUNETOKEN_NONESCAPING_QUOTE, RUNETOKEN_ESCAPE, RUNETOKEN_COMMENT:
					{
						state = STATE_INWORD
						value = append(value, nextRune)
					}
				default:
					{
						return nil, errors.New(fmt.Sprintf("Uknown rune: %v", nextRune))
					}
				}
			}
		case STATE_ESCAPING_QUOTED: // the next rune after an escape character, in double quotes
			{
				switch nextRuneType {
				case RUNETOKEN_EOF:
					{
						err = errors.New("EOF found after escape character")
						break SCAN
					}
				case RUNETOKEN_CHAR, RUNETOKEN_SPACE, RUNETOKEN_ESCAPING_QUOTE, RUNETOKEN_NONESCAPING_QUOTE, RUNETOKEN_ESCAPE, RUNETOKEN_COMMENT:
					{
						state = STATE_QUOTED_ESCAPING
						value = append(value, nextRune)
					}
				default:
					{
						return nil, errors.New(fmt.Sprintf("Uknown rune: %v", nextRune))
					}
				}
			}
		case STATE_QUOTED_ESCAPING: // in escaping double quotes
			{
				switch nextRuneType {
				case RUNETOKEN_EOF:
					{
						err = errors.New("EOF found when expecting closing quote.")
						break SCAN
					}
				case RUNETOKEN_CHAR, RUNETOKEN_UNKNOWN, RUNETOKEN_SPACE, RUNETOKEN_NONESCAPING_QUOTE, RUNETOKEN_COMMENT:
					{
						value = append(value, nextRune)
					}
				case RUNETOKEN_ESCAPING_QUOTE:
					{
						state = STATE_INWORD
					}
				case RUNETOKEN_ESCAPE:
					{
						state = STATE_ESCAPING_QUOTED
					}
				default:
					{
						return nil, errors.New(fmt.Sprintf("Uknown rune: %v", nextRune))
					}
				}
			}
		case STATE_QUOTED: // in non-escaping single quotes
			{
				switch nextRuneType {
				case RUNETOKEN_EOF:
					{
						err = errors.New("EOF found when expecting closing quote.")
						break SCAN
					}
				case RUNETOKEN_CHAR, RUNETOKEN_UNKNOWN, RUNETOKEN_SPACE, RUNETOKEN_ESCAPING_QUOTE, RUNETOKEN_ESCAPE, RUNETOKEN_COMMENT:
					{
						value = append(value, nextRune)
					}
				case RUNETOKEN_NONESCAPING_QUOTE:
					{
						state = STATE_INWORD
					}
				default:
					{
						return nil, errors.New(fmt.Sprintf("Uknown rune: %v", nextRune))
					}
				}
			}
		case STATE_COMMENT:
			{
				switch nextRuneType {
				case RUNETOKEN_EOF:
					{
						break SCAN
					}
				case RUNETOKEN_CHAR, RUNETOKEN_UNKNOWN, RUNETOKEN_ESCAPING_QUOTE, RUNETOKEN_ESCAPE, RUNETOKEN_COMMENT, RUNETOKEN_NONESCAPING_QUOTE:
					{
						value = append(value, nextRune)
					}
				case RUNETOKEN_SPACE:
					{
						if nextRune == '\n' {
							state = STATE_START
							break SCAN
						} else {
							value = append(value, nextRune)
						}
					}
				default:
					{
						return nil, errors.New(fmt.Sprintf("Uknown rune: %v", nextRune))
					}
				}
			}
		default:
			{
				panic(fmt.Sprintf("Unexpected state: %v", state))
			}
		}
	}
	token := &Token{
		tokenType: tokenType,
		value:     string(value)}
	return token, err
}
1342871
func parseFieldLine(l string) (field, rest string, ok bool) {
	if m := fieldRe.FindStringSubmatch(l); m != nil {
		return m[1], m[2], true
	}
	return
}
736916
func splitLines(data []byte, eof bool) (advance int, token []byte, err error) {
	if eof && len(data) == 0 {
		return 0, nil, nil
	}

	if i := bytes.IndexByte(data, '\n'); i >= 0 {
		orgStdout.Write(data[nextOutPos : i+1])
		nextOutPos = 0
		return i + 1, data[0:i], nil
	}

	orgStdout.Write(data[nextOutPos:])
	nextOutPos = len(data)

	if eof {
		// This is a final, non-terminated line. Return it.
		return len(data), data, nil
	}

	return 0, nil, nil
}
377177
func Splitlines(s string) []string {
	s = strings.Replace(s, "\r", "\n", -1)
	s = strings.Replace(s, "\r\n", "\n", -1)
	return MapS(trimnewlines, FilterS(nonempty, strings.Split(s, "\n")))
}
358578
func (self *_parser) Scan() (tkn token.Token, literal string, idx file.Idx) {
	return self.scan()
}
522457
func (p *parser) scanBlank() error {
	if p.useOptionX() {
		for {
			for p.charsRight() > 0 && isSpace(p.rightChar(0)) {
				p.moveRight(1)
			}

			if p.charsRight() == 0 {
				break
			}

			if p.rightChar(0) == '#' {
				for p.charsRight() > 0 && p.rightChar(0) != '\n' {
					p.moveRight(1)
				}
			} else if p.charsRight() >= 3 && p.rightChar(2) == '#' &&
				p.rightChar(1) == '?' && p.rightChar(0) == '(' {
				for p.charsRight() > 0 && p.rightChar(0) != ')' {
					p.moveRight(1)
				}
				if p.charsRight() == 0 {
					return p.getErr(ErrUnterminatedComment)
				}
				p.moveRight(1)
			} else {
				break
			}
		}
	} else {
		for {
			if p.charsRight() < 3 || p.rightChar(2) != '#' ||
				p.rightChar(1) != '?' || p.rightChar(0) != '(' {
				return nil
			}

			for p.charsRight() > 0 && p.rightChar(0) != ')' {
				p.moveRight(1)
			}
			if p.charsRight() == 0 {
				return p.getErr(ErrUnterminatedComment)
			}
			p.moveRight(1)
		}
	}
	return nil
}
1117207
func (s *Scanner) ScanRegex() (tok Token, pos Pos, lit string) {
	_, pos = s.r.curr()

	// Start & end sentinels.
	start, end := '/', '/'
	// Valid escape chars.
	escapes := map[rune]rune{'/': '/'}

	b, err := ScanDelimited(s.r, start, end, escapes, true)

	if err == errBadEscape {
		_, pos = s.r.curr()
		return BADESCAPE, pos, lit
	} else if err != nil {
		return BADREGEX, pos, lit
	}
	return REGEX, pos, string(b)
}
1156719
func (s *Scanner) Next() *Token {
	if s.err != nil {
		return s.err
	}
	if s.pos >= len(s.input) {
		s.err = &Token{TokenEOF, "", s.row, s.col}
		return s.err
	}
	if s.pos == 0 {
		// Test BOM only once, at the beginning of the file.
		if strings.HasPrefix(s.input, "\uFEFF") {
			return s.emitSimple(TokenBOM, "\uFEFF")
		}
	}
	// There's a lot we can guess based on the first byte so we'll take a
	// shortcut before testing multiple regexps.
	input := s.input[s.pos:]
	switch input[0] {
	case '\t', '\n', ' ':
		// Whitespace.
		return s.emitToken(TokenS, matchers[TokenS].FindString(input))
	case '.':
		// Dot is too common to not have a quick check.
		// We'll test if this is a Char; if it is followed by a number it is a
		// dimension/percentage/number, and this will be matched later.
		if len(input) > 1 && !unicode.IsDigit(rune(input[1])) {
			return s.emitSimple(TokenChar, ".")
		}
	case '#':
		// Another common one: Hash or Char.
		if match := matchers[TokenHash].FindString(input); match != "" {
			return s.emitToken(TokenHash, match)
		}
		return s.emitSimple(TokenChar, "#")
	case '@':
		// Another common one: AtKeyword or Char.
		if match := matchers[TokenAtKeyword].FindString(input); match != "" {
			return s.emitSimple(TokenAtKeyword, match)
		}
		return s.emitSimple(TokenChar, "@")
	case ':', ',', ';', '%', '&', '+', '=', '>', '(', ')', '[', ']', '{', '}':
		// More common chars.
		return s.emitSimple(TokenChar, string(input[0]))
	case '"', '\'':
		// String or error.
		match := matchers[TokenString].FindString(input)
		if match != "" {
			return s.emitToken(TokenString, match)
		}

		s.err = &Token{TokenError, "unclosed quotation mark", s.row, s.col}
		return s.err
	case '/':
		// Comment, error or Char.
		if len(input) > 1 && input[1] == '*' {
			match := matchers[TokenComment].FindString(input)
			if match != "" {
				return s.emitToken(TokenComment, match)
			} else {
				s.err = &Token{TokenError, "unclosed comment", s.row, s.col}
				return s.err
			}
		}
		return s.emitSimple(TokenChar, "/")
	case '~':
		// Includes or Char.
		return s.emitPrefixOrChar(TokenIncludes, "~=")
	case '|':
		// DashMatch or Char.
		return s.emitPrefixOrChar(TokenDashMatch, "|=")
	case '^':
		// PrefixMatch or Char.
		return s.emitPrefixOrChar(TokenPrefixMatch, "^=")
	case '$':
		// SuffixMatch or Char.
		return s.emitPrefixOrChar(TokenSuffixMatch, "$=")
	case '*':
		// SubstringMatch or Char.
		return s.emitPrefixOrChar(TokenSubstringMatch, "*=")
	case '<':
		// CDO or Char.
		return s.emitPrefixOrChar(TokenCDO, "<!--")
	}
	// Test all regexps, in order.
	for _, token := range matchOrder {
		if match := matchers[token].FindString(input); match != "" {
			return s.emitToken(token, match)
		}
	}
	// We already handled unclosed quotation marks and comments,
	// so this can only be a Char.
	r, width := utf8.DecodeRuneInString(input)
	token := &Token{TokenChar, string(r), s.row, s.col}
	s.col += width
	s.pos += width
	return token
}
1276584
func (s *scanner) scan() (token, string, error) {
	ch, _, err := s.r.ReadRune()

	if err != nil && err != io.EOF {
		return tokenError, "", err
	}

	switch ch {
	case zeroRune:
		return eof, "", nil
	case ':':
		return colon, string(ch), nil
	case '~':
		return tilde, string(ch), nil
	case '^':
		return caret, string(ch), nil
	case '.':
		return dot, string(ch), nil
	case '/':
		return slash, string(ch), nil
	case '{':
		return obrace, string(ch), nil
	case '}':
		return cbrace, string(ch), nil
	case '-':
		return minus, string(ch), nil
	case '@':
		return at, string(ch), nil
	case '\\':
		return aslash, string(ch), nil
	case '?':
		return qmark, string(ch), nil
	case '*':
		return asterisk, string(ch), nil
	case '[':
		return obracket, string(ch), nil
	case '!':
		return emark, string(ch), nil
	}

	if unicode.IsSpace(ch) {
		return space, string(ch), nil
	}

	if unicode.IsControl(ch) {
		return control, string(ch), nil
	}

	if unicode.IsLetter(ch) {
		return tokenizeExpression(ch, word, unicode.IsLetter, s.r)
	}

	if unicode.IsNumber(ch) {
		return tokenizeExpression(ch, number, unicode.IsNumber, s.r)
	}

	return tokenError, string(ch), nil
}
208059
func (p *Parser) scan() (token, string, error) {
	if p.unreadLastChar {
		p.unreadLastChar = false
		return p.currentParsedChar.tok, p.currentParsedChar.lit, nil
	}

	tok, lit, err := p.s.scan()

	p.currentParsedChar.tok, p.currentParsedChar.lit = tok, lit

	return tok, lit, err
}
974698
func (s *Scanner) Scan() bool {
	for s.source.Scan() {
		line := s.source.Text()
		status, fields := s.parseLine(line)
		switch status {
		case qffail:
			continue
		case qfendmultiline:
			s.mlpoped = true
			s.entry = s.qi.qflist[len(s.qi.qflist)-1]
			return true
		case qfignoreline:
			continue
		}
		var lastml *Entry // last multiline entry which isn't poped out
		if !s.mlpoped {
			lastml = s.qi.qflist[len(s.qi.qflist)-1]
		}
		qfl := &Entry{
			Filename: fields.namebuf,
			Lnum:     fields.lnum,
			Col:      fields.col,
			Nr:       fields.enr,
			Pattern:  fields.pattern,
			Text:     fields.errmsg,
			Vcol:     fields.useviscol,
			Valid:    fields.valid,
			Type:     rune(fields.etype),
			Lines:    fields.lines,
		}
		if qfl.Filename == "" && s.qi.currfile != "" {
			qfl.Filename = s.qi.currfile
		}
		s.qi.qflist = append(s.qi.qflist, qfl)
		if s.qi.multiline {
			s.mlpoped = false // mark multiline entry is not poped
			// if there is last multiline entry which isn't poped out yet, pop it out now.
			if lastml != nil {
				s.entry = lastml
				return true
			}
			continue
		}
		// multiline flag doesn't be reset with new entry.
		// %Z or nomach are the only way to reset multiline flag.
		s.entry = qfl
		return true
	}
	// pop last not-ended multiline entry
	if !s.mlpoped {
		s.mlpoped = true
		s.entry = s.qi.qflist[len(s.qi.qflist)-1]
		return true
	}
	return false
}
221666
func scanString(buf []byte) ([]byte, error) {
	n := len(buf)
	if n == 0 || n == 1 {
		return buf, nil
	}
	// Is the string quoted?
	quote := buf[0]
	if quote != '"' && quote != '\'' {
		// Not quoted.
		return buf, nil
	}

	// Quoted.
	var escapers []int
	idx := 1
	for buf[idx] != quote {
		if c := buf[idx]; c == '\\' {
			escapers = append(escapers, idx)
			idx++
		}
		idx++
		if idx == n {
			return nil, errInvalidKeyValue
		}
	}
	buf = buf[1:idx]

	if len(escapers) == 0 {
		return buf, nil
	}

	// Remove escapers.
	for i, pos := range escapers {
		copy(buf[pos-i-1:], buf[pos-i:])
	}

	return buf[:len(buf)-len(escapers)], nil
}
886216
func (o *UUID) Scan(src interface{}) error {
	if src == nil {
		return nil
	}
	if src == "" {
		return nil
	}
	switch src := src.(type) {

	case string:
		return o.UnmarshalText([]byte(src))

	case []byte:
		if len(src) == length {
			return o.UnmarshalBinary(src)
		} else {
			return o.UnmarshalText(src)
		}

	default:
		return fmt.Errorf("uuid: cannot scan type [%T] into UUID", src)
	}
}
7736
func separatorsScanner(separators []rune) func(data []byte, atEOF bool) (int, []byte, error) {
	return func(data []byte, atEOF bool) (advance int, token []byte, err error) {
		if atEOF && len(data) == 0 {
			return 0, nil, nil
		}

		// Transform the separators into a map (for efficient lookup)
		seps := make(map[rune]bool)
		for _, r := range separators {
			seps[r] = true
		}

		// Scan until separator, marking the end of a token
		for width, i := 0, 0; i < len(data); i += width {
			var r rune
			r, width = utf8.DecodeRune(data[i:])
			if seps[r] {
				if i == 0 {
					// Separator token
					return i + width, data[0 : i+width], nil
				}

				// Normal token
				return i, data[0:i], nil
			}
		}

		// If we're at EOF, we have a final, non-empty, non-terminated token: return it
		if atEOF && len(data) > 0 {
			return len(data), data[0:], nil
		}

		// Request more data
		return 0, nil, nil
	}
}