// Tokenize breaks text into words while preserving their character position, whether it starts
// a new line, and new paragraph.
975483
func (p *DefaultWordTokenizer) Tokenize(text string, onlyPeriodContext bool) []*Token {
	textLength := len(text)

	if textLength == 0 {
		return nil
	}

	tokens := make([]*Token, 0, 50)
	lastSpace := 0
	lineStart := false
	paragraphStart := false
	getNextWord := false

	for i, char := range text {
		if !unicode.IsSpace(char) && i != textLength - 1 {
			continue
		}

		if char == '\n' {
			if lineStart {
				paragraphStart = true
			}
			lineStart = true
		}

		var cursor int
		if i == textLength - 1 {
			cursor = textLength
		} else {
			cursor = i
		}

		word := strings.TrimSpace(text[lastSpace:cursor])

		if word == "" {
			continue
		}

		hasSentencePunct := p.PunctStrings.HasSentencePunct(word)
		if onlyPeriodContext && !hasSentencePunct && !getNextWord {
			lastSpace = cursor
			continue
		}

		token := NewToken(word)
		token.Position = cursor
		token.ParaStart = paragraphStart
		token.LineStart = lineStart
		tokens = append(tokens, token)

		lastSpace = cursor
		lineStart = false
		paragraphStart = false

		if hasSentencePunct {
			getNextWord = true
		} else {
			getNextWord = false
		}
	}

	if len(tokens) == 0 {
		token := NewToken(text)
		token.Position = textLength
		tokens = append(tokens, token)
	}

	return tokens
}
703955
func (l *lexer) tokenize() {
	for l.state = lexData; l.state != nil; {
		l.state = l.state(l)
	}
}
497657
func Parse(tokenString string, keyFunc Keyfunc) (*Token, error) {
	return new(Parser).Parse(tokenString, keyFunc)
}
575728
func Colorize(c Color, b BackgroundColor, text string) string {
	return fmt.Sprintf("%s%s%s%s%s", ResetColors, c, b, text, ResetColors.String())
}
19631
func tokens(src string) <-chan *sep {
	tc := make(chan *sep)
	go tokenize(src, tc)
	return tc
}
140898
func (t *Tree) Parse(text string) (err error) {
	defer t.recover(&err)
	t.startParse(lex(text))
	t.Text = text
	t.parse()
	t.stopParse()
	return nil
}
868774
func tokenize(src string, tc chan *sep) {
	var s scanner.Scanner
	fset := token.NewFileSet() // boilerplate stuff for scanner...
	file := fset.AddFile("", fset.Base(), len(src))
	s.Init(file, []byte(src), nil, 1)
	for {
		filePos, tok, _ := s.Scan()
		tc <- &sep{int(filePos) - file.Base(), tok}
		if tok == token.EOF {
			close(tc)
			break
		}
	}
}
1187389
func (parser *Parser) tokenChar(value string) bool {
	token := parser.nextToken()
	return (token.Type == scanner.TokenChar) && (token.Value == value)
}
877650
func (o *Minifier) Minify(m *minify.M, w io.Writer, r io.Reader, _ map[string]string) error {
	omitSpace := true // on true the next text token must not start with a space

	attrByteBuffer := make([]byte, 0, 64)

	l := xml.NewLexer(r)
	defer l.Restore()

	tb := NewTokenBuffer(l)
	for {
		t := *tb.Shift()
		if t.TokenType == xml.CDATAToken {
			if len(t.Text) == 0 {
				continue
			}
			if text, useText := xml.EscapeCDATAVal(&attrByteBuffer, t.Text); useText {
				t.TokenType = xml.TextToken
				t.Data = text
			}
		}
		switch t.TokenType {
		case xml.ErrorToken:
			if l.Err() == io.EOF {
				return nil
			}
			return l.Err()
		case xml.DOCTYPEToken:
			if _, err := w.Write(t.Data); err != nil {
				return err
			}
		case xml.CDATAToken:
			if _, err := w.Write(t.Data); err != nil {
				return err
			}
			if len(t.Text) > 0 && parse.IsWhitespace(t.Text[len(t.Text)-1]) {
				omitSpace = true
			}
		case xml.TextToken:
			t.Data = parse.ReplaceMultipleWhitespace(t.Data)

			// whitespace removal; trim left
			if omitSpace && (t.Data[0] == ' ' || t.Data[0] == '\n') {
				t.Data = t.Data[1:]
			}

			// whitespace removal; trim right
			omitSpace = false
			if len(t.Data) == 0 {
				omitSpace = true
			} else if t.Data[len(t.Data)-1] == ' ' || t.Data[len(t.Data)-1] == '\n' {
				omitSpace = true
				i := 0
				for {
					next := tb.Peek(i)
					// trim if EOF, text token with whitespace begin or block token
					if next.TokenType == xml.ErrorToken {
						t.Data = t.Data[:len(t.Data)-1]
						omitSpace = false
						break
					} else if next.TokenType == xml.TextToken {
						// this only happens when a comment, doctype, cdata startpi tag was in between
						// remove if the text token starts with a whitespace
						if len(next.Data) > 0 && parse.IsWhitespace(next.Data[0]) {
							t.Data = t.Data[:len(t.Data)-1]
							omitSpace = false
						}
						break
					} else if next.TokenType == xml.CDATAToken {
						if len(next.Text) > 0 && parse.IsWhitespace(next.Text[0]) {
							t.Data = t.Data[:len(t.Data)-1]
							omitSpace = false
						}
						break
					} else if next.TokenType == xml.StartTagToken || next.TokenType == xml.EndTagToken {
						if !o.KeepWhitespace {
							t.Data = t.Data[:len(t.Data)-1]
							omitSpace = false
						}
						break
					}
					i++
				}
			}

			if _, err := w.Write(t.Data); err != nil {
				return err
			}
		case xml.StartTagToken:
			if o.KeepWhitespace {
				omitSpace = false
			}
			if _, err := w.Write(t.Data); err != nil {
				return err
			}
		case xml.StartTagPIToken:
			if _, err := w.Write(t.Data); err != nil {
				return err
			}
		case xml.AttributeToken:
			if _, err := w.Write(spaceBytes); err != nil {
				return err
			}
			if _, err := w.Write(t.Text); err != nil {
				return err
			}
			if _, err := w.Write(isBytes); err != nil {
				return err
			}

			if len(t.AttrVal) < 2 {
				if _, err := w.Write(t.AttrVal); err != nil {
					return err
				}
			} else {
				// prefer single or double quotes depending on what occurs more often in value
				val := xml.EscapeAttrVal(&attrByteBuffer, t.AttrVal[1:len(t.AttrVal)-1])
				if _, err := w.Write(val); err != nil {
					return err
				}
			}
		case xml.StartTagCloseToken:
			next := tb.Peek(0)
			skipExtra := false
			if next.TokenType == xml.TextToken && parse.IsAllWhitespace(next.Data) {
				next = tb.Peek(1)
				skipExtra = true
			}
			if next.TokenType == xml.EndTagToken {
				// collapse empty tags to single void tag
				tb.Shift()
				if skipExtra {
					tb.Shift()
				}
				if _, err := w.Write(voidBytes); err != nil {
					return err
				}
			} else {
				if _, err := w.Write(t.Text); err != nil {
					return err
				}
			}
		case xml.StartTagCloseVoidToken:
			if _, err := w.Write(t.Text); err != nil {
				return err
			}
		case xml.StartTagClosePIToken:
			if _, err := w.Write(t.Text); err != nil {
				return err
			}
		case xml.EndTagToken:
			if o.KeepWhitespace {
				omitSpace = false
			}
			if len(t.Data) > 3+len(t.Text) {
				t.Data[2+len(t.Text)] = '>'
				t.Data = t.Data[:3+len(t.Text)]
			}
			if _, err := w.Write(t.Data); err != nil {
				return err
			}
		}
	}
}
1223447
func (l *lexer) whitespace() ([]rune, Error) {
	// read until we hit the first non-whitespace rune
	_ws := make([]rune, 0, 1)

	// loop until there's nothing more to do
	for {
		_next, _err := l.read()
		if _err != nil {
			return _ws, _err
		}

		// what is this next rune?
		switch _next {
		// space or tab is consumed
		case _SPACE:
			fallthrough
		case _TAB:
			break

		// non-whitespace rune
		default:
			// return the rune to the buffer and we're done
			l.unread(_next)
			return _ws, nil
		}

		// add this rune to the whitespace
		_ws = append(_ws, _next)
	}
}
420866
func Tokenize(input []rune) ([]Token, error) {
	tokens := []Token{}
	for i, t, err := nextToken(0, input); err != errEndOfInput; i, t, err = nextToken(i, input) {
		if err != nil {
			return nil, err
		}

		tokens = append(tokens, t)
	}

	return tokens, nil
}
546357
func (z *Lexer) Lexeme() []byte {
	return z.buf[z.start:z.pos]
}
569447
func (t *SimpleTokenizer) Tokenize(sentence string) []string {
	// is the tokenizer really the best place to be making
	// everything lowercase? is this really a sanitizaion func?
	return strings.Split(strings.ToLower(sentence), t.SplitOn)
}
417899
func (p *parser) shift() *lexer.Token {
	var result *lexer.Token

	p.ensure(0)

	result, p.tokens = p.tokens[0], p.tokens[1:]

	// check error token
	if result.Kind == lexer.TokenError {
		errToken(result, "Lexer error")
	}

	return result
}
439651
func Tokenize(usage string) ([]*Token, error) {
	pos := 0
	var res []*Token
	var (
		tk = func(t TokenType, v string) {
			res = append(res, &Token{t, v, pos})
		}

		tkp = func(t TokenType, v string, p int) {
			res = append(res, &Token{t, v, p})
		}

		err = func(msg string) *ParseError {
			return &ParseError{usage, msg, pos}
		}
	)
	eof := len(usage)
	for pos < eof {
		switch c := usage[pos]; c {
		case ' ':
			pos++
		case '\t':
			pos++
		case '[':
			tk(TTOpenSq, "[")
			pos++
		case ']':
			tk(TTCloseSq, "]")
			pos++
		case '(':
			tk(TTOpenPar, "(")
			pos++
		case ')':
			tk(TTClosePar, ")")
			pos++
		case '|':
			tk(TTChoice, "|")
			pos++
		case '.':
			start := pos
			pos++
			if pos >= eof || usage[pos] != '.' {
				return nil, err("Unexpected end of usage, was expecting '..'")
			}
			pos++
			if pos >= eof || usage[pos] != '.' {
				return nil, err("Unexpected end of usage, was expecting '.'")
			}
			tkp(TTRep, "...", start)
			pos++
		case '-':
			start := pos
			pos++
			if pos >= eof {
				return nil, err("Unexpected end of usage, was expecting an option name")
			}

			switch o := usage[pos]; {
			case isLetter(o):
				pos++
				for ; pos < eof; pos++ {
					ok := isLetter(usage[pos])
					if !ok {
						break
					}
				}
				typ := TTShortOpt
				opt := usage[start:pos]
				if pos-start > 2 {
					typ = TTOptSeq
					opt = opt[1:]
				}
				tkp(typ, opt, start)
				if pos < eof && usage[pos] == '-' {
					return nil, err("Invalid syntax")
				}
			case o == '-':
				pos++
				if pos == eof || usage[pos] == ' ' {
					tkp(TTDoubleDash, "--", start)
					continue
				}
				for pos0 := pos; pos < eof; pos++ {
					ok := isOkLongOpt(usage[pos], pos == pos0)
					if !ok {
						break
					}
				}
				opt := usage[start:pos]
				if len(opt) == 2 {
					return nil, err("Was expecting a long option name")
				}
				tkp(TTLongOpt, opt, start)
			}

		case '=':
			start := pos
			pos++
			if pos >= eof || usage[pos] != '<' {
				return nil, err("Unexpected end of usage, was expecting '=<'")
			}
			closed := false
			for ; pos < eof; pos++ {
				closed = usage[pos] == '>'
				if closed {
					break
				}
			}
			if !closed {
				return nil, err("Unclosed option value")
			}
			if pos-start == 2 {
				return nil, err("Was expecting an option value")
			}
			pos++
			value := usage[start:pos]

			tkp(TTOptValue, value, start)

		default:
			switch {
			case isUppercase(c):
				start := pos
				for pos = pos + 1; pos < eof; pos++ {
					if !isOkInArg(usage[pos]) {
						break
					}
				}
				s := usage[start:pos]
				typ := TTArg
				if s == "OPTIONS" {
					typ = TTOptions
				}
				tkp(typ, s, start)
			default:
				return nil, err("Unexpected input")
			}

		}
	}

	return res, nil
}
948667
func Parse(name, text string) (*Parser, error) {
	p := NewParser(name)
	err := p.Parse(text)
	if err != nil {
		p = nil
	}
	return p, err
}
580271
func (tt TokenType) String() string {
	switch tt {
	case ErrorToken:
		return "Error"
	case UnknownToken:
		return "Unknown"
	case WhitespaceToken:
		return "Whitespace"
	case LineTerminatorToken:
		return "LineTerminator"
	case SingleLineCommentToken:
		return "SingleLineComment"
	case MultiLineCommentToken:
		return "MultiLineComment"
	case IdentifierToken:
		return "Identifier"
	case PunctuatorToken:
		return "Punctuator"
	case NumericToken:
		return "Numeric"
	case StringToken:
		return "String"
	case RegexpToken:
		return "Regexp"
	case TemplateToken:
		return "Template"
	}
	return "Invalid(" + strconv.Itoa(int(tt)) + ")"
}