// Tokenize converts a rune slice into a Token slice.
420866
func Tokenize(input []rune) ([]Token, error) {
	tokens := []Token{}
	for i, t, err := nextToken(0, input); err != errEndOfInput; i, t, err = nextToken(i, input) {
		if err != nil {
			return nil, err
		}

		tokens = append(tokens, t)
	}

	return tokens, nil
}
243915
func (p *Parser) Parse(tokenString string, keyFunc Keyfunc) (*Token, error) {
	return p.ParseWithClaims(tokenString, MapClaims{}, keyFunc)
}
1327931
func (s *scanner) consume(runes int) {
	if len(s.buffer) < runes {
		panic(fmt.Sprintf("Unable to consume %d runes from buffer.", runes))
	}

	s.lastTokenLine = s.line
	s.lastTokenCol = s.col
	s.lastTokenSize = runes

	s.buffer = s.buffer[runes:]
	s.col += runes
}
874158
func Decode(token string) (Token, error) {
	var t Token
	err := (&t).UnmarshalText([]byte(token))
	return t, err
}
1318176
func (p *Parser) scan() {
	for {
		token, literal := p.l.Lex()
		p.scannedItems = append(p.scannedItems, ScannedItem{token, literal})
		if token == EndOfStringToken {
			break
		}
	}
}
113142
func (c *c14N10CommentCanonicalizer) Canonicalize(el *etree.Element) ([]byte, error) {
	scope := make(map[string]struct{})
	return canonicalSerialize(canonicalPrep(el, scope))
}
1311100
func (r *Lexer) Skip() {
	if r.token.kind == tokenUndef && r.Ok() {
		r.FetchToken()
	}
	r.consume()
}
294646
func (p *Lexer) Rune() (rune, error) {

	p.fill()

	r, size := rune(p.buf[p.r]), 1
	if r >= utf8.RuneSelf {
		r, size = utf8.DecodeRune(p.buf[p.r:p.lastByte])
	}
	p.r += size
	p.lastRuneSize = append(p.lastRuneSize, size)
	return r, nil
}
537678
func (t *Tree) Parse() error {
	go t.lex.tokenize()
	for {
		n, err := t.parse()
		if err != nil {
			return t.enrichError(err)
		}
		if n == nil {
			break
		}
		t.root.Append(n)
	}
	t.traverse(t.root)
	return nil
}
428937
func (t Tokenizer) Analyze(input string, mode TokenizeMode) (tokens []Token) {
	la := lattice.New(t.dic, t.udic)
	defer la.Free()
	la.Build(input)
	m := lattice.Normal
	switch mode {
	case Normal:
		m = lattice.Normal
	case Search:
		m = lattice.Search
	case Extended:
		m = lattice.Extended
	}
	la.Forward(m)
	la.Backward(m)
	size := len(la.Output)
	tokens = make([]Token, 0, size)
	for i := range la.Output {
		n := la.Output[size-1-i]
		tok := Token{
			ID:      n.ID,
			Class:   TokenClass(n.Class),
			Start:   n.Start,
			End:     n.Start + utf8.RuneCountInString(n.Surface),
			Surface: n.Surface,
			dic:     t.dic,
			udic:    t.udic,
		}
		if tok.ID == lattice.BosEosID {
			if i == 0 {
				tok.Surface = "BOS"
			} else {
				tok.Surface = "EOS"
			}
		}
		tokens = append(tokens, tok)
	}
	return
}
964324
func (l *lexer) emit(t tokenType) {
	l.tokens <- token{
		t,
		l.input[l.start:l.pos],
		l.lineNum(),
		l.columnNum(),
	}
	l.start = l.pos
}
740449
func (d *Document) Initialize() {
	d.WordFrequency = make(map[string]int)
	for i, paragraph := range strings.Split(d.Content, "\n\n") {
		for _, s := range d.SentenceTokenizer.Tokenize(paragraph) {
			wordCount := d.NumWords
			d.NumSentences++
			words := []Word{}
			for _, word := range d.WordTokenizer.Tokenize(s) {
				word = strings.TrimSpace(word)
				if len(word) == 0 {
					continue
				}
				d.NumCharacters += countChars(word)
				if _, found := d.WordFrequency[word]; found {
					d.WordFrequency[word]++
				} else {
					d.WordFrequency[word] = 1
				}
				syllables := Syllables(word)
				words = append(words, Word{Text: word, Syllables: syllables})
				d.NumSyllables += float64(syllables)
				if syllables > 2 {
					d.NumPolysylWords++
				}
				if isComplex(word, syllables) {
					d.NumComplexWords++
				}
				d.NumWords++
			}
			d.Sentences = append(d.Sentences, Sentence{
				Text:      strings.TrimSpace(s),
				Length:    int(d.NumWords - wordCount),
				Words:     words,
				Paragraph: i})
		}
		d.NumParagraphs++
	}
}
190159
func (l Lexer) TokenizeString(s string) ([]Token, error) {
	r := bufio.NewReader(strings.NewReader(s))
	tokens := []Token{}
	err := l.Tokenize(r, func(t Token) error {
		tokens = append(tokens, t)
		return nil
	})
	return tokens, err
}
191156
func (l *lexer) emit(t TokenType) {
	l.tokens <- Token{
		Type: t,
		Text: l.input[l.start:l.pos],
	}
	l.start = l.pos
}
1307048
func (p *DefaultWordTokenizer) FirstUpper(t *Token) bool {
	if t.Tok == "" {
		return false
	}

	runes := []rune(t.Tok)
	return unicode.IsUpper(runes[0])
}
836916
func (c *c14N10ExclusiveCanonicalizer) Canonicalize(el *etree.Element) ([]byte, error) {
	err := etreeutils.TransformExcC14n(el, c.prefixList)
	if err != nil {
		return nil, err
	}

	return canonicalSerialize(el)
}
1223447
func (l *lexer) whitespace() ([]rune, Error) {
	// read until we hit the first non-whitespace rune
	_ws := make([]rune, 0, 1)

	// loop until there's nothing more to do
	for {
		_next, _err := l.read()
		if _err != nil {
			return _ws, _err
		}

		// what is this next rune?
		switch _next {
		// space or tab is consumed
		case _SPACE:
			fallthrough
		case _TAB:
			break

		// non-whitespace rune
		default:
			// return the rune to the buffer and we're done
			l.unread(_next)
			return _ws, nil
		}

		// add this rune to the whitespace
		_ws = append(_ws, _next)
	}
}