// Tokenize reads from the given input and emits tokens to the output channel.
// Will end on any error from the reader, including io.EOF to signify the end
// of input.
335206
func (l Lexer) Tokenize(br *bufio.Reader, emit func(Token) error) error {
	states, err := l.States.Compile()
	if err != nil {
		return err
	}

	emit = l.Filters.Filter(emit)

	stack := &Stack{"root"}
	eol := false
	var subject = ""
	for {
		next, err := br.ReadString('\n')

		if err == bufio.ErrBufferFull {
			eol = false
		} else if err == io.EOF {
			eol = true
		} else if err != nil {
			return emit(EndToken)
		} else {
			eol = strings.HasSuffix(next, "\n")
		}

		subject = subject + next

		if subject == "" && err == io.EOF {
			emit(EndToken)
			return err
		}

		for subject != "" {
			// Match current state against current subject
			stateName := stack.Peek()
			state := states.Get(stateName)

			// Tokenize input
			n, rule, tokens, err := state.Match(subject)
			if err != nil {
				return emit(EndToken)
			}

			// No rules matched
			if rule == nil {
				if !eol {
					// Read more data for the current line
					break
				} else {
					// Emit entire subject as an error
					tokens = []Token{{Value: subject, Type: Error}}
					n = len(subject)
				}
			}

			// Emit each token to the output
			for _, t := range tokens {
				t.State = stateName
				if err := emit(t); err != nil {
					emit(EndToken)
					return err
				}
			}

			// Update state
			if rule == nil {
				if !eol {
					// Didn't match at all, reset to root state
					stack.Empty()
					stack.Push("root")
				}
			} else {
				// Push new states as appropriate
				for _, state := range rule.Stack() {
					if state == "#pop" {
						stack.Pop()
					} else if state != "" {
						stack.Push(state)
					}
				}
			}

			if stack.Len() == 0 {
				return emit(EndToken)
			}

			// Consume matched part
			subject = subject[n:]
		}
	}

	return nil
}
964324
func (l *lexer) emit(t tokenType) {
	l.tokens <- token{
		t,
		l.input[l.start:l.pos],
		l.lineNum(),
		l.columnNum(),
	}
	l.start = l.pos
}
975483
func (p *DefaultWordTokenizer) Tokenize(text string, onlyPeriodContext bool) []*Token {
	textLength := len(text)

	if textLength == 0 {
		return nil
	}

	tokens := make([]*Token, 0, 50)
	lastSpace := 0
	lineStart := false
	paragraphStart := false
	getNextWord := false

	for i, char := range text {
		if !unicode.IsSpace(char) && i != textLength - 1 {
			continue
		}

		if char == '\n' {
			if lineStart {
				paragraphStart = true
			}
			lineStart = true
		}

		var cursor int
		if i == textLength - 1 {
			cursor = textLength
		} else {
			cursor = i
		}

		word := strings.TrimSpace(text[lastSpace:cursor])

		if word == "" {
			continue
		}

		hasSentencePunct := p.PunctStrings.HasSentencePunct(word)
		if onlyPeriodContext && !hasSentencePunct && !getNextWord {
			lastSpace = cursor
			continue
		}

		token := NewToken(word)
		token.Position = cursor
		token.ParaStart = paragraphStart
		token.LineStart = lineStart
		tokens = append(tokens, token)

		lastSpace = cursor
		lineStart = false
		paragraphStart = false

		if hasSentencePunct {
			getNextWord = true
		} else {
			getNextWord = false
		}
	}

	if len(tokens) == 0 {
		token := NewToken(text)
		token.Position = textLength
		tokens = append(tokens, token)
	}

	return tokens
}
700232
func (w *eventWorker) emit(token string) {
	// set the token on the HTTPSink
	w.sink.AuthToken = token
	w.stats.EVBatchSizes.Add(float64(len(w.buffer)))
	// emit datapoints and handle any errors
	err := w.sink.AddEvents(context.Background(), w.buffer)
	w.handleError(err, token, w.buffer, w.sink.AddEvents)
	// account for the emitted datapoints
	atomic.AddInt64(&w.stats.TotalEventsBuffered, int64(len(w.buffer)*-1))
	w.buffer = w.buffer[:0]
}
1294046
func (r *tokenReader) Begin() {
	r.runes = r.runes[r.cursor:]
	r.cursor = 0
	r.oldPos = r.pos
}
71454
func (s *Scanner) Tokenize() (cs chan string) {
	cs = make(chan string)
	go func() {
		defer close(cs)
		for r := s.Scan(); r != EOF; r = s.Scan() {
			cs <- s.TokenText()
		}
		cs <- s.TokenText()
	}()
	return
}
1289537
func Token(r *http.Request) string {
	if val, err := contextGet(r, tokenKey); err == nil {
		if maskedToken, ok := val.(string); ok {
			return maskedToken
		}
	}

	return ""
}
930441
func (l *Lexer) errorToken(format string, args ...interface{}) StateFn {
	//fmt.Sprintf(format, args...)
	l.Emit(TokenError)
	return nil
}
473934
func (l *lexer) scan() *token {
	for {
		r, eof := l.next()
		if eof {
			return &token{tEOF, tEOF, ""}
		} else if isWhitespace(r) || l.eof() { // we just scanned some meaningful data
			token := l.scanIdent()
			l.scanWhitespaces()
			l.moveon()
			return token
		}
	}
}
874158
func Decode(token string) (Token, error) {
	var t Token
	err := (&t).UnmarshalText([]byte(token))
	return t, err
}
428937
func (t Tokenizer) Analyze(input string, mode TokenizeMode) (tokens []Token) {
	la := lattice.New(t.dic, t.udic)
	defer la.Free()
	la.Build(input)
	m := lattice.Normal
	switch mode {
	case Normal:
		m = lattice.Normal
	case Search:
		m = lattice.Search
	case Extended:
		m = lattice.Extended
	}
	la.Forward(m)
	la.Backward(m)
	size := len(la.Output)
	tokens = make([]Token, 0, size)
	for i := range la.Output {
		n := la.Output[size-1-i]
		tok := Token{
			ID:      n.ID,
			Class:   TokenClass(n.Class),
			Start:   n.Start,
			End:     n.Start + utf8.RuneCountInString(n.Surface),
			Surface: n.Surface,
			dic:     t.dic,
			udic:    t.udic,
		}
		if tok.ID == lattice.BosEosID {
			if i == 0 {
				tok.Surface = "BOS"
			} else {
				tok.Surface = "EOS"
			}
		}
		tokens = append(tokens, tok)
	}
	return
}
354715
func NewTokenLimiter(count uint) *TokenLimiter {
	tl := &TokenLimiter{count: count, ch: make(chan *Token, count)}
	for i := uint(0); i < count; i++ {
		tl.ch <- &Token{}
	}

	return tl
}
1106204
func (r *Lexer) Bytes() []byte {
	if r.token.kind == tokenUndef && r.Ok() {
		r.FetchToken()
	}
	if !r.Ok() || r.token.kind != tokenString {
		r.errInvalidToken("string")
		return nil
	}
	ret := make([]byte, base64.StdEncoding.DecodedLen(len(r.token.byteValue)))
	n, err := base64.StdEncoding.Decode(ret, r.token.byteValue)
	if err != nil {
		r.fatalError = &LexerError{
			Reason: err.Error(),
		}
		return nil
	}

	r.consume()
	return ret[:n]
}
1113987
func (p PunktSentenceTokenizer) Tokenize(text string) []string {
	sents := []string{}
	for _, s := range p.tokenizer.Tokenize(text) {
		sents = append(sents, s.Text)
	}
	return sents
}
698715
func (p *Peeker) Peek() *Token {
	if p.peeked == nil {
		p.peeked = <-p.ch
	}
	return p.peeked
}
866713
func (l *Lexer) emit(kind TokenKind) {
	l.produce(kind, l.input[l.start:l.pos])
}
222915
func Register(name string, t Tokenizer) {
	if tokenizers == nil {
		tokenizers = map[string]Tokenizer{}
	}
	tokenizers[name] = t
}