// Analyze tokenizes a sentence in the specified mode.
428937
func (t Tokenizer) Analyze(input string, mode TokenizeMode) (tokens []Token) {
	la := lattice.New(t.dic, t.udic)
	defer la.Free()
	la.Build(input)
	m := lattice.Normal
	switch mode {
	case Normal:
		m = lattice.Normal
	case Search:
		m = lattice.Search
	case Extended:
		m = lattice.Extended
	}
	la.Forward(m)
	la.Backward(m)
	size := len(la.Output)
	tokens = make([]Token, 0, size)
	for i := range la.Output {
		n := la.Output[size-1-i]
		tok := Token{
			ID:      n.ID,
			Class:   TokenClass(n.Class),
			Start:   n.Start,
			End:     n.Start + utf8.RuneCountInString(n.Surface),
			Surface: n.Surface,
			dic:     t.dic,
			udic:    t.udic,
		}
		if tok.ID == lattice.BosEosID {
			if i == 0 {
				tok.Surface = "BOS"
			} else {
				tok.Surface = "EOS"
			}
		}
		tokens = append(tokens, tok)
	}
	return
}
1122434
func tokenizePath(path string) []string {
	if "/" == path {
		return nil
	}
	return strings.Split(strings.Trim(path, "/"), "/")
}
496911
func Tokenise(expr string) []string {

	spaced := unary_minus_rx.ReplaceAllString(expr, "$1 @")
	spaced = fp_rx.ReplaceAllString(spaced, " ${1} ")
	spaced = functions_rx.ReplaceAllString(spaced, " ${1} ")

	if symbols_rx != nil {
		spaced = symbols_rx.ReplaceAllString(spaced, " ${1} ")
	}

	symbols := []string{"(", ")"}
	for _, symbol := range symbols {
		spaced = strings.Replace(spaced, symbol, fmt.Sprintf(" %s ", symbol), -1)
	}

	stripped := whitespace_rx.ReplaceAllString(strings.TrimSpace(spaced), "|")
	result := strings.Split(stripped, "|")
	return result
}
1318176
func (p *Parser) scan() {
	for {
		token, literal := p.l.Lex()
		p.scannedItems = append(p.scannedItems, ScannedItem{token, literal})
		if token == EndOfStringToken {
			break
		}
	}
}
672926
func (s *CountService) AnalyzeWildcard(analyzeWildcard bool) *CountService {
	s.analyzeWildcard = &analyzeWildcard
	return s
}
1211418
func matchLiteral(literal, subject string) bool {
	li := 0
	ll := len(literal)
	for i := 0; i < len(subject); i++ {
		if li >= ll {
			return false
		}
		b := subject[i]
		switch b {
		case pwc:
			// Skip token in literal
			ll := len(literal)
			for {
				if li >= ll || literal[li] == btsep {
					li--
					break
				}
				li++
			}
		case fwc:
			return true
		default:
			if b != literal[li] {
				return false
			}
		}
		li++
	}
	// Make sure we have processed all of the literal's chars..
	return li >= ll
}
71454
func (s *Scanner) Tokenize() (cs chan string) {
	cs = make(chan string)
	go func() {
		defer close(cs)
		for r := s.Scan(); r != EOF; r = s.Scan() {
			cs <- s.TokenText()
		}
		cs <- s.TokenText()
	}()
	return
}
202871
func (p *Parser) Match(typ TokenType, val string) *Token {
	if t := p.Peek(typ, val); t != nil {
		p.Consume()
		return t
	}
	return nil
}
692513
func (s *UpdateByQueryService) AnalyzeWildcard(analyzeWildcard bool) *UpdateByQueryService {
	s.analyzeWildcard = &analyzeWildcard
	return s
}
1356289
func (s *DocumentStatistics) analyzeDocument(source string, document *openapi.Document) {
	s.Name = source

	s.Title = document.Info.Title
	for _, pair := range document.Paths.Path {
		path := pair.Value
		if path.Get != nil {
			s.analyzeOperation("get", "paths"+pair.Name+"/get", path.Get)
		}
		if path.Post != nil {
			s.analyzeOperation("post", "paths"+pair.Name+"/post", path.Post)
		}
		if path.Put != nil {
			s.analyzeOperation("put", "paths"+pair.Name+"/put", path.Put)
		}
		if path.Delete != nil {
			s.analyzeOperation("delete", "paths"+pair.Name+"/delete", path.Delete)
		}
	}
	if document.Definitions != nil {
		for _, pair := range document.Definitions.AdditionalProperties {
			definition := pair.Value
			s.analyzeDefinition("definitions/"+pair.Name, definition)
		}
	}
}
356254
func (s *IndicesAnalyzeService) Tokenizer(tokenizer string) *IndicesAnalyzeService {
	s.request.Tokenizer = tokenizer
	return s
}
640343
func (s *DeleteByQueryService) AnalyzeWildcard(analyzeWildcard bool) *DeleteByQueryService {
	s.analyzeWildcard = &analyzeWildcard
	return s
}
412882
func (p *PragmaticSegmenter) Tokenize(text string) []string {
	return p.processor.process(text)
}
569447
func (t *SimpleTokenizer) Tokenize(sentence string) []string {
	// is the tokenizer really the best place to be making
	// everything lowercase? is this really a sanitizaion func?
	return strings.Split(strings.ToLower(sentence), t.SplitOn)
}
590073
func (q *MultiMatchQuery) Analyzer(analyzer string) *MultiMatchQuery {
	q.analyzer = analyzer
	return q
}
675287
func tokenize(msgText string) tokenMap {
	if msgText == "" {
		return nil
	}

	// split the message text up on basic punctuation/spaces
	tokens := splitExpr.Split(msgText, -1)
	tokenMap := tokenMap{}
	for _, token := range tokens {
		if len(token) < MinTokenLength {
			continue
		}

		token = strings.ToLower(token)
		if _, ok := tokenMap[token]; !ok {
			tokenMap[token] = map[string]chat1.EmptyStruct{}
		}

		// strip separators to raw tokens which we count as an alias to the
		// original token
		stripped := stripExpr.Split(token, -1)
		for _, s := range stripped {
			if s == "" {
				continue
			}
			tokenMap[token][s] = chat1.EmptyStruct{}

			// add the stem as an alias
			stemmed := porterstemmer.StemWithoutLowerCasing([]rune(s))
			tokenMap[token][string(stemmed)] = chat1.EmptyStruct{}

			// calculate prefixes to alias to the token
			for _, prefix := range prefixes(s) {
				tokenMap[token][prefix] = chat1.EmptyStruct{}
			}
		}
		// drop the original token from the set of aliases
		delete(tokenMap[token], token)
	}
	return tokenMap
}
783654
func (s *DeleteByQueryService) Analyzer(analyzer string) *DeleteByQueryService {
	s.analyzer = analyzer
	return s
}