// English customized sentence tokenizer.
559990
func newSentenceTokenizer(s *sentences.Storage) (*sentences.DefaultSentenceTokenizer, error) {
	training := s

	if training == nil {
		b, err := data.Asset("data/english.json")
		if err != nil {
			return nil, err
		}

		training, err = sentences.LoadTraining(b)
		if err != nil {
			return nil, err
		}
	}

	// supervisor abbreviations
	abbrevs := []string{"sgt", "gov", "no", "mt"}
	for _, abbr := range abbrevs {
		training.AbbrevTypes.Add(abbr)
	}

	lang := sentences.NewPunctStrings()
	word := newWordTokenizer(lang)
	annotations := sentences.NewAnnotations(training, lang, word)

	ortho := &sentences.OrthoContext{
		Storage:      training,
		PunctStrings: lang,
		TokenType:    word,
		TokenFirst:   word,
	}

	multiPunct := &multiPunctWordAnnotation{
		Storage:      training,
		TokenParser:  word,
		TokenGrouper: &sentences.DefaultTokenGrouper{},
		Ortho:        ortho,
	}

	annotations = append(annotations, multiPunct)

	tokenizer := &sentences.DefaultSentenceTokenizer{
		Storage:       training,
		PunctStrings:  lang,
		WordTokenizer: word,
		Annotations:   annotations,
	}

	return tokenizer, nil
}
35136
func Sentences(text string) []string {
	sentences := sentenceRegex.Split(text, -1)

	if sentences[len(sentences)-1] == "" {
		return sentences[0 : len(sentences)-1]
	}

	return sentences
}
404533
func (p *Parser) Next() (GrammarType, TokenType, []byte) {
	p.err = nil

	if p.prevEnd {
		p.tt, p.data = RightBraceToken, endBytes
		p.prevEnd = false
	} else {
		p.tt, p.data = p.popToken(true)
	}
	gt := p.state[len(p.state)-1](p)
	return gt, p.tt, p.data
}
1266089
func (l Lorem) Sentence(v reflect.Value) (interface{}, error) {
	sentence := l.sentence()
	return sentence, nil
}
675287
func tokenize(msgText string) tokenMap {
	if msgText == "" {
		return nil
	}

	// split the message text up on basic punctuation/spaces
	tokens := splitExpr.Split(msgText, -1)
	tokenMap := tokenMap{}
	for _, token := range tokens {
		if len(token) < MinTokenLength {
			continue
		}

		token = strings.ToLower(token)
		if _, ok := tokenMap[token]; !ok {
			tokenMap[token] = map[string]chat1.EmptyStruct{}
		}

		// strip separators to raw tokens which we count as an alias to the
		// original token
		stripped := stripExpr.Split(token, -1)
		for _, s := range stripped {
			if s == "" {
				continue
			}
			tokenMap[token][s] = chat1.EmptyStruct{}

			// add the stem as an alias
			stemmed := porterstemmer.StemWithoutLowerCasing([]rune(s))
			tokenMap[token][string(stemmed)] = chat1.EmptyStruct{}

			// calculate prefixes to alias to the token
			for _, prefix := range prefixes(s) {
				tokenMap[token][prefix] = chat1.EmptyStruct{}
			}
		}
		// drop the original token from the set of aliases
		delete(tokenMap[token], token)
	}
	return tokenMap
}
325813
func (ti *tokenIterator) skipToLeftCurly() bool {
	return ti.skipToLeft(token.LBRACE, token.RBRACE)
}
388316
func Ellipsis(s string, maxlen int) string {
	r := []rune(s)
	if len(r) <= maxlen {
		return s
	}
	if maxlen <= 3 {
		return string(r[:maxlen])
	}
	return string(r[:maxlen-3]) + "..."
}
439651
func Tokenize(usage string) ([]*Token, error) {
	pos := 0
	var res []*Token
	var (
		tk = func(t TokenType, v string) {
			res = append(res, &Token{t, v, pos})
		}

		tkp = func(t TokenType, v string, p int) {
			res = append(res, &Token{t, v, p})
		}

		err = func(msg string) *ParseError {
			return &ParseError{usage, msg, pos}
		}
	)
	eof := len(usage)
	for pos < eof {
		switch c := usage[pos]; c {
		case ' ':
			pos++
		case '\t':
			pos++
		case '[':
			tk(TTOpenSq, "[")
			pos++
		case ']':
			tk(TTCloseSq, "]")
			pos++
		case '(':
			tk(TTOpenPar, "(")
			pos++
		case ')':
			tk(TTClosePar, ")")
			pos++
		case '|':
			tk(TTChoice, "|")
			pos++
		case '.':
			start := pos
			pos++
			if pos >= eof || usage[pos] != '.' {
				return nil, err("Unexpected end of usage, was expecting '..'")
			}
			pos++
			if pos >= eof || usage[pos] != '.' {
				return nil, err("Unexpected end of usage, was expecting '.'")
			}
			tkp(TTRep, "...", start)
			pos++
		case '-':
			start := pos
			pos++
			if pos >= eof {
				return nil, err("Unexpected end of usage, was expecting an option name")
			}

			switch o := usage[pos]; {
			case isLetter(o):
				pos++
				for ; pos < eof; pos++ {
					ok := isLetter(usage[pos])
					if !ok {
						break
					}
				}
				typ := TTShortOpt
				opt := usage[start:pos]
				if pos-start > 2 {
					typ = TTOptSeq
					opt = opt[1:]
				}
				tkp(typ, opt, start)
				if pos < eof && usage[pos] == '-' {
					return nil, err("Invalid syntax")
				}
			case o == '-':
				pos++
				if pos == eof || usage[pos] == ' ' {
					tkp(TTDoubleDash, "--", start)
					continue
				}
				for pos0 := pos; pos < eof; pos++ {
					ok := isOkLongOpt(usage[pos], pos == pos0)
					if !ok {
						break
					}
				}
				opt := usage[start:pos]
				if len(opt) == 2 {
					return nil, err("Was expecting a long option name")
				}
				tkp(TTLongOpt, opt, start)
			}

		case '=':
			start := pos
			pos++
			if pos >= eof || usage[pos] != '<' {
				return nil, err("Unexpected end of usage, was expecting '=<'")
			}
			closed := false
			for ; pos < eof; pos++ {
				closed = usage[pos] == '>'
				if closed {
					break
				}
			}
			if !closed {
				return nil, err("Unclosed option value")
			}
			if pos-start == 2 {
				return nil, err("Was expecting an option value")
			}
			pos++
			value := usage[start:pos]

			tkp(TTOptValue, value, start)

		default:
			switch {
			case isUppercase(c):
				start := pos
				for pos = pos + 1; pos < eof; pos++ {
					if !isOkInArg(usage[pos]) {
						break
					}
				}
				s := usage[start:pos]
				typ := TTArg
				if s == "OPTIONS" {
					typ = TTOptions
				}
				tkp(typ, s, start)
			default:
				return nil, err("Unexpected input")
			}

		}
	}

	return res, nil
}
945176
func (pt *PerceptronTagger) Train(sentences TupleSlice, iterations int) {
	var guess string
	var found bool

	pt.makeTagMap(sentences)
	for i := 0; i < iterations; i++ {
		for _, tuple := range sentences {
			words, tags := tuple[0], tuple[1]
			p1, p2 := "-START-", "-START2-"
			context := []string{p1, p2}
			for _, w := range words {
				if w == "" {
					continue
				}
				context = append(context, normalize(w))
			}
			context = append(context, []string{"-END-", "-END2-"}...)
			for i, word := range words {
				if guess, found = pt.tagMap[word]; !found {
					feats := featurize(i, context, word, p1, p2)
					guess = pt.model.predict(feats)
					pt.model.update(tags[i], guess, feats)
				}
				p2 = p1
				p1 = guess
			}
		}
		shuffle.Shuffle(sentences)
	}
	pt.model.averageWeights()
}
1026626
func (p *parser) atomWords() (a atom, start int, err error) {
	i := p.peek()
	start = i.start
	a = atom{}
	switch i.typ {
	case tokenLiteral:
		err = newParseExpError(noLiteralSupport, *i)
		return
	case tokenQuotedLiteral:
		err = newParseExpError(noQuotedLiteralSupport, *i)
		return
	case tokenColon:
		err = newParseExpError(predicateError, *i)
		return
	case tokenPredicate:
		i := p.next()
		a.predicate = i.val
	}
	for {
		switch p.peek().typ {
		case tokenColon:
			p.next()
			continue
		case tokenArg:
			i := p.next()
			a.args = append(a.args, i.val)
			continue
		case tokenQuotedArg:
			i := p.next()
			var uq string
			uq, err = strconv.Unquote(i.val)
			if err != nil {
				return
			}
			a.args = append(a.args, uq)
			continue
		}
		return
	}
}
221670
func (typ TokenType) MultiWord() bool {
	tokInfo, ok := TokenNameMap[typ]
	if ok {
		return tokInfo.HasSpaces
	}
	return false
}
245072
func Sentence() string {
	var words []string
	for i := 0; i < 3+r.Intn(12); i++ {
		word := Word()
		if r.Intn(5) == 0 {
			word += ","
		}
		words = append(words, Word())
	}

	sentence := strings.Join(words, " ")

	if r.Intn(8) == 0 {
		sentence += "!"
	} else {
		sentence += "."
	}

	return sentence
}
50110
func (parser *Parser) ParseDeclarations() ([]*css.Declaration, error) {
	result := []*css.Declaration{}

	if parser.tokenChar("{") {
		parser.shiftToken()
	}

	for parser.tokenParsable() {
		if parser.tokenIgnorable() {
			parser.shiftToken()
		} else if parser.tokenChar("}") {
			// end of block
			parser.shiftToken()
			break
		} else {
			declaration, err := parser.ParseDeclaration()
			if err != nil {
				return result, err
			}

			result = append(result, declaration)
		}
	}

	return result, parser.err()
}
986038
func ReplaceLiteral(expression, replacement, input string) (string, error) {
	re, err := stdre.Compile(expression)
	if err != nil {
		return "", err
	}
	return re.ReplaceAllLiteralString(input, replacement), nil
}
1187389
func (parser *Parser) tokenChar(value string) bool {
	token := parser.nextToken()
	return (token.Type == scanner.TokenChar) && (token.Value == value)
}
869235
func (h *Client) UserLanguage(userLanguage string) *Client {
	h.userLanguage = userLanguage
	h.userLanguageSet = true
	return h
}
59509
func newStandardTokenizer(matchVersion util.Version, input io.RuneReader) *StandardTokenizer {
	ans := &StandardTokenizer{
		Tokenizer:      NewTokenizer(input),
		maxTokenLength: DEFAULT_MAX_TOKEN_LENGTH,
	}
	ans.termAtt = ans.Attributes().Add("CharTermAttribute").(CharTermAttribute)
	ans.offsetAtt = ans.Attributes().Add("OffsetAttribute").(OffsetAttribute)
	ans.posIncrAtt = ans.Attributes().Add("PositionIncrementAttribute").(PositionIncrementAttribute)
	ans.typeAtt = ans.Attributes().Add("TypeAttribute").(TypeAttribute)
	ans.init(matchVersion)
	return ans
}