// TokenizeString is a convenience method
190159
func (l Lexer) TokenizeString(s string) ([]Token, error) {
	r := bufio.NewReader(strings.NewReader(s))
	tokens := []Token{}
	err := l.Tokenize(r, func(t Token) error {
		tokens = append(tokens, t)
		return nil
	})
	return tokens, err
}
496911
func Tokenise(expr string) []string {

	spaced := unary_minus_rx.ReplaceAllString(expr, "$1 @")
	spaced = fp_rx.ReplaceAllString(spaced, " ${1} ")
	spaced = functions_rx.ReplaceAllString(spaced, " ${1} ")

	if symbols_rx != nil {
		spaced = symbols_rx.ReplaceAllString(spaced, " ${1} ")
	}

	symbols := []string{"(", ")"}
	for _, symbol := range symbols {
		spaced = strings.Replace(spaced, symbol, fmt.Sprintf(" %s ", symbol), -1)
	}

	stripped := whitespace_rx.ReplaceAllString(strings.TrimSpace(spaced), "|")
	result := strings.Split(stripped, "|")
	return result
}
975483
func (p *DefaultWordTokenizer) Tokenize(text string, onlyPeriodContext bool) []*Token {
	textLength := len(text)

	if textLength == 0 {
		return nil
	}

	tokens := make([]*Token, 0, 50)
	lastSpace := 0
	lineStart := false
	paragraphStart := false
	getNextWord := false

	for i, char := range text {
		if !unicode.IsSpace(char) && i != textLength - 1 {
			continue
		}

		if char == '\n' {
			if lineStart {
				paragraphStart = true
			}
			lineStart = true
		}

		var cursor int
		if i == textLength - 1 {
			cursor = textLength
		} else {
			cursor = i
		}

		word := strings.TrimSpace(text[lastSpace:cursor])

		if word == "" {
			continue
		}

		hasSentencePunct := p.PunctStrings.HasSentencePunct(word)
		if onlyPeriodContext && !hasSentencePunct && !getNextWord {
			lastSpace = cursor
			continue
		}

		token := NewToken(word)
		token.Position = cursor
		token.ParaStart = paragraphStart
		token.LineStart = lineStart
		tokens = append(tokens, token)

		lastSpace = cursor
		lineStart = false
		paragraphStart = false

		if hasSentencePunct {
			getNextWord = true
		} else {
			getNextWord = false
		}
	}

	if len(tokens) == 0 {
		token := NewToken(text)
		token.Position = textLength
		tokens = append(tokens, token)
	}

	return tokens
}
458265
func SanitizeString(s string) string {
	var out bytes.Buffer
	allowed := "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-"
	for _, c := range s {
		if strings.IndexRune(allowed, c) != -1 {
			out.WriteRune(c)
		} else {
			out.WriteRune('_')
		}
	}

	return string(out.Bytes())
}
561257
func ParseString(s string) (*Query, error) {
	return NewParser(strings.NewReader(s)).Parse()
}
494232
func ParseString(data string, unmarshal func([]byte, interface{}) error) (*Config, error) {
	conf := &Config{}
	if err := unmarshal([]byte(data), conf); err != nil {
		return nil, err
	}

	if err := conf.Sanitize(); err != nil {
		return nil, err
	}
	return conf, nil
}
56997
func GetString(n ast.Node) (string, error) {
	if node, ok := n.(*ast.BasicLit); ok && node.Kind == token.STRING {
		return strconv.Unquote(node.Value)
	}
	return "", fmt.Errorf("Unexpected AST node type: %T", n)
}
1266076
func (i *Iter) InitString(f Form, src string) {
	i.p = 0
	if len(src) == 0 {
		i.setDone()
		i.rb.nsrc = 0
		return
	}
	i.multiSeg = nil
	i.rb.initString(f, src)
	i.next = i.rb.f.nextMain
	i.asciiF = nextASCIIString
	i.info = i.rb.f.info(i.rb.src, i.p)
}
432131
func (i tokenType) String() string {
	s := tokenName[i]
	if s == "" {
		return fmt.Sprintf("t_unknown_%d", int(i))
	}
	return s
}
938184
func (r RegexpTokenizer) Tokenize(text string) []string {
	var tokens []string
	if r.gaps {
		temp := r.regex.Split(text, -1)
		if r.discard {
			for _, s := range temp {
				if s != "" {
					tokens = append(tokens, s)
				}
			}
		} else {
			tokens = temp
		}
	} else {
		tokens = r.regex.FindAllString(text, -1)
	}
	return tokens
}
1270633
func (t Token) String() string {
	return fmt.Sprintf("%s %s %s", t.Pos.String(), t.Type.String(), t.Text)
}
35623
func jsonStringify(b interface{}) string {
	out, err := json.MarshalIndent(b, "", "  ")
	if err != nil {
		return fmt.Sprint("error:", err)
	}
	return string(out)
}
903885
func TokenResource(uri string) string {
	resource := uri
	if !strings.HasSuffix(resource, "/") {
		resource += "/"
	}
	return resource
}
239656
func (inst *DenseInstances) RowString(row int) string {
	var buffer bytes.Buffer
	first := true
	for _, p := range inst.ags {
		if first {
			first = false
		} else {
			buffer.WriteString(" ")
		}
		p.appendToRowBuf(row, &buffer)
	}
	return buffer.String()
}
569447
func (t *SimpleTokenizer) Tokenize(sentence string) []string {
	// is the tokenizer really the best place to be making
	// everything lowercase? is this really a sanitizaion func?
	return strings.Split(strings.ToLower(sentence), t.SplitOn)
}
161576
func (t TokenType) String() string {
	switch t {
	case ILLEGAL:
		return "ILLEGAL"
	case EOF:
		return "EOF"
	case EOL:
		return "EOL"
	case WHITESPACE:
		return "WHITESPACE"
	case COMMENT:
		return "COMMENT"
	case SEPARATOR:
		return "SEPARATOR"
	case NEGATION:
		return "NEGATION"
	case PATTERN:
		return "PATTERN"
	case ANY:
		return "ANY"
	default:
		return "BAD TOKEN"
	}
}
878480
func (m Metadata) JsonString() (string, error) {
	// NB: method defined on value
	b, err := json.Marshal(m)
	return string(b), err
}