// helper to parse words (i.e space delimited or quoted strings) in a statement.
// The quotes are preserved as part of this function and they are stripped later
// as part of processWords().
973631
func parseWords(rest string, d *Directive) []string {
	const (
		inSpaces = iota // looking for start of a word
		inWord
		inQuote
	)

	words := []string{}
	phase := inSpaces
	word := ""
	quote := '\000'
	blankOK := false
	var ch rune
	var chWidth int

	for pos := 0; pos <= len(rest); pos += chWidth {
		if pos != len(rest) {
			ch, chWidth = utf8.DecodeRuneInString(rest[pos:])
		}

		if phase == inSpaces { // Looking for start of word
			if pos == len(rest) { // end of input
				break
			}
			if unicode.IsSpace(ch) { // skip spaces
				continue
			}
			phase = inWord // found it, fall through
		}
		if (phase == inWord || phase == inQuote) && (pos == len(rest)) {
			if blankOK || len(word) > 0 {
				words = append(words, word)
			}
			break
		}
		if phase == inWord {
			if unicode.IsSpace(ch) {
				phase = inSpaces
				if blankOK || len(word) > 0 {
					words = append(words, word)
				}
				word = ""
				blankOK = false
				continue
			}
			if ch == '\'' || ch == '"' {
				quote = ch
				blankOK = true
				phase = inQuote
			}
			if ch == d.escapeToken {
				if pos+chWidth == len(rest) {
					continue // just skip an escape token at end of line
				}
				// If we're not quoted and we see an escape token, then always just
				// add the escape token plus the char to the word, even if the char
				// is a quote.
				word += string(ch)
				pos += chWidth
				ch, chWidth = utf8.DecodeRuneInString(rest[pos:])
			}
			word += string(ch)
			continue
		}
		if phase == inQuote {
			if ch == quote {
				phase = inWord
			}
			// The escape token is special except for ' quotes - can't escape anything for '
			if ch == d.escapeToken && quote != '\'' {
				if pos+chWidth == len(rest) {
					phase = inWord
					continue // just skip the escape token at end
				}
				pos += chWidth
				word += string(ch)
				ch, chWidth = utf8.DecodeRuneInString(rest[pos:])
			}
			word += string(ch)
		}
	}

	return words
}
35136
func Sentences(text string) []string {
	sentences := sentenceRegex.Split(text, -1)

	if sentences[len(sentences)-1] == "" {
		return sentences[0 : len(sentences)-1]
	}

	return sentences
}
245072
func Sentence() string {
	var words []string
	for i := 0; i < 3+r.Intn(12); i++ {
		word := Word()
		if r.Intn(5) == 0 {
			word += ","
		}
		words = append(words, Word())
	}

	sentence := strings.Join(words, " ")

	if r.Intn(8) == 0 {
		sentence += "!"
	} else {
		sentence += "."
	}

	return sentence
}
15860
func quote(s string) string {
	q := make([]byte, 2+2*len(s))
	i := 0
	q[i], i = '"', i+1
	for _, c := range []byte(s) {
		if c == '"' {
			q[i], i = '\\', i+1
			q[i], i = '"', i+1
		} else {
			q[i], i = c, i+1
		}
	}
	q[i], i = '"', i+1
	return string(q[:i])
}
410040
func (list *DefinitionList) parseAndAppendString(text string) {
	emptyLineRE := regexp.MustCompile("^[[:space:]]*$")
	depthRE := regexp.MustCompile("^( *)(.*)")

	startDefRE := regexp.MustCompile("^ *- (.*)")
	keyedDefRE := regexp.MustCompile("^([^:]+):[[:space:]]*(.*)")

	descTextRE := regexp.MustCompile("(?sUm:^(.*) *-)")

	var (
		currentList  *DefinitionList
		currentItem  *Definition
		currentDepth int
	)

	descMatch := descTextRE.FindStringSubmatch(text)
	if len(descMatch) > 1 {
		list.desc = descMatch[1]
		text = text[len(descMatch[1]):]
	}

	currentList = list

	for lineNo, line := range strings.Split(text, "\n") {
		// nesting depth == nr. of indentation spaces
		if emptyLineRE.MatchString(line) {
			continue
		}
		newDepth := len(depthRE.ReplaceAllString(line, "$1"))

		if startDefRE.MatchString(line) {
			// Start new definition
			start := startDefRE.ReplaceAllString(line, "$1")

			newItem := &Definition{}
			if keyedDefRE.MatchString(start) {
				newItem.name = keyedDefRE.ReplaceAllString(start, "$1")
				newItem.desc = keyedDefRE.ReplaceAllString(start, "$2") + "\n"

			} else {
				newItem.name = ""
				newItem.desc = start + "\n"
			}

			if newDepth == currentDepth {
				// Keep current nesting level
				if currentItem != nil {
					newItem.parent = currentItem.parent
				}

			} else if newDepth > currentDepth {
				// Dive deeper
				newItem.parent = currentItem
				currentItem.children = DefinitionList{}
				currentList = &currentItem.children

			} else if newDepth < currentDepth {
				// One up
				if currentItem.parent.parent != nil {
					currentList = &currentItem.parent.parent.children
				} else {
					currentList = list
				}
			}

			currentList.add(newItem)
			currentItem = newItem
			currentDepth = newDepth

		} else if currentItem == nil {
			// Parse error
			panic(fmt.Sprintf("Comment line %d: Parse error: currentItem == nil "+
				"near \"%s\" (plain paragraphs mixed with definition list?)\n", lineNo, line))

		} else if newDepth != currentDepth {
			// Parse error
			panic(fmt.Sprintf("Comment line %d: Indentation error: expected indentation "+
				"to remain at %d, found %d near \"%s\"\n", lineNo, currentDepth, newDepth, line))

		} else {
			// Append to current definition (without indentation)
			currentItem.desc += depthRE.ReplaceAllString(line, "$2") + "\n"
		}
	}
}
220648
func (d *WordDecoder) Decode(word string) (string, error) {
	fields := strings.Split(word, "?") // TODO: remove allocation?
	if len(fields) != 5 || fields[0] != "=" || fields[4] != "=" || len(fields[2]) != 1 {
		return "", errInvalidWord
	}

	content, err := decode(fields[2][0], fields[3])
	if err != nil {
		return "", err
	}

	buf := getBuffer()
	defer putBuffer(buf)

	if err := d.convert(buf, fields[1], content); err != nil {
		return "", err
	}

	return buf.String(), nil
}
1059685
func (tokenizer *Tokenizer) parseString() (string, error) {
	start := tokenizer.scanner.Index()
	lastChar := tokenizer.scanner.Char()
	tokenizer.scanner.Advance()
	for tokenizer.CanRead() {
		if tokenizer.scanner.Char() == '"' {
			if lastChar == '\\' {
				lastChar = tokenizer.scanner.Char()
				tokenizer.scanner.Advance()
				continue
			}
			str := tokenizer.scanner.Substring(start, tokenizer.scanner.Index()+1)
			lang := ""
			datatype := ""
			canPeek, nextChar := tokenizer.scanner.Peek()
			var err error
			if canPeek {
				switch nextChar {
				case '@':
					tokenizer.scanner.Advance()
					lang = tokenizer.parseLanguage()
					str += lang
				case '^':
					tokenizer.scanner.Advance()
					datatype, err = tokenizer.parseType()
					str += datatype
				}
			}
			return str, err
		}
		lastChar = tokenizer.scanner.Char()
		tokenizer.scanner.Advance()
	}
	return "", tokenizer.Error("String did not end with \"")
}
1028257
func ParseString(f interface{}) (string, error) {
	if s, ok := f.(string); ok {
		return s, nil
	}

	// Useful for tests
	if q, ok := f.(Quoted); ok {
		return string(q), nil
	}
	if a, ok := f.(Atom); ok {
		return string(a), nil
	}

	if l, ok := f.(Literal); ok {
		b := make([]byte, l.Len())
		if _, err := io.ReadFull(l, b); err != nil {
			return "", err
		}
		return string(b), nil
	}

	return "", newParseError("expected a string")
}
1302083
func (i *Identifier) String() string {
	if i.Unquoted != nil {
		return *i.Unquoted
	}
	return string(*i.Quoted)
}
629776
func parse(s string) (string, string) {
	if s == "/" {
		return s, ""
	}
	for i := 1; i < len(s); i++ {
		if s[i] == '/' {
			return s[0:i], s[i:]
		}
	}

	return s, ""
}
804712
func ParseCommand(line string) *Command {
	words := strings.Split(line, " ")
	command := strings.ToUpper(words[0])
	args := strings.Join(words[1:len(words)], " ")

	return &Command{
		verb: command,
		args: args,
		orig: line,
	}
}
1268985
func (u Utils) QuotationOrSpaceFields(s string) []string {
	lastQuote := rune(0)
	f := func(c rune) bool {
		switch {
		case c == lastQuote:
			lastQuote = rune(0)
			return false
		case lastQuote != rune(0):
			return false
		case unicode.In(c, unicode.Quotation_Mark):
			lastQuote = c
			return false
		default:
			return unicode.IsSpace(c)
		}
	}

	args := strings.FieldsFunc(s, f)
	for i := range args {
		args[i] = u.replacer.Replace(args[i])
	}
	return args
}
1007053
func Parse(input string) (result *ast.Program, err error) {
	// recover error
	defer errRecover(&err)

	parser := new(input)

	// parse
	result = parser.parseProgram()

	// check last token
	token := parser.shift()
	if token.Kind != lexer.TokenEOF {
		// Parsing ended before EOF
		errToken(token, "Syntax error")
	}

	// fix whitespaces
	processWhitespaces(result)

	// named returned values
	return
}
226104
func (p *parser) parseStmtList() (list []ast.Stmt) {
	if p.trace {
		defer un(trace(p, "StatementList"))
	}

	for p.tok != token.CASE && p.tok != token.DEFAULT && p.tok != token.RBRACE && p.tok != token.EOF {
		list = append(list, p.parseStmt())
	}

	return
}
206370
func (p *Parser) parseQuote(cur *ListNode, end rune) error {
Loop:
	for {
		switch p.next() {
		case eof, '\n':
			return fmt.Errorf("unterminated quoted string")
		case end:
			//if it's not escape break the Loop
			if p.input[p.pos-2] != '\\' {
				break Loop
			}
		}
	}
	value := p.consumeText()
	s, err := UnquoteExtend(value)
	if err != nil {
		return fmt.Errorf("unquote string %s error %v", value, err)
	}
	cur.append(newText(s))
	return p.parseInsideAction(cur)
}
324831
func (a Statements) String() string {
	var str []string
	for _, stmt := range a {
		str = append(str, stmt.String())
	}
	return strings.Join(str, ";\n")
}
824519
func (p *Lexer) Quoted(ind int) (string, bool, error) {

	c1, _ := p.Byte()
	if c1 != '"' && c1 != '\'' {
		p.UnreadByte()
		return "", false, nil
	}

	var buf []byte
	var c2 byte

	for {
		c, _ := p.Byte()
		if IsEndChar(c) {
			return "", false, ErrUnterminatedQuotedString
		}

		if c == c1 && c2 != '\\' {
			break
		}
		if c == '\\' {
			c2 = c
			continue
		}

		// \" -> "
		// \' -> '
		if c2 == '\\' && !(c != '\'' || c == '"') {
			buf = append(buf, '\\')
		}

		buf = append(buf, c)

		if c == 10 {
			n, u := p.Space()
			if u == 0 {
				return "", false, ErrSpaceNotUniform
			}
			// There are n spaces. Skip lnl spaces and add rest.
			for ; n-ind > 0; n-- {
				buf = append(buf, u)
			}
		}
		c2 = c
	}

	return string(buf), true, nil
}