// nextToken returns the next token from the given input.
326135
func nextToken(in []byte) (*token, error) {
	left := in[:]
	parenCount := 0
	var cur token
	if len(left) == 0 {
		return nil, nil
	}

	if left[0] == '-' {
		return &token{
			Type:  tokNegate,
			Text:  []byte{'-'},
			Input: in[:1],
		}, nil
	}

	foundSpace := false

loop:
	for len(left) > 0 {
		c := left[0]
		switch c {
		case '(':
			parenCount++
			cur.Text = append(cur.Text, c)
			left = left[1:]
		case ')':
			if parenCount == 0 {
				if len(cur.Text) == 0 {
					cur.Text = []byte{')'}
					left = left[1:]
				}
				break loop
			}

			cur.Text = append(cur.Text, c)
			left = left[1:]
			parenCount--

		case '"':
			t, n, err := parseStringLiteral(left)
			if err != nil {
				return nil, err
			}
			cur.Text = append(cur.Text, t...)
			left = left[n:]
		case '\\':
			left = left[1:]
			if len(left) == 0 {
				return nil, fmt.Errorf("query: lone \\ at end")
			}
			c := left[0]
			cur.Text = append(cur.Text, '\\', c)
			left = left[1:]

		case ' ', '\n', '\t':
			if parenCount > 0 {
				foundSpace = true
			}
			break loop
		default:
			cur.Text = append(cur.Text, c)
			left = left[1:]
		}
	}

	if len(cur.Text) == 0 {
		return nil, nil
	}

	if foundSpace && cur.Text[0] == '(' {
		cur.Text = cur.Text[:1]
		cur.Input = in[:1]
	} else {
		cur.Input = in[:len(in)-len(left)]
	}
	cur.setType()
	return &cur, nil
}
1041077
func (p *parser) nextToken() Position {
	oldpos := p.tokval.pos
	p.tok = p.in.nextToken(&p.tokval)
	// enable to see the token stream
	if debug {
		log.Printf("nextToken: %-20s%+v\n", p.tok, p.tokval.pos)
	}
	return oldpos
}
785742
func (p *parser) next() (*Token, Error) {
	// do we have any previously read tokens?
	_length := len(p._undo)
	if _length > 0 {
		_token := p._undo[_length-1]
		p._undo = p._undo[:_length-1]
		return _token, nil
	}

	// otherwise, attempt to retrieve the next token from the lexer
	return p._lexer.Next()
}
154373
func (s *SearchIndexInput) SetNextToken(v string) *SearchIndexInput {
	s.NextToken = &v
	return s
}
116782
func (s *ListResolversByFunctionInput) SetNextToken(v string) *ListResolversByFunctionInput {
	s.NextToken = &v
	return s
}
778711
func (s *ListBackupPlansInput) SetNextToken(v string) *ListBackupPlansInput {
	s.NextToken = &v
	return s
}
151113
func (s *DescribeQueriesInput) SetNextToken(v string) *DescribeQueriesInput {
	s.NextToken = &v
	return s
}
977718
func (s *SearchInput) SetNextToken(v string) *SearchInput {
	s.NextToken = &v
	return s
}
524644
func (c *Compiler) next() token {
	token := c.tokens[c.pos]
	c.pos++
	return token
}
1112132
func nextToken(s string) (token, rest string) {
	i := 0
	for ; i < len(s); i++ {
		if !isTokenOctet[s[i]] {
			break
		}
	}
	return s[:i], s[i:]
}
961760
func (s *ListDatasetContentsInput) SetNextToken(v string) *ListDatasetContentsInput {
	s.NextToken = &v
	return s
}
64203
func (s *DescribeAlarmsOutput) SetNextToken(v string) *DescribeAlarmsOutput {
	s.NextToken = &v
	return s
}
631131
func (scanner *Scanner) NextToken() (s string, delim int, err error) {
	buf := bytes.NewBufferString("")
	first := true
	escape := false
	rawq := false
	infield := false
	quote := NO_QUOTE    // invalid character - not a quote
	brackets := []rune{} // stack of open brackets

	for {
		if c, _, e := scanner.in.ReadRune(); e == nil {
			//
			// check escape character
			//
			if c == ESCAPE_CHAR && !escape && !rawq {
				escape = true
				first = false
				continue
			}

			//
			// if escaping, just add the character
			//
			if escape {
				escape = false
				buf.WriteString(string(c))
				continue
			}

			//
			// checks for beginning of token
			//
			if first {
				if unicode.IsSpace(c) {
					//
					// skip leading spaces
					//
					continue
				}

				first = false

				if strings.ContainsRune(QUOTE_CHARS, c) {
					//
					// start quoted token
					//
					quote = c
					rawq = c == RAW_QUOTE
					continue
				}

				if b, ok := BRACKETS[c]; ok {
					//
					// start a bracketed session
					//
					delim = int(c)
					brackets = append(brackets, b)
					buf.WriteString(string(c))
					continue
				}

				if strings.ContainsRune(SYMBOL_CHARS, c) {
					//
					// if it's a symbol, return  all the remaining characters
					//
					buf.WriteString(string(c))
					_, err = io.Copy(buf, scanner.in)
					s = buf.String()
					return // (token, delim, err)
				}
			}

			if len(brackets) == 0 {
				//
				// terminate on spaces
				//
				if unicode.IsSpace(c) && quote == NO_QUOTE {
					s = buf.String()
					delim = int(c)
					return // (token, delim, nil)
				}

				//
				// close quote and terminate
				//
				if c == quote {
					quote = NO_QUOTE
					rawq = false
					if infield {
						buf.WriteString(string(c))
					}
					s = buf.String()
					delim = int(c)
					return // (token, delim, nil)
				}

				if scanner.InfieldBrackets {
					if b, ok := BRACKETS[c]; ok {
						//
						// start a bracketed session
						//
						brackets = append(brackets, b)
						infield = true
					}

					if quote == NO_QUOTE && strings.ContainsRune(QUOTE_CHARS, c) {
						//
						// start quoted token
						//
						quote = c
						rawq = c == RAW_QUOTE
						infield = true
					}
				}

                                if quote == NO_QUOTE && strings.ContainsRune(scanner.UserTokens, c) {
                                        //
                                        // user defined token
                                        //
                                        s = buf.String()
                                        delim = int(c)
                                        return
                                }

				//
				// append to buffer
				//
				buf.WriteString(string(c))
			} else {
				//
				// append to buffer
				//
				buf.WriteString(string(c))

				last := len(brackets) - 1

				if quote == NO_QUOTE {
					if c == brackets[last] {
						brackets = brackets[:last] // pop

						if len(brackets) == 0 {
							s = buf.String()
							return // (token, delim, nil)
						}
					} else if strings.ContainsRune(QUOTE_CHARS, c) {
						//
						// start quoted token
						//
						quote = c
						rawq = c == RAW_QUOTE
					} else if b, ok := BRACKETS[c]; ok {
						brackets = append(brackets, b)
					}
				} else if c == quote {
					quote = NO_QUOTE
					rawq = false
				}
			}
		} else {
			if e == io.EOF {
				if buf.Len() > 0 {
					s = buf.String()
					return // (token, 0, nil)
				}
			}
			err = e
			return // ("", 0, io.EOF)
		}
	}

	return
}
249196
func (s *ListThreatIntelSetsOutput) SetNextToken(v string) *ListThreatIntelSetsOutput {
	s.NextToken = &v
	return s
}
74777
func (s *SearchResourcesInput) SetNextToken(v string) *SearchResourcesInput {
	s.NextToken = &v
	return s
}
985395
func (s *DescribeChangeSetOutput) SetNextToken(v string) *DescribeChangeSetOutput {
	s.NextToken = &v
	return s
}
1362383
func (s *DescribeBudgetsOutput) SetNextToken(v string) *DescribeBudgetsOutput {
	s.NextToken = &v
	return s
}