// Lex returns a pair of Token and the literal
// literal is meaningfull only for IdentifierToken token
976160
func (l *Lexer) Lex() (tok Token, lit string) {
	switch ch := l.skipWhiteSpaces(l.read()); {
	case ch == 0:
		return EndOfStringToken, ""
	case isSpecialSymbol(ch):
		l.unread()
		return l.scanSpecialSymbol()
	default:
		l.unread()
		return l.scanIdOrKeyword()
	}
}
629182
func LexIdentityOrValue(l *Lexer) StateFn {

	l.SkipWhiteSpaces()

	r := l.Peek()
	if r == '(' {
		return nil
	}
	//u.Debugf("LexIdentityOrValue identity?%v expr?%v %v peek5='%v'", l.isIdentity(), l.isExpr(), string(l.Peek()), string(l.PeekX(5)))
	// Expressions end in Parens:     LOWER(item)
	if l.isExpr() {
		return nil
	} else if l.isIdentity() {
		// Non Expressions are Identities, or Columns
		//u.Warnf("in expr is identity? %s", l.PeekWord())
		// by passing nil here, we are going to go back to Pull items off stack)
		return LexIdentifier(l)
	} else {
		//u.Warnf("LexIdentityOrValue ??? -> LexValue")
		return LexValue(l)
	}
}
184677
func lexExpression(l *Lexer) lexFunc {
	// search close mustache delimiter
	if l.isString(closeMustache) || l.isString(closeStripMustache) || l.isString(closeUnescapedStripMustache) {
		return lexCloseMustache
	}

	// search some patterns before advancing scanning position

	// "as |"
	if str := l.findRegexp(rOpenBlockParams); str != "" {
		l.pos += len(str)
		l.emit(TokenOpenBlockParams)
		return lexExpression
	}

	// ..
	if l.isString("..") {
		l.pos += len("..")
		l.emit(TokenID)
		return lexExpression
	}

	// .
	if str := l.findRegexp(rDotID); str != "" {
		l.pos += len(".")
		l.emit(TokenID)
		return lexExpression
	}

	// true
	if str := l.findRegexp(rTrue); str != "" {
		l.pos += len("true")
		l.emit(TokenBoolean)
		return lexExpression
	}

	// false
	if str := l.findRegexp(rFalse); str != "" {
		l.pos += len("false")
		l.emit(TokenBoolean)
		return lexExpression
	}

	// let's scan next character
	switch r := l.next(); {
	case r == eof:
		return l.errorf("Unclosed expression")
	case isIgnorable(r):
		return lexIgnorable
	case r == '(':
		l.emit(TokenOpenSexpr)
	case r == ')':
		l.emit(TokenCloseSexpr)
	case r == '=':
		l.emit(TokenEquals)
	case r == '@':
		l.emit(TokenData)
	case r == '"' || r == '\'':
		l.backup()
		return lexString
	case r == '/' || r == '.':
		l.emit(TokenSep)
	case r == '|':
		l.emit(TokenCloseBlockParams)
	case r == '+' || r == '-' || (r >= '0' && r <= '9'):
		l.backup()
		return lexNumber
	case r == '[':
		return lexPathLiteral
	case strings.IndexRune(unallowedIDChars, r) < 0:
		l.backup()
		return lexIdentifier
	default:
		return l.errorf("Unexpected character in expression: '%c'", r)
	}

	return lexExpression
}
515241
func (l *lexerContext) split() []token {
	result := make([]token, 0, 16)
	for !l.finished() {
		result = append(result, l.nextToken())
	}
	return result
}
1037063
func allTokens(input io.Reader) ([]Token, error) {
	l := new(lexer)
	err := l.load(input)
	if err != nil {
		return nil, err
	}
	var tokens []Token
	for l.next() {
		tokens = append(tokens, l.token)
	}
	return tokens, nil
}
1024207
func (t *tree) expect(expected lex.TokenType, context string) lex.Token {
	token := t.Cur()
	if token.T != expected {
		t.unexpected(token, context)
	}
	return token
}
277221
func lexKeyEnd(lx *lexer) stateFn {
	r := lx.next()
	switch {
	case unicode.IsSpace(r):
		return lexSkip(lx, lexKeyEnd)
	case isKeySeparator(r):
		return lexSkip(lx, lexValue)
	case r == eof:
		lx.emit(itemEOF)
		return nil
	}
	// We start the value here
	lx.backup()
	return lexValue
}
312618
func Lex(fn string, input string) (Tokens, error) {
	l := makeLexer(fn, input)

	var err error
	for true {
		newLines, indent := l.lexWhitespace()
		// If it's the end of the file, discard final whitespace.
		if l.peek() == lexEOF {
			l.next()
			l.resetTokenStart()
			break
		}
		if newLines > 0 {
			// Otherwise store whitespace in fodder.
			blanks := newLines - 1
			l.addFodder(ast.FodderLineEnd, blanks, indent, []string{})
		}
		l.resetTokenStart() // Don't include whitespace in actual token.
		r := l.next()
		switch r {
		case '{':
			l.emitToken(tokenBraceL)
		case '}':
			l.emitToken(tokenBraceR)
		case '[':
			l.emitToken(tokenBracketL)
		case ']':
			l.emitToken(tokenBracketR)
		case ',':
			l.emitToken(tokenComma)
		case '.':
			l.emitToken(tokenDot)
		case '(':
			l.emitToken(tokenParenL)
		case ')':
			l.emitToken(tokenParenR)
		case ';':
			l.emitToken(tokenSemicolon)

		case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
			l.backup()
			err = l.lexNumber()
			if err != nil {
				return nil, err
			}

			// String literals
		case '"':
			stringStartLoc := l.prevLocation()
			for r = l.next(); ; r = l.next() {
				if r == lexEOF {
					return nil, l.makeStaticErrorPoint("Unterminated String", stringStartLoc)
				}
				if r == '"' {
					// Don't include the quotes in the token data
					l.emitFullToken(tokenStringDouble, l.input[l.tokenStart+1:l.pos.byteNo-1], "", "")
					l.resetTokenStart()
					break
				}
				if r == '\\' && l.peek() != lexEOF {
					r = l.next()
				}
			}
		case '\'':
			stringStartLoc := l.prevLocation()
			for r = l.next(); ; r = l.next() {
				if r == lexEOF {
					return nil, l.makeStaticErrorPoint("Unterminated String", stringStartLoc)
				}
				if r == '\'' {
					// Don't include the quotes in the token data
					l.emitFullToken(tokenStringSingle, l.input[l.tokenStart+1:l.pos.byteNo-1], "", "")
					l.resetTokenStart()
					break
				}
				if r == '\\' && l.peek() != lexEOF {
					r = l.next()
				}
			}
		case '@':
			// Verbatim string literals.
			// ' and " quoting is interpreted here, unlike non-verbatim strings
			// where it is done later by jsonnet_string_unescape.  This is OK
			// in this case because no information is lost by resoving the
			// repeated quote into a single quote, so we can go back to the
			// original form in the formatter.
			var data []rune
			stringStartLoc := l.prevLocation()
			quot := l.next()
			var kind tokenKind
			if quot == '"' {
				kind = tokenVerbatimStringDouble
			} else if quot == '\'' {
				kind = tokenVerbatimStringSingle
			} else {
				return nil, l.makeStaticErrorPoint(
					fmt.Sprintf("Couldn't lex verbatim string, junk after '@': %v", quot),
					stringStartLoc,
				)
			}
			for r = l.next(); ; r = l.next() {
				if r == lexEOF {
					return nil, l.makeStaticErrorPoint("Unterminated String", stringStartLoc)
				} else if r == quot {
					if l.peek() == quot {
						l.next()
						data = append(data, r)
					} else {
						l.emitFullToken(kind, string(data), "", "")
						l.resetTokenStart()
						break
					}
				} else {
					data = append(data, r)
				}
			}

		default:
			if isIdentifierFirst(r) {
				l.backup()
				l.lexIdentifier()
			} else if isSymbol(r) || r == '#' {
				l.backup()
				err = l.lexSymbol()
				if err != nil {
					return nil, err
				}
			} else {
				return nil, l.makeStaticErrorPoint(
					fmt.Sprintf("Could not lex the character %s", strconv.QuoteRuneToASCII(r)),
					l.prevLocation())
			}

		}
	}

	// We are currently at the EOF.  Emit a special token to capture any
	// trailing fodder
	l.emitToken(tokenEndOfFile)
	return l.tokens, nil
}
912312
func lexFloat(lx *lexer) stateFn {
	r := lx.next()
	if unicode.IsDigit(r) {
		return lexFloat
	}

	// Not a digit, if its another '.', need to see if we falsely assumed a float.
	if r == '.' {
		return lexIPAddr
	}

	lx.backup()
	lx.emit(itemFloat)
	return lx.pop()
}
838876
func lexLine(l *lexer) stateFn {
	for {
		switch r := l.next(); {
		case r == '\n':
			l.emit(lineEnd)
			l.ignore()
			l.lineno++

			l.emit(lineStart)
		case r == ';' && l.peek() == ';':
			return lexComment
		case isSpace(r):
			l.ignore()
		case isLetter(r) || r == '_':
			return lexElement
		case isNumber(r):
			return lexNumber
		case r == '@':
			l.ignore()
			return lexLabel
		case r == '"':
			return lexInsideString
		default:
			return nil
		}
	}
}
1318176
func (p *Parser) scan() {
	for {
		token, literal := p.l.Lex()
		p.scannedItems = append(p.scannedItems, ScannedItem{token, literal})
		if token == EndOfStringToken {
			break
		}
	}
}
25997
func Tokenise(lexer Lexer, options *TokeniseOptions, text string) ([]Token, error) {
	var out []Token
	it, err := lexer.Tokenise(options, text)
	if err != nil {
		return nil, err
	}
	for t := it(); t != EOF; t = it() {
		out = append(out, t)
	}
	return out, nil
}
1127414
func lex(input string, capacity int) (*lexer, <-chan Token) {
	l := &lexer{
		input:  input,
		tokens: make(chan Token, capacity),
	}
	go l.run() // Concurrently run state machine.
	return l, l.tokens
}
1070088
func (l *lexer) Next() (*Token, Error) {
	// are we at the beginning of the line?
	_beginning := l.beginning()

	// read the next rune
	_r, _err := l.read()
	if _err != nil {
		return nil, _err
	}

	switch _r {
	// end of file
	case _EOF:
		return l.token(EOF, nil, nil)

	// whitespace ' ', '\t'
	case _SPACE:
		fallthrough
	case _TAB:
		l.unread(_r)
		_rtn, _err := l.whitespace()
		return l.token(WHITESPACE, _rtn, _err)

	// end of line '\n' or '\r\n'
	case _CR:
		fallthrough
	case _NEWLINE:
		l.unread(_r)
		_rtn, _err := l.eol()
		return l.token(EOL, _rtn, _err)

	// separator '/'
	case _SEPARATOR:
		return l.token(SEPARATOR, []rune{_r}, nil)

	// '*' or any '**'
	case _WILDCARD:
		// is the wildcard followed by another wildcard?
		//      - does this represent the "any" token (i.e. "**")
		_next, _err := l.peek()
		if _err != nil {
			return nil, _err
		} else if _next == _WILDCARD {
			// we know read() will succeed here since we used peek() above
			l.read()
			return l.token(ANY, []rune{_WILDCARD, _WILDCARD}, nil)
		}

		// we have a single wildcard, so treat this as a pattern
		l.unread(_r)
		_rtn, _err := l.pattern()
		return l.token(PATTERN, _rtn, _err)

	// comment '#'
	case _COMMENT:
		l.unread(_r)

		// if we are at the start of the line, then we treat this as a comment
		if _beginning {
			_rtn, _err := l.comment()
			return l.token(COMMENT, _rtn, _err)
		}

		// otherwise, we regard this as a pattern
		_rtn, _err := l.pattern()
		return l.token(PATTERN, _rtn, _err)

	// negation '!'
	case _NEGATION:
		if _beginning {
			return l.token(NEGATION, []rune{_r}, nil)
		}
		fallthrough

	// pattern
	default:
		l.unread(_r)
		_rtn, _err := l.pattern()
		return l.token(PATTERN, _rtn, _err)
	}
}
1311100
func (r *Lexer) Skip() {
	if r.token.kind == tokenUndef && r.Ok() {
		r.FetchToken()
	}
	r.consume()
}
543365
func LexExpressionOrIdentity(l *Lexer) StateFn {

	l.SkipWhiteSpaces()

	r := l.Peek()
	if r == '(' {
		l.Next()
		l.Emit(TokenLeftParenthesis)
		l.Push("LexParenRight", LexParenRight)
		return LexExpressionOrIdentity
	}
	// u.Debugf("LexExpressionOrIdentity identity?%v expr?%v %v peek5='%v'", l.isIdentity(), l.isExpr(), string(l.Peek()), string(l.PeekX(5)))
	// Expressions end in Parens:     LOWER(item)
	if l.isExpr() {
		return lexExpressionIdentifier(l)
	} else if l.isIdentity() {
		// Non Expressions are Identities, or Columns
		// by passing nil here, we are going to go back to Pull items off stack)
		return LexIdentifier(l)
	} else {
		//u.Warnf("LexExpressionOrIdentity ??? -> LexValue")
		return LexValue(l)
	}
}
702932
func LexDataType(forToken TokenType) StateFn {

	return func(l *Lexer) StateFn {
		l.SkipWhiteSpaces()

		//u.Debugf("LexDataType: %v", l.PeekX(5))

		// Since we escaped this with a quote we allow laxIdentifier characters
		for {
			r := l.Next()
			//u.Infof("r=%v %v    ws=%v", string(r), r, isWhiteSpace(r))
			switch {
			case r == '[' || r == ']':
				// ok, continue
			case isWhiteSpace(r):
				l.backup()
				l.Emit(forToken)
				return nil
			case isLaxIdentifierRune(r):
				//ok, continue
			case IsBreak(r):
				l.backup()
				l.Emit(forToken)
				return nil
			}
		}
	}
}