call the appropriate handler based on the first character in a token suspect
430003
function tokenize() {
		var ch = walker.ch;
	
		if (ch === " " || ch === "\t") {
			return white();
		}

		if (ch === '/') {
			return comment();
		} 

		if (ch === '"' || ch === "'") {
			return str();
		}
		
		if (ch === '(') {
			return brace();
		}
	
		if (ch === '-' || ch === '.' || isDigit(ch)) { // tricky - char: minus (-1px) or dash (-moz-stuff)
			return num();
		}
	
		if (isNameChar(ch)) {
			return identifier();
		}

		if (isOp(ch)) {
			return op();
		}

		if (ch === '\r') {
			if (walker.peek() === '\n') {
				ch += walker.nextChar();
			}

			tokener(ch, 'line');
			walker.nextChar();
			return;
		}
		
		if (ch === '\n') {
			tokener(ch, 'line');
			walker.nextChar();
			return;
		}
		
		raiseError("Unrecognized character '" + ch + "'");
	}
1100083
function expect(value) {
  var token = lex();
  if (token.type !== TokenPunctuator || token.value !== value) {
    throwUnexpected(token);
  }
}
1167569
def _sexp_slash_handler(c, ctx, whence=None, pending_event=None):
    
    assert c == _SLASH
    if whence is None:
        whence = ctx.whence
    c, self = yield
    ctx.queue.unread(c)
    if c == _ASTERISK or c == _SLASH:
        yield ctx.immediate_transition(_comment_handler(_SLASH, ctx, whence))
    else:
        if pending_event is not None:
            # Since this is the start of a new value and not a comment, the pending event must be emitted.
            assert pending_event.event is not None
            yield _CompositeTransition(pending_event, ctx, partial(_operator_symbol_handler, _SLASH))
        yield ctx.immediate_transition(_operator_symbol_handler(_SLASH, ctx))
1131209
function parse(rbp, initial) {
        var left;
        var o;
        if (nexttoken.id === '(end)') {
            error("Unexpected early end of program.", token);
        }
        advance();
        if (option.adsafe && token.value === 'ADSAFE') {
            if (nexttoken.id !== '.' || !(peek(0).identifier) ||
                    peek(1).id !== '(') {
                warning('ADsafe violation.', token);
            }
        }
        if (initial) {
            anonname = 'anonymous';
            funct['(verb)'] = token.value;
        }
        if (initial && token.fud) {
            left = token.fud();
        } else {
            if (token.nud) {
                o = token.exps;
                left = token.nud();
            } else {
                if (nexttoken.type === '(number)' && token.id === '.') {
                    warning(
"A leading decimal point can be confused with a dot: '.{a}'.",
                            token, nexttoken.value);
                    advance();
                    return token;
                } else {
                    error("Expected an identifier and instead saw '{a}'.",
                            token, token.id);
                }
            }
            while (rbp < nexttoken.lbp) {
                o = nexttoken.exps;
                advance();
                if (token.led) {
                    left = token.led(left);
                } else {
                    error("Expected an operator and instead saw '{a}'.",
                        token, token.id);
                }
            }
            if (initial && !o) {
                warning(
"Expected an assignment or function call and instead saw an expression.",
                        token);
            }
        }
        if (!option.evil && left && left.value === 'eval') {
            warning("eval is evil.", left);
        }
        return left;
    }
353837
private void checkTokenOrderIsCorrect(ESqlToken tokenTobeAdded) {

        switch (tokenTobeAdded) {

        case COLUMN:
            if (mPreviousToken != ESqlToken.CONDITION && mPreviousToken != ESqlToken.NOTHING) {
                throw new D6RuntimeException("AND or OR method is required at here." + " '" + mSgSQL.toString() + "'<==error occurred around here.");
            }
            break;

        case OPERATOR:
            if (mPreviousToken != ESqlToken.COLUMN) {
                throw new D6RuntimeException("VALUE is required at here." + " '" + mSgSQL.toString() + "'<==error occurred around here.");
            }
            break;

        case VALUE:
            if (mPreviousToken != ESqlToken.OPERATOR) {
                throw new D6RuntimeException("COLUMN or PROPERTY method is required at here." + " '" + mSgSQL.toString() + "'<==error occurred around here.");
            }
            break;

        case CONDITION:
            if (mPreviousToken != ESqlToken.VALUE && mPreviousToken != ESqlToken.WHERE) {
                throw new D6RuntimeException("VALUE is required at here." + " '" + mSgSQL.toString() + "'<==error occurred around here.");
            }
            break;

        case WHERE:
            if (mPreviousToken != ESqlToken.CONDITION && mPreviousToken != ESqlToken.NOTHING) {
                throw new D6RuntimeException("CONDITION is required at here." + " '" + mSgSQL.toString() + "'<==error occurred around here.");
            }
            break;
        default:
        }

        mPreviousToken = tokenTobeAdded;
    }
219609
def _process(self, name):
        """"""
        if self.token.nature == name:
            self.token = self.lexer.next_token()
        else:
            self._error()
225029
function _checkIsHttpToken(val) {
	if (typeof val !== 'string' || val.length === 0) {
		return false;
	}
	if (!_isValidTokenChar(val.charCodeAt(0))) {
		return false;
	}
	const len = val.length;
	if (len > 1) {
		if (!_isValidTokenChar(val.charCodeAt(1))) {
			return false;
		}
		if (len > 2) {
			if (!_isValidTokenChar(val.charCodeAt(2))) {
				return false;
			}
			if (len > 3) {
				if (!_isValidTokenChar(val.charCodeAt(3))) {
					return false;
				}
				for (var i = 4; i < len; i++) {
					if (!_isValidTokenChar(val.charCodeAt(i))) {
						return false;
					}
				}
			}
		}
	}
	return true;
}
1137427
def _lob_start_handler(c, ctx):
    """"""
    assert c == _OPEN_BRACE
    c, self = yield
    trans = ctx.immediate_transition(self)
    quotes = 0
    while True:
        if c in _WHITESPACE:
            if quotes > 0:
                _illegal_character(c, ctx)
        elif c == _DOUBLE_QUOTE:
            if quotes > 0:
                _illegal_character(c, ctx)
            ctx.set_ion_type(IonType.CLOB).set_unicode(quoted_text=True)
            yield ctx.immediate_transition(_short_string_handler(c, ctx))
        elif c == _SINGLE_QUOTE:
            if not quotes:
                ctx.set_ion_type(IonType.CLOB).set_unicode(quoted_text=True)
            quotes += 1
            if quotes == 3:
                yield ctx.immediate_transition(_long_string_handler(c, ctx))
        else:
            yield ctx.immediate_transition(_blob_end_handler(c, ctx))
        c, _ = yield trans
535767
function(context: ParserContext): ?Token {
    if (!context.reader.match("/")) return null;

    const previousNonWsToken = context.token(context.count() - 1);
    let previousToken = null;
    if (context.defaultData.text !== "")
      previousToken = context.createToken("default", context.defaultData.text);

    if (!previousToken) previousToken = previousNonWsToken;

    if (
      // The first token of the string
      previousToken !== undefined &&
      // Since Ruby doesn't require statement terminators, if the previous token
      // was whitespace and contained a newline, then we're good.
      (previousToken.name !== "default" ||
        previousToken.value.indexOf("\n") < 0)
    ) {
      // In these case the / stands for division.
      if (
        util.contains(["keyword", "ident", "number"], previousNonWsToken.name)
      )
        return null;
      if (
        previousNonWsToken.name === "punctuation" &&
        !util.contains(["(", "{", "[", ","], previousNonWsToken.value)
      )
        return null;
    }

    // read the regex literal
    let regexLiteral = context.reader.read();
    let charClass = false;
    while (!context.reader.isEOF()) {
      const next = context.reader.read();
      regexLiteral += next;

      if (next === "\\") regexLiteral += context.reader.read();
      else if (next === "[") charClass = true;
      else if (next === "]") charClass = false;
      else if (next === "/" && !charClass) break;
    }

    // Read the regex modifiers. For the sake of simplicity we accept all
    // characters, but currently only "ioxmusen" are supported.
    while (!context.reader.isEOF() && /[A-Za-z]/.test(context.reader.peek()))
      regexLiteral += context.reader.read();

    return context.createToken("regexLiteral", regexLiteral);
  }
1261433
def handle_token(cls, parser, token):
        """"""
        tokens = token.contents.split()
        if tokens[1] != 'for':
            raise template.TemplateSyntaxError(
                "Second argument in %r tag must be 'for'" % tokens[0]
            )

        # {% get_whatever for obj as varname %}
        if len(tokens) == 5:
            if tokens[3] != 'as':
                raise template.TemplateSyntaxError("Third argument in %r must be 'as'" % tokens[0])
            return cls(
                object_expr=parser.compile_filter(tokens[2]),
                as_varname=tokens[4],
            )

        # {% get_whatever for app.model pk as varname %}
        elif len(tokens) == 6:
            if tokens[4] != 'as':
                raise template.TemplateSyntaxError("Fourth argument in %r must be 'as'" % tokens[0])
            return cls(
                ctype = BaseCommentNode.lookup_content_type(tokens[2], tokens[0]),
                object_pk_expr = parser.compile_filter(tokens[3]),
                as_varname = tokens[5]
            )

        else:
            raise template.TemplateSyntaxError("%r tag requires 4 or 5 arguments" % tokens[0])
934523
async def handler(event):
    """"""
    q1 = event.pattern_match.group(1)
    q2 = urllib.parse.quote(q1)
    await asyncio.wait([
        event.delete(),
        event.respond(DOCS.format(q1, q2), reply_to=event.reply_to_msg_id)
    ])
395396
function(context: ParserContext): ?Token {
    if (
      !context.reader.match("<<") ||
      !/[\w'"`-]/.test(context.reader.peekWithOffset(2))
    )
      return null;

    // cannot be preceded by an a number or a string
    const walker = context.getTokenWalker();
    if (walker.hasPrev()) {
      const prevToken = walker.prev();
      if (util.contains(["number", "string"], prevToken.name)) return null;
    }

    // there are still cases where heredocs are falsely detected, because it
    // would require performingstatic analysis

    // e.g. foo <<a
    // if foo is an object that has the "<<" method defined, then it will perform a left shift
    // if foo is a function that takes a string argument, it will interpret it as a heredoc

    // so, we just force you to have whitespace between << and the rhs operand in these ambiguous cases

    // can be between quotes (double, single or back) or not, or preceded by a hyphen
    let value = context.reader.read(2);
    let ident = "";
    let current = context.reader.read();
    if (current === "-") {
      value += current;
      ident += current;
      current = context.reader.read();
    }

    let delimiter = "";
    if (util.contains(['"', "'", "`"], current)) delimiter = current;
    else ident += current;

    value += current;

    while (!context.reader.isEOF()) {
      const peek = context.reader.peek();
      if (peek === "\n" || (delimiter === "" && /\W/.test(peek))) break;
      value += context.reader.read();
      if (peek === delimiter) break;
      ident += peek;
    }

    if (Array.isArray(context.items.heredocQueue))
      context.items.heredocQueue.push(ident);
    else
      logger.errorInvalidValue(
        `context.items.heredocQueue is not an array.`,
        context.items.heredocQueue
      );

    return context.createToken("heredocDeclaration", value);
  }
285191
def parse(self):
        """"""
        token_type = self.current_token.type.lower()
        handler = getattr(self, f'parse_{token_type}', None)
        if handler is None:
            raise self.error(f'Invalid literal {self.current_token.value!r}')
        return handler()
483856
private static void handleRemainingCharIfAny(final String messagePattern, final int len,
            final StringBuilder buffer, final int escapeCounter, final int i) {
        if (i == len - 1) {
            final char curChar = messagePattern.charAt(i);
            handleLastChar(buffer, escapeCounter, curChar);
        }
    }
639091
function(tokenStr) {
            if (typeof tokenStr !== 'string') {
                return null;
            }
            var token = tokenCache[tokenStr];
            if (token !== undefined) {
                return token;
            } else {
                try {
                    token = tokens.validate(tokenStr, trustedPubKey);
                    tokenCache[tokenStr] = token;
                    return token;
                } catch (err) {
                    // did not validate, cache negative result.
                    tokenCache[tokenStr] = null;
                    return null;
                }
            }
        }
1338283
function firstStrongChar(str: string): ?string {
  const match = REGEX_STRONG.exec(str);
  return match == null ? null : match[0];
}
239507
function handleEscaping(tokens, token, index) {
	if (index === 0) {
		return [token];
	}

	const previousToken = tokens[index - 1];

	if (!previousToken.endsWith('\\')) {
		return [...tokens, token];
	}

	return [...tokens.slice(0, index - 1), `${previousToken.slice(0, -1)} ${token}`];
}