Compile a token string into a function.

@param {string} token
@param {string} arg
@return {function}
@private
977145
function compileToken (token, arg) {
  let fn = tokens[token] || noOp

  return function () {
    return arg ? fn.call(this, arg) : fn.call(this)
  }
}
997188
function parser(tokens) {
  // add Close token before each Else
  const toks = tokens.reduce((prev, tok) => {
    if (tok.tokenType === 'Else') {
      return [...prev, new Close(), tok];
    }

    return [...prev, tok];
  }, []);

  const [body] = parse(toks, '', 1);
  return body;
}
396923
function extract(token) {
        var obj = {};
        Object.keys(token).map(function(k) {
            if ( !ignore.has(k) ) {
                obj[k] = token[k];
            }
        });
        return obj;
    }
130250
function (token) {
	if (!token) {
		return '';
	}
	if (typeof token === 'string') {
		return token;
	}
	if (typeof token.content === 'string') {
		return token.content;
	}
	return token.content.map(stringifyToken).join('');
}
263606
function Parser( tokenStream ){
   
    var debug = false;

    var currentToken, lastComment, types = [];
   
    function isAnyOf( type, possibleTypes ){
        for(var i = 0; i < possibleTypes.length; i++) {
            if ( possibleTypes[i] === type ){ 
                return true;
            }
        }
    
        return false;
    }

    function error( message ){
        throw new Error( message + 
            "\nAt line: " + currentToken.line + ", col: " +  currentToken.col + "\n\n" +
            tokenStream.neighborhood()
        );
    }

    function consume( expectedType ){
        currentToken = tokenStream.consume();

        if (debug) console.log("currentToken", currentToken);

        if ( expectedType && !isAnyOf( currentToken.type, Array.prototype.slice.call(arguments) )){
            error( "syntax error - expected \"" + Array.prototype.join.call( arguments, ", ") + "\", but got \"" + currentToken.type + "\"" ); 
        } 

        return currentToken;
    }

    function peak(){
        return tokenStream.peak();
    }

    function parseExtendsList(){
        
        consume("extends");

        return parseIdList();
    }
   

    function parseExtends(){
        
        consume("extends");

        // get the type name
        var r = consume();
        return r.contents;
    }
   
    function parseIdList(){

        var idlist = []
            , peaked;

        do {
            consume();

            if ( currentToken.type === "id" ){
                idlist.push( currentToken.contents );
            }
        } while ( (peaked = peak()) && (peaked.type === "id" || peaked.type === ",") )

        return idlist;
    }

    function parseImplements(){
         
        consume("implements");
        return parseIdList();

    }

    function parseClass( visibility ){
     
        // get visibility from current token or default
        visibility = visibility || "public";
        
        consume("class");  
       
        // get the type name for the class
        var typeName = consume().contents;
    
        // build the class type
        var def = new Class( typeName, visibility ); 
        def.line = currentToken.line;
    
        // set the description
        def.description = getLastComment();

        var peaked = peak(); 
        var parsedExtends = false;
    
        // parent class
        if ( peaked.type === "extends" ){
            parsedExtends = true;
            def.parentClass = parseExtends();
            peaked = peak();
        }
        
        // parse interfaces
        if ( peaked.type === "implements" ){
            def.interfaces = parseImplements();                       
            peaked = peak();
        }
 
        // parent class
        if ( peaked.type === "extends" ){
            if (parsedExtends) error("Duplicate extends encountered");
            def.parentClass = parseExtends();
        }       

        parseClassBody( def );

        if (debug) console.log(def);

        return def; 
    }

    function consumeBlock(){

        consume("{");
        var parenCount = 1;

        while ( parenCount != 0 ){
            consume();

            if (currentToken.type === "{") {
                parenCount++;
            } else if (currentToken.type === "}"){
                parenCount--;
            }
        }
    }

    function consumeWhile(){
        
        var possibleTypes = Array.prototype.slice.call(arguments);
        
        var consumed = [];
        while (isAnyOf( peak().type, possibleTypes)){
            consumed.push(consume.apply(null, arguments));
        }
        return consumed;
    }

    function consumeIf(){

        var possibleTypes = Array.prototype.slice.call(arguments);
        var peaked = peak();
        if (isAnyOf(peaked.type, possibleTypes)) consume();
    }

    function consumeUntil(){

        var possibleTypes = Array.prototype.slice.call(arguments);
        var peaked;
        while ( (peaked = peak()) && !isAnyOf( peaked.type, possibleTypes) ){
            consume();
        }
    }

    function consumeUntilInclusive(char){
        consumeUntil(char);
        consume(char);
    }

    function parseProperty( isStatic ){

        var currentLine = currentToken.line;

        consume("var");

        var name = consume().contents;
        var type;

        var peaked = peak();
        
        if (peaked.type === ":"){
           
            consume(":");
            type = consume("id").contents; 
            peaked = peak();
            
        }
       
        consumeUntilInclusive(";");

        var prop = new Property( name, type, getLastComment() ); 
        prop.line = currentLine;
        return prop;
    }
  
    function parseMethodArgument(){
    
        // id : Type = expression
        var name = consume("id").contents;
      
        var peaked = peak();

        var type;
        if (peaked.type === ":"){
            type = parseTypeAnnotation();
            peaked = peak();
        }

        var defaultVal;
        if (peaked.type === "="){    
            consume("=");
            defaultVal = consume("number", "string", "boolean", "null").contents; 
        }

        return new MethodArgument(name, type, defaultVal);
    }

    function parseTypeAnnotation(){
        consume(":");
        var ids = consumeWhile("id","->");
        return ids.reduce((a,x) => x.contents + a, "");
    }

    function parseMethodArguments(){

        // ( methodArgument, methodArgument )
        consume("(");

        var peaked, args = [];

        while ( (peaked = peak()) && peaked.type != ")"){
            
            if (peaked.type === "id"){
                args.push( parseMethodArgument() );
            } else if (peaked.type === ","){
                consume(","); // consume the ","
            } else {
                error("Failed to parse method arguments!");                
            }

        }
       
        consume(")");
        return args;
    }

    function getLastComment(){

        var lc = lastComment;
        lastComment = undefined;
        return lc ? lc.contents : lc;

    }

    function parseMethod( isStatic ){
    
        var currentLine = currentToken.line;

        // function MethodName( methodArgument, methodArgument ) : ExpectedType { ... } 
        consume("function");

        var name = consume("id", "new").contents;
        
        var args = parseMethodArguments();

        peaked = peak();
   
        var type;
        if (peaked.type === ":"){
           type = parseTypeAnnotation(); 
        }
    
        consumeBlock();

        var method = new Method( name, args, type, getLastComment() );
        method.line = currentLine;
        method.isStatic = isStatic;
        return method;
    }

    function parseClassMember( def, visibility ){

        // parse "public" or "private"
        var visibility = visibility ? visibility : consume("visibility").contents;     

        // hack to ignore private members
        if ( visibility === "private"){
            def = new Class();
        }

        var peaked = peak();
        var isStatic = false;
    
        if ( peaked.type === "static" ){
            isStatic = true;
            
            consume("static");
            peaked = peak();
        }

        if ( peaked.type === "inline" ){
           
            // TODO - parse inline
            consume("inline");
            peaked = peak();
        }
        
        if ( peaked.type === "var" ){
            return def.properties.push( parseProperty( isStatic ));
        } else if (peaked.type === "function"){
            return def.methods.push( parseMethod( isStatic )); 
        }

        throw new Error("Unknown class member encountered" + JSON.stringify( peaked ));
    }

    function parseClassBody( def ){

        consume("{");

        var peaked; 

        while ( (peaked = peak()) && peaked.type != "}"){
            if (peaked.type === "comment"){    
                parseComment();
                continue;
            } else if (peaked.type === "visibility"){
                parseClassMember( def );
                continue;
            } else if (peaked.type === "function"){
                parseClassMember( def, "private" );
                continue;
            } else if (peaked.type === "var"){
                parseClassMember( def, "private" );
                continue;
            } else if (peaked.type === "static"){
                parseClassMember( def, "private" );
                continue; 
            }

            consume();
        }

        consume("}");
    }

    function parseMethodDefinition(){
    
        // function MethodName( methodArgument, methodArgument ) : ExpectedType; 
        consume("function");

        var currentLine = currentToken.line;
        var name = consume("id", "new").contents;
        
        var args = parseMethodArguments();

        peaked = peak();
   
        var type;
        if (peaked.type === ":"){
           type = parseTypeAnnotation(); 
        }
    
        consumeUntilInclusive(";");

        var method = new Method( name, args, type, getLastComment() );
        method.line = currentLine;
        return method;
    }

    function parseInterfaceMember( def, visibility ){

        var visibility = visibility ? visibility : consume("visibility").contents;     

        // hack to ignore private members
        if ( visibility === "private"){
            def = new Interface();
        }

        var peaked = peak();
    
        if ( peaked.type === "var" ){
            return def.properties.push( parseProperty());
        } else if (peaked.type === "function"){
            return def.methods.push( parseMethodDefinition()); 
        }

        throw new Error("Unknown interface member type encountered" + JSON.stringify( peaked ));
    }
    
    function parseInterfaceBody( def ){

        consume("{");

        var peaked; 

        while ( (peaked = peak()) && peaked.type != "}"){
            if (peaked.type === "comment"){    
                parseComment();
                continue;
            } else if (peaked.type === "visibility"){
                parseInterfaceMember( def );
                continue;
            } else if (peaked.type === "function"){
                parseInterfaceMember( def, "public" );
                continue;
            }  

            consume();
        }

        consume("}");
    }

    function parseInterface( visibility ){

        visibility = visibility || "public";
        
        consume("interface");  
       
        // get the type name for the class
        var typeName = consume().contents;
    
        var def = new Interface( typeName, visibility ); 
        def.line = currentToken.line; 
        def.description = getLastComment();
        
        if ( peak().type === "extends" ){
            def.interfaces = parseExtendsList();
        }
        
        parseInterfaceBody( def );

        if (debug) console.log( def );

        return def; 
    }

    function parseTypedefBody( def ){

        def.isAnonymousObject = true;
        consume("{");

        var peaked; 

        while ( (peaked = peak()) && peaked.type != "}"){
            if (peaked.type === "comment"){    
                parseComment();
                continue;
            } else if (peaked.type === "var"){
                def.anonymousObject.properties.push(parseProperty());
                continue;
            } 

            consume();
        }

        consume("}");
    }

    function parseTypedef( visibility ){
      
        visibility = visibility || "public";
        
        consume("typedef");  
       
        var typeName = consume().contents;
   
        consume("=");

        var def = new Typedef( typeName, visibility ); 
        def.line = currentToken.line;
        def.description = getLastComment();
      
        // anonymous object body
        if (peak().type === "{"){
            parseTypedefBody( def );
        } else {
           def.alias = consume("id").contents;
        }

        if( debug ) console.log( def );

        return def; 
    }

    function parseComment() {
        var squashed = consume("comment");

        var peaked;
        while ( (peaked = peak()) && peaked.type === "comment" ){
            currentToken = consume();
            
            squashed.contents += "\n";
            squashed.contents += currentToken.contents; 
        }
    
        return lastComment = squashed;
    }

    function parseType(){ 
        
        var peaked = peak();
        var visibility = "public";
        
        if (peaked.type === "visibility"){
            consume("visibility");
            peaked = peak();
        }

        if (peaked.type === "class"){
            return parseClass( visibility ); 
        } else if (peaked.type === "interface"){
            return parseInterface( visibility ); 
        } else if (peaked.type === "typedef"){
            return parseTypedef( visibility );    
        } 

        error("Parser for type not implemented");    
    }
    
    this.parse = () => {
       
        var peaked;

        var types = [];
    
        while ( (peaked = peak()) ){
            if (peaked.type === "expose"){
                consume("expose"); 
                types.push( parseType() ); 
            } else if (peaked.type === "class"){
                parseClass(); 
            } else if (peaked.type === "typedef"){
                types.push( parseTypedef() ); 
            } else if (peaked.type === "interface"){
                types.push( parseInterface() ); 
            } else if ( peaked.type === "visibility" ){
                if (peaked.contents === "private") {
                    // parse, but don't add to types
                    parseType();
                    continue;
                }
                types.push( parseType() ); 
            } else if (peaked.type === "comment"){
                parseComment();
            } else {
                consume();
            }
        }

        return types;
    }
}
987616
function compile(prop, name){
  var output = {};

  if(prop.__params) return prop.__params;

  var str = prop.toString(), //.replace(/(?:\/\*(?:[\s\S]*?)\*\/)|(?:([\s;])+\/\/(?:.*)$)/gm, '$1'), // String representation of function sans comments
      token = tokenizer(str, {
        ecmaVersion: 6,
        sourceType: 'script'
      }),
      finishedPaths = [],
      listening = 0,
      paths = [],
      attrs = [],
      workingpath = [];

  do {

    // console.log(token.type.label, token.value);
    token.nextToken();

    if(token.value === 'this'){
      listening++;
      workingpath = [];
    }

    // TODO: handle gets on collections
    if(token.value === 'get'){
      token.nextToken();
      while(_.isUndefined(token.value)){
        token.nextToken();
      }
      // Replace any access to a collection with the generic @each placeholder and push dependancy
      workingpath.push(token.value.replace(/\[.+\]/g, ".@each").replace(/^\./, ''));
    }

    if(token.value === 'pluck'){
      token.nextToken();
      while(_.isUndefined(token.value)){
        token.nextToken();
      }

      workingpath.push('@each.' + token.value);
    }

    if(token.value === 'slice' || token.value === 'clone' || token.value === 'filter'){
      token.nextToken();
      if(token.type.label === '(') workingpath.push('@each');
    }

    if(token.value === 'at'){
      token.nextToken();
      while(_.isUndefined(token.value)){
        token.nextToken();
      }
      workingpath.push('@each');
    }

    if(token.value === 'where' || token.value === 'findWhere'){
      workingpath.push('@each');
      token.nextToken();
      attrs = [];
      var itr = 0;
      while(token.type.label !== ')'){
        if(token.value){
          if(itr%2 === 0){
            attrs.push(token.value);
          }
          itr++;
        }
        token.nextToken();
      }
      workingpath.push(attrs);
    }

    if(listening && (_.indexOf(TERMINATORS, token.type.label) > -1 || _.indexOf(TERMINATORS, token.value) > -1)){
      workingpath = _.reduce(workingpath, reduceMemos, ['']);
      finishedPaths = _.compact(_.union(finishedPaths, workingpath));
      workingpath = [];
      listening--;
    }

  } while (token.start !== token.end && token.type !== tokTypes.eof);

  // Save our finished paths directly on the function
  prop.__params = finishedPaths;

  // Return the dependancies list
  return finishedPaths;

}
291002
function() {
    var captures;
    if (captures = /^u\+[0-9a-f?]{1,6}(?:-[0-9a-f]{1,6})?/i.exec(this.str)) {
      this.skip(captures);
      return new Token('literal', new nodes.Literal(captures[0]));
    }
  }
84249
function process_token(token, index, tokens) {
	var last = (index === (tokens.length - 1));
	if (token[0] === "+") return process_single(token, last);
	else if (token[0] === "#") return process_multi(token, last);
	else return process_raw(token, last);
}
133270
function compile(args) {
  var filenames = [];
  var opts = {
    grammar: null,
    b: false,  // Benchmark (print matching times).
    v: false   // Verbose
  };

  // Super basic command line option parsing.
  for (var i = 0; i < args.length; ++i) {
    if (args[i] === '-g') {
      opts.grammar = args[++i];
    } else if (args[i][0] === '-' && opts.hasOwnProperty(args[i][1])) {
      opts[args[i][1]] = true;
    } else {
      filenames.push(args[i]);
    }
  }

  var lang = opts.grammar ? loadModule(opts.grammar) : es5;

  var files = filenames.map(function(name) {
    // If benchmarking, read in all the files at once, so that we can just time the matching.
    return [name, opts.b ? removeShebang(fs.readFileSync(name).toString()) : null];
  });

  var matchStartTime = Date.now();

  // Parsing -- bails out when the first error is encountered.
  var results = [];
  var succeeded = files.every(function(arr) {
    var source = arr[1] || removeShebang(fs.readFileSync(arr[0]).toString());
    if (opts.v) { console.error(arr[0]); }
    var result = lang.grammar.match(source, 'Program');
    if (result.succeeded()) {
      results.push(result);
      return true;
    }
    console.error(arr[0] + ':\n' + result.message);
    return false;
  });

  if (succeeded) {
    if (opts.b) { console.error('Matching:', (Date.now() - matchStartTime) + 'ms'); }

    // Codegen
    var code = '';
    var codegenStartTime = Date.now();
    results.forEach(function(r) { code += ';\n' + lang.semantics(r).toES5(); });
    if (opts.b) {
      console.error('Codegen:', (Date.now() - codegenStartTime) + 'ms');
    }
    return code;
  }
  return null;
}
192721
function compile (tree) {
    if (typeof tree === "string") tree = parse(tree);

    // Every compiled function gets called with a reference to the runtime.
    return new Function(["args", "runt"], [
      'var ret = [];',
      compileArgumentList(tree),
      'return ret;'
    ].join('\n'));
  }
1112956
function removeTokenFromString(token, string) {
        var tokens = split(string),
            index = tokens.indexOf(token);
        if (index !== -1) {
          tokens.splice(index, 1);
        }
        return tokens.join(' ');
      }
886761
function(context: ParserContext): ?Token {
    const delimiter = context.reader.peek();
    if (delimiter !== '"' && delimiter !== "'") return null;
    if (!isInsideOpenBracket(context)) return null;

    // read until the delimiter
    let stringValue = context.reader.read();
    while (!context.reader.isEOF()) {
      const next = context.reader.read();
      stringValue += next;

      if (next === delimiter) break;
    }

    return context.createToken("string", stringValue);
  }
483128
function parse(tokens) {
    var token;
    return process();

    function process(operation) {
        operation = operation || null;
        var args = [];

        while (next()) {
            if (token.type == 'boundary') {
                if (token.value == '(') args.push(process());
                else if (token.value == ')') return node(operation, args);
            }

            else if (token.type == 'variable') {
                args.push(node('substitution', [ token.value ]));
                if (isUnary(operation)) return node(operation, args);
            }

            else if (token.type == 'operator') {
                if (isUnary(token.value)) {
                    args.push(process(token.value));
                    continue;
                }

                if (operation) {
                    var tmp = args.slice(0);
                    args = [];
                    args.push(node(operation, tmp));
                }

                operation = token.value;
            }
        }

        return node(operation, args);
    }

    function next() {
        //TODO: use pointer instead of shifting
        //(parse would not need to clone tokens array)
        return (token = tokens.shift());
    }

    function node(action, args) {
        return {
            action: translate(action),
            args: args
        };
    }

    function translate(operator) {
        var map = {
            '!': 'negation',
            '|': 'disjunction',
            '&': 'conjunction',
            '->': 'implication',
            '<->': 'equivalence'
        };

        return map[operator] || operator;
    }

    function isUnary(op) {
        return op === '!';
    }

    function syntaxError() {
        throw new Error('Syntax error!');
    }
}
913344
public static String render(List<Token> tokens, Map<String, Object> arguments) {
    StringBuilder builder = new StringBuilder();
    for (Token token : tokens) {
      builder.append(token.render(arguments));
    }

    return builder.toString();
  }
1055740
function (arg, fn, opts) {

  if (typeof arg === T_STRING) {

    // 2nd parameter is optional, but can be null
    if (isObject(fn)) {
      opts = fn;
      fn = false;
    }

    // `riot.compile(tag [, callback | true][, options])`
    if (/^\s*</m.test(arg)) {
      var js = compiler.compile(arg, opts);
      if (fn !== true) { globalEval(js); }
      if (isFunction(fn)) { fn(js, arg, opts); }
      return js
    }

    // `riot.compile(url [, callback][, options])`
    GET(arg, function (str, opts, url) {
      var js = compiler.compile(str, opts, url);
      globalEval(js, url);
      if (fn) { fn(js, str, opts); }
    }, opts);

  } else if (isArray(arg)) {
    var i = arg.length;
    // `riot.compile([urlsList] [, callback][, options])`
    arg.forEach(function(str) {
      GET(str, function (str, opts, url) {
        var js = compiler.compile(str, opts, url);
        globalEval(js, url);
        i --;
        if (!i && fn) { fn(js, str, opts); }
      }, opts);
    });
  } else {

    // `riot.compile([callback][, options])`
    if (isFunction(arg)) {
      opts = fn;
      fn = arg;
    } else {
      opts = arg;
      fn = undefined;
    }

    if (ready) {
      return fn && fn()
    }

    if (promise) {
      if (fn) { promise.on('ready', fn); }

    } else {
      promise = observable$1();
      compileScripts(fn, opts);
    }
  }

}
826168
function parseVariable(str, line) {
    var tokens = lexer.read(utils.strip(str)),
      parser,
      out;

    parser = new TokenParser(tokens, filters, escape, line, opts.filename);
    out = parser.parse().join('');

    if (parser.state.length) {
      utils.throwError('Unable to parse "' + str + '"', line, opts.filename);
    }

    /**
     * A parsed variable token.
     * @typedef {object} VarToken
     * @property {function} compile Method for compiling this token.
     */
    return {
      compile: function () {
        return '_output += ' + out + ';\n';
      }
    };
  }
1340387
def string_token
      if %w(' ").include?(@chunk[0])
        result = process_string @chunk[0..-1], @chunk[0]
        if @looking_for_args
          token :Argument, result.str, attrs: { quoted_by: @chunk[0] }
        else
          token :Command, result.str
        end
        result.consumed_length
      else
        did_not_match
      end
    end