Returns an object of identifiers with
empty string or NO-OP function
values.

@private
@param {Array} tokens
@return {Object}
491375
function getIdentifiersFromTokens (tokens) {
  const identifiers = {}

  /**
   * Predicate to determine if token is an identifier.
   *
   * @private
   * @param {Object} token
   * @return {Boolean}
   */

  const isIdentifier = token => 'Identifier' == token.type

  /**
   * Mark token as a function identifier.
   *
   * @private
   * @param {Object} token
   * @param {Number} index
   * @return {Object} token
   */

  const markFunction = (token, index) => {
    const next = tokens[index + 1] || null
    token.isFunction = ( 'Identifier' == token.type
                        && 'object' == typeof next && next
                        && 'Punctuator' == next.type
                        && '(' == next.value
                          ? true : false )
    return token
  }

  /**
   * Mark token as a object identifier.
   *
   * @private
   * @param {Object} token
   * @param {Number} index
   * @return {Object} token
   */

  const markObject = (token, index) => {
    const next = tokens[index + 1] || null
    token.isObject = ( 'Identifier' == token.type
                      && 'object' == typeof next && next
                      && 'Punctuator' == next.type
                      && '.' == next.value
                        ? true : false )
    return token
  }

  /**
   * Assign token value to identifierss map.
   *
   * @private
   * @param {Object} map
   * @param {Object} token
   * @return {Object} map
   */

  const assign = (map, token) => {
    const value = token.value
    if (token.isFunction)
      map[value] = _ => ''
    else if (token.isObject)
      map[value] = {}
    else
      map[value] = ''
    return map
  }

  // resolve identifierss and return map
  return ( tokens
          .map((t, i) => markFunction(t, i))
          .map((t, i) => markObject(t, i))
          .filter(t => isIdentifier(t))
          .reduce((map, t) => assign(map, t), identifiers) )
}
615982
function getToken() {
  tokenType = TOKENTYPE.NULL;
  token = '';

  // skip over whitespaces: space, tab, newline, and carriage return
  while (c === ' ' || c === '\t' || c === '\n' || c === '\r') {
    next();
  }

  // check for delimiters
  if (DELIMITERS[c]) {
    tokenType = TOKENTYPE.DELIMITER;
    token = c;
    next();
    return;
  }

  // check for a number
  if (isDigit(c) || c === '-') {
    tokenType = TOKENTYPE.NUMBER;

    if (c === '-') {
      token += c;
      next();

      if (!isDigit(c)) {
        throw createSyntaxError('Invalid number, digit expected', index);
      }
    }
    else if (c === '0') {
      token += c;
      next();
    }
    else {
      // digit 1-9, nothing extra to do
    }

    while (isDigit(c)) {
      token += c;
      next();
    }

    if (c === '.') {
      token += c;
      next();

      if (!isDigit(c)) {
        throw createSyntaxError('Invalid number, digit expected', index);
      }

      while (isDigit(c)) {
        token += c;
        next();
      }
    }

    if (c === 'e' || c === 'E') {
      token += c;
      next();

      if (c === '+' || c === '-') {
        token += c;
        next();
      }

      if (!isDigit(c)) {
        throw createSyntaxError('Invalid number, digit expected', index);
      }

      while (isDigit(c)) {
        token += c;
        next();
      }
    }

    return;
  }

  // check for a string
  if (c === '"') {
    tokenType = TOKENTYPE.STRING;
    next();

    while (c !== '' && c !== '"') {

      if (c === '\\') {
        // handle escape characters
        next();

        let unescaped = ESCAPE_CHARACTERS[c];
        if (unescaped !== undefined) {
          token += unescaped;
          next();
        }
        else if (c === 'u') {
          // parse escaped unicode character, like '\\u260E'
          next();

          let hex = '';
          for (let u = 0; u < 4; u++) {
            if (!isHex(c)) {
              throw createSyntaxError('Invalid unicode character');
            }
            hex += c;
            next();
          }

          token += String.fromCharCode(parseInt(hex, 16));
        }
        else {
          throw createSyntaxError('Invalid escape character "\\' + c + '"', index);
        }
      }
      else {
        // a regular character
        token += c;
        next();
      }
    }

    if (c !== '"') {
      throw createSyntaxError('End of string expected');
    }
    next();

    return;
  }

  // check for symbols (true, false, null)
  if (isAlpha(c)) {
    tokenType = TOKENTYPE.SYMBOL;

    while (isAlpha(c)) {
      token += c;
      next();
    }

    return;
  }

  // something unknown is found, wrong characters -> a syntax error
  tokenType = TOKENTYPE.UNKNOWN;
  while (c !== '') {
    token += c;
    next();
  }
  throw createSyntaxError('Syntax error in part "' + token + '"');
}
501564
function nestTokens(tokens) {
    var tree = [];
    var collector = tree;
    var sections = [];
    var token, section;

    for (var i = 0; i < tokens.length; ++i) {
      token = tokens[i];

      switch (token[0]) {
      case "#":
      case "^":
        token[4] = [];
        sections.push(token);
        collector.push(token);
        collector = token[4];
        break;
      case "/":
        if (sections.length === 0) {
          throw new Error("Unopened section: " + token[1]);
        }

        section = sections.pop();

        if (section[1] !== token[1]) {
          throw new Error("Unclosed section: " + section[1]);
        }

        if (sections.length > 0) {
          collector = sections[sections.length - 1][4];
        } else {
          collector = tree;
        }
        break;
      default:
        collector.push(token);
      }
    }

    // Make sure there were no open sections when we're done.
    section = sections.pop();

    if (section) {
      throw new Error("Unclosed section: " + section[1]);
    }

    return tree;
  }
372458
function () {
            if (vm.inputText == "" || tokenizer == null) {
                vm.tokens = [];
                // lattice = null;
                return;
            }
            try {
                // lattice = tokenizer.getLattice(vm.inputText);
                vm.tokens = tokenizer.tokenize(vm.inputText);
            } catch (e) {
                console.log(e);
                // lattice = null;
                vm.tokens = [];
            }
        }
608148
function(tokens) {
            	if(typeof tokens === "string") {
            		tokens = [tokens];
            	}
                    var results = [];
            	var rule_count = rules$3.length;
            	var num_tokens = tokens.length;
                    var i, token, r, rule;
                
                    for (i = 0; i < num_tokens; i++) {
                        token = tokens[i];
                        // Check the conversion table
                        if (conversionTable[token.toLowerCase()]) {
                            results = results.concat(conversionTable[token.toLowerCase()].split(/\W+/));
                        }
                        
                        // Apply the rules
                        else {
                            var matched = false;
                            for ( r = 0; r < rule_count; r++) {
                                rule = rules$3[r];
                                if (token.match(rule.regex)) {
                                    results = results.concat(token.replace(rule.regex, rule.output).split(/\W+/));
                                    matched = true;
                                    break;
                                }
                            }
                            if (!matched) {
                                results.push(token);
                            }
                        }
                    }

            	return results;
            }
1357620
function getTokensLength(tokens) {
  if (typeof tokens === 'string') {
    return tokens.length;
  }
  return tokens.reduce((acc, token) => {
    return acc + (token.length ? token.length : 0);
  }, 0);
}
886761
function(context: ParserContext): ?Token {
    const delimiter = context.reader.peek();
    if (delimiter !== '"' && delimiter !== "'") return null;
    if (!isInsideOpenBracket(context)) return null;

    // read until the delimiter
    let stringValue = context.reader.read();
    while (!context.reader.isEOF()) {
      const next = context.reader.read();
      stringValue += next;

      if (next === delimiter) break;
    }

    return context.createToken("string", stringValue);
  }
26107
function implodeArray(tokens) {
	var message = "", i, l;
	l = tokens.length;
	for (i = 0; i < l; i++) {
		message += (i > 0 ? globals.SEP : "") + tokens[i];
	}
	return message;
}
1205287
function squashTokens (tokens) {
    var squashedTokens = [];

    var token, lastToken;
    for (var i = 0, numTokens = tokens.length; i < numTokens; ++i) {
      token = tokens[i];

      if (token) {
        if (token[0] === 'text' && lastToken && lastToken[0] === 'text') {
          lastToken[1] += token[1];
          lastToken[3] = token[3];
        } else {
          squashedTokens.push(token);
          lastToken = token;
        }
      }
    }

    return squashedTokens;
  }
882134
function getPrefixTokens (node) {
    const prefixes = []

    let token = tokenStore.getFirstToken(node)
    while (token != null && token.range[1] <= node.key.range[0]) {
      prefixes.push(token)
      token = tokenStore.getTokenAfter(token)
    }
    while (isLeftParen(last(prefixes)) || isLeftBracket(last(prefixes))) {
      prefixes.pop()
    }

    return prefixes
  }
87858
function (token) {
        var any = token.peek();
        if (any !== "any")
            throw _cs.exception("validate", "parse error: invalid any type \"" + any + "\"");
        token.skip();
        return { type: "any" };
    }
475761
function() {
    var tok;
    this.skipSpaces();

    // Parenthesis
    if (this.accept('(')) {
      ++this.parens;
      var expr = this.expression()
        , paren = this.expect(')');
      --this.parens;
      if (this.accept('%')) expr.push(new nodes.Ident('%'));
      tok = this.peek();
      // (1 + 2)px, (1 + 2)em, etc.
      if (!paren.space
        && 'ident' == tok.type
        && ~units.indexOf(tok.val.string)) {
        expr.push(new nodes.Ident(tok.val.string));
        this.next();
      }
      return expr;
    }

    tok = this.peek();

    // Primitive
    switch (tok.type) {
      case 'null':
      case 'unit':
      case 'color':
      case 'string':
      case 'literal':
      case 'boolean':
      case 'comment':
        return this.next().val;
      case !this.cond && '{':
        return this.object();
      case 'atblock':
        return this.atblock();
      // property lookup
      case 'atrule':
        var id = new nodes.Ident(this.next().val);
        id.property = true;
        return id;
      case 'ident':
        return this.ident();
      case 'function':
        return tok.anonymous
          ? this.functionDefinition()
          : this.functionCall();
    }
  }
766444
def t_ANY_ID(self, token):
        '
        if token.value in self.KEYWORDS:
            if (token.value == 'annotation_type') and self.cur_indent:
                # annotation_type was added as a reserved keyword relatively
                # late, when there could be identifers with the same name
                # in existing specs. because annotation_type-the-keyword can
                # only be used at the beginning of a non-indented line, this
                # check lets both the keyword and the identifer coexist and
                # maintains backward compatibility.
                # Note: this is kind of a hack, and we should get rid of it if
                # the lexer gets better at telling keywords from identifiers in general.
                return token
            token.type = self.RESERVED.get(token.value, 'KEYWORD')
            return token
        else:
            return token
1087489
function processIdentifiers(tokenizer, scope) {
  let params = [];

  if (tokenizer.token.match(types.IDENTIFIER)) {
    const keys = [tokenizer.token.value];

    tokenizer.skipToken();

    while (tokenizer.token.match(types.DOT, types.OPEN_SQUARE_BRACKET)) {
      const token = tokenizer.token;

      tokenizer.skipToken();

      const isValidKey = token.match(types.OPEN_SQUARE_BRACKET) ?
        tokenizer.token.match(types.CONSTANT) :
        (
          (tokenizer.token.match(types.CONSTANT) && (typeof tokenizer.token.value === 'number')) ||
          tokenizer.token.match(types.IDENTIFIER)
        );

      if (!isValidKey) {
        throw new SyntaxError('Unexpected object key notation');
      }

      keys.push(tokenizer.token.value);
      tokenizer.skipToken();

      if (token.match(types.OPEN_SQUARE_BRACKET)) {
        if (!tokenizer.token.match(types.CLOSE_SQUARE_BRACKET)) {
          throw new SyntaxError('Closing square bracket expected');
        }

        tokenizer.skipToken();
      }
    }

    const identifier = keys.join('.');

    if (!tokenizer.token.match(types.OPEN_PARENTHESES)) {
      scope.identifiers.push(identifier);

      return createIdentifierNode(identifier);
    }

    params = [];
    tokenizer.skipToken();
    if (!tokenizer.token.match(types.CLOSE_PARENTHESES)) {
      params.push(processLogicalOr(tokenizer, scope));

      while (tokenizer.token.match(types.DELIMITER)) {
        params.push(processLogicalOr(tokenizer.skipToken(), scope));
      }
    }

    if (!tokenizer.token.match(types.CLOSE_PARENTHESES)) {
      throw new SyntaxError('Unexpected end of expression');
    }

    tokenizer.skipToken();
    return createOperatorNode(identifier, params);
  }

  return processConstants(tokenizer, scope);
}
1112956
function removeTokenFromString(token, string) {
        var tokens = split(string),
            index = tokens.indexOf(token);
        if (index !== -1) {
          tokens.splice(index, 1);
        }
        return tokens.join(' ');
      }
1332828
function() {
    var captures
      , tok;
    if (captures = /^(not|and|or|is a|is defined|isnt|is not|is)(?!-)\b([ \t]*)/.exec(this.str)) {
      var op = captures[1];
      this.skip(captures);
      if (this.isPartOfSelector()) {
        tok = new Token('ident', new nodes.Ident(captures[0]));
      } else {
        op = alias[op] || op;
        tok = new Token(op, op);
      }
      tok.space = captures[2];
      return tok;
    }
  }
1207287
function(source) {
    var proto = Lexer.prototype;
    var tokens = source.match(proto.regs.token);
    for (var i = 0, l = tokens.length; i < l; i ++) {
        if (proto.regs.strip.test(tokens[i])) {
            tokens.splice(i, 1);
        }
    }
    for (var n in proto) tokens[n] = proto[n];
    tokens.index = 0;
    return tokens;
}