Split source strings and taxonimize language.
@param  {string|function} src String to tokenize. If it's a function, leave it.
@return {array}      Array of objects, each a token.
961750
function _tokenize( prop ) {

  var tokens = [],
      patterns,
      src = prop.source;

  src = typeof src !== 'string' ? src : src.split(' ');

  patterns = {
    operator: /^[+\-\*\/\%]$/, // +-*/%
    // @TODO Improve this regex to cover numbers like 1,000,000
    number: /^\$?\d+[\.,]?\d+$/
  };

  function _pushToken( val, type ) {
    var token = {
      value: val,
      type: type
    };
    tokens.push( token );
  }

  // Return early if it's a function.
  if ( typeof src === 'function' ) {
    _pushToken( src, 'function' );
    return tokens;
  }

  // @TODO DRY this up.
  for ( var i = 0, len = src.length; i < len; i++ ) {
    if ( patterns.operator.test(src[i]) ) {
      _pushToken( src[i].match(patterns.operator)[0], 'operator' );
    } else if ( patterns.number.test(src[i]) ) {
      _pushToken( parseFloat( src[i].match(patterns.number)[0] ), 'number' );
    } else {
      _pushToken( src[i], 'name' );
    }
  }
  return tokens;

}
724623
function Tokenizer(str, tags) {
            this.str = str;
            this.index = 0;
            this.len = str.length;
            this.lineno = 0;
            this.colno = 0;

            this.in_code = false;

            tags = tags || {};
            this.tags = {
                BLOCK_START: tags.blockStart || BLOCK_START,
                BLOCK_END: tags.blockEnd || BLOCK_END,
                VARIABLE_START: tags.variableStart || VARIABLE_START,
                VARIABLE_END: tags.variableEnd || VARIABLE_END,
                COMMENT_START: tags.commentStart || COMMENT_START,
                COMMENT_END: tags.commentEnd || COMMENT_END
            };
        }
1106328
function Lexer(str, options) {
  options = options || {};
  if (typeof str !== 'string') {
    throw new Error('Expected source code to be a string but got "' + (typeof str) + '"')
  }
  if (typeof options !== 'object') {
    throw new Error('Expected "options" to be an object but got "' + (typeof options) + '"')
  }
  //Strip any UTF-8 BOM off of the start of `str`, if it exists.
  str = str.replace(/^\uFEFF/, '');
  this.input = str.replace(/\r\n|\r/g, '\n');
  this.originalInput = this.input;
  this.filename = options.filename;
  this.interpolated = options.interpolated || false;
  this.lineno = options.startingLine || 1;
  this.colno = options.startingColumn || 1;
  this.plugins = options.plugins || [];
  this.indentStack = [0];
  this.indentRe = null;
  // If #{} or !{} syntax is allowed when adding text
  this.interpolationAllowed = true;

  this.tokens = [];
  this.ended = false;
}
989352
function(source) {
			// transform tokens
			var tokens = this.lex(source), pos = 0, token;
			for (var i = 0, il = tokens.length; i < il; i++) {
				token = tokens[i];
				token.start = pos;
				token.end = (pos += token.value.length);
			}
			return tokens;
		}
93342
function listParser(str) {
	var list = str.split(',');
	for (var i = 0; i < list.length; i++)
		list[i] = list[i].replace(/(^\s?)|(\s?$)/g, '');
	return list;
}
359830
public static String[] tokenize(final String source, final char delimiter) {
		final Vector<String> v = tokenize2vector(source, delimiter);
		final String[] ret = new String[v.size()];
		for (int i = 0; i < v.size(); i++) {
			ret[i] = v.elementAt(i);
		}
		return ret;
	}
1364369
function (source) {
			walker.init(source);
			session.tokens = [];

			// for empty source, return single space token
			if (!source) {
				session.tokens.push(this.white());
			} else {
				while (walker.ch !== '') {
					tokenize();
				}
			}

			var tokens = session.tokens;
			session.tokens = null;
			return tokens;
		}
1099687
function(str, delimiter){

	var retlist = null;

	if( ! us.isUndefined(str) ){
	    if( us.isUndefined(delimiter) ){
		delimiter = /\s+/;
	    }
	    
	    retlist = str.split(delimiter);
	}

	return retlist;
    }
1121969
function tokensToFunction (tokens) {
    // Compile all the tokens into regexps.
    var matches = new Array(tokens.length);

    // Compile all the patterns before compilation.
    for (var i = 0; i < tokens.length; i++) {
      if (typeof tokens[i] === 'object') {
        matches[i] = new RegExp('^(?:' + tokens[i].pattern + ')$');
      }
    }

    return function (data, options) {
      var path = '';
      var encode = (options && options.encode) || encodeURIComponent;

      for (var i = 0; i < tokens.length; i++) {
        var token = tokens[i];

        if (typeof token === 'string') {
          path += token;
          continue
        }

        var value = data ? data[token.name] : undefined;
        var segment;

        if (Array.isArray(value)) {
          if (!token.repeat) {
            throw new TypeError('Expected "' + token.name + '" to not repeat, but got array')
          }

          if (value.length === 0) {
            if (token.optional) { continue }

            throw new TypeError('Expected "' + token.name + '" to not be empty')
          }

          for (var j = 0; j < value.length; j++) {
            segment = encode(value[j], token);

            if (!matches[i].test(segment)) {
              throw new TypeError('Expected all "' + token.name + '" to match "' + token.pattern + '"')
            }

            path += (j === 0 ? token.prefix : token.delimiter) + segment;
          }

          continue
        }

        if (typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean') {
          segment = encode(String(value), token);

          if (!matches[i].test(segment)) {
            throw new TypeError('Expected "' + token.name + '" to match "' + token.pattern + '", but got "' + segment + '"')
          }

          path += token.prefix + segment;
          continue
        }

        if (token.optional) { continue }

        throw new TypeError('Expected "' + token.name + '" to be ' + (token.repeat ? 'an array' : 'a string'))
      }

      return path
    }
  }
594474
function splitObject(input) {
  return input.split(",").reduce(function(obj, value) {
    obj[value.trim()] = ">=" + MIN_VERSION;
    return obj;
  }, {});
}
1245419
function deepSplitString (input, separators) {
  if (input === null || typeof input === 'undefined') {
    return input
  }
  if (separators.length === 0) {
    return input
  }

  const sep = separators.shift()
  let output = input

  if (typeof input === 'string') {
    output = splitString(input, sep)
  } else if (Array.isArray(input)) {
    output = input.map(i => splitString(i, sep))
  } else if (typeof input === 'object') {
    output = {}
    Object.keys(input).forEach(key => {
      output[key] = splitString(input[key], sep)
    })
  }

  if (separators.length > 0) {
    return deepSplitString(output, separators)
  } else {
    return output
  }
}
691427
function lex(source) {
    var tokens = [];
    var chunk = clean(source);
    var total = chunk.length;
    var iterations = 0;

    while (chunk.length > 0) {
        for (var regexpName in REGEXPS) {
            var regexp = REGEXPS[regexpName];
            var match = regexp.exec(chunk);

            if (match) {
                var original = match[0];
                var str = original;
                if (regexpName === 'string') str = str.replace(/\\"/g, '"');
                tokens.push({ type: regexpName, string: str, original: original });
                // Need to slice the chunk at the original match length
                chunk = chunk.slice(match[0].length, chunk.length);
                break;
            }
        }

        // We've probably failed to parse correctly if we get here
        if (iterations++ > total) {
            var parts = [
                'Runiq: Lexer ran too many iterations!',
                '--- Failed at chunk: `' + chunk + '`'
            ];
            throw new Error(parts.join('\n'));
        }
    }

    return tokens;
}
177896
def exec_func_src3(func, globals_, sentinal=None, verbose=False,
                   start=None, stop=None):
    
    import utool as ut
    sourcecode = ut.get_func_sourcecode(func, stripdef=True, stripret=True)
    if sentinal is not None:
        sourcecode = ut.replace_between_tags(sourcecode, '', sentinal)
    if start is not None or stop is not None:
        sourcecode = '\n'.join(sourcecode.splitlines()[slice(start, stop)])
    if verbose:
        print(ut.color_text(sourcecode, 'python'))
    six.exec_(sourcecode, globals_)
617213
function (source) {
  var ast = parser.parseEval(source)
  var array = ast.children.find((child) => child.kind === 'array')
  return parseValue(array)
}
675154
function parse_javascript(file /*, options*/) {
  let source = fs.readFileSync(file, 'utf8');

  let nodes, list = {};

  try {
    nodes = parser.parse(source);
  } catch (err) {
    throw new Error(`${err} [${file}:${err.line},${err.column}]`);
  }

  // do pre-distribute early work
  _.each(nodes, node /*, id*/ => {
    let clone;

    // assign hierarchy helpers
    node.aliases  = [];
    node.children = [];

    // set source fiel of the node
    node.file = file;

    if (node.type === 'class') {
      node.subclasses = [];
    }

    // collect sections
    if (node.type === 'section') {
      list[node.id] = node;
      return;
    }

    // elements with undefined section get '' section,
    // and will be resolved later, when we'll have full
    // element list
    list[(node.section || '') + '.' + node.id] = node;

    // bound methods produce two methods with the same description but different signatures
    // E.g. Element.foo(@element, a, b) becomes
    // Element.foo(element, a, b) and Element#foo(a, b)
    if (node.type === 'method' && node.bound) {
      clone = _.clone(node);
      clone.id = node.id.replace(/(.+)\.(.+)$/, '$1#$2');

      // link to methods
      node.bound = clone.id;
      clone.bound = node.id;

      // insert bound method clone
      list[(node.section || '') + '.' + clone.id] = clone;
    }
  });

  // TODO: section.related_to should mark related element as belonging to the section
  //_.each(list, function (node, id) {
  //  var ref_id = '.' + node.related_to, ref;
  //  if ('section' === node.type && node.related_to && list[ref_id]) {
  //    ref = list[ref_id];
  //    ref.id = node.id + '.' + node.related_to;
  //    delete list[ref_id];
  //    list[ref.id] = ref;
  //  }
  //});

  return list;
}
1040069
function splitAll(source, cb, rules) {
  var done = false
    , stash = '';
  do {
    splitNext( source, function(event, response) {
        response.stash = stash;
        if (event == 'end') done = true;
        else {
          
          source = response.rhs;
          stash += response.lhs + response.token;

          response.consume = function(length) {
            stash += source.substr( 0, length );
            source = source.substr( length, source.length );
          };

          response.resetStash = function() {
            stash = '';
          };
          
          response.break = function() {
            done = true; 
          };
        }
        cb( event, response );
      }, 
      rules );
  } 
  while(!done);
}
932018
protected Collection<String> normalize(String string, final boolean tokenize) {
        if (string == null) {
            return Collections.emptyList();
        }
        if (tokenize) {
            final Collection<String> result = new LinkedHashSet<>();

            result.addAll(normalize(string, false));

            final Splitter splitter = Splitter.on(' ').omitEmptyStrings();
            final List<String> tokens = splitter.splitToList(string);
            for (final String token : tokens) {
                final Collection<String> normalizedTokens = normalize(token, false);
                result.addAll(normalizedTokens);
            }
            return result;
        } else {
            string = StringUtils.replaceWhitespaces(string, "");
            string = StringUtils.replaceAll(string, "-", "");
            string = StringUtils.replaceAll(string, "_", "");
            string = StringUtils.replaceAll(string, "|", "");
            string = StringUtils.replaceAll(string, "*", "");
            string = string.toUpperCase();
            if (string.isEmpty()) {
                return Collections.emptyList();
            }
            final String withoutNumbers = string.replaceAll("[0-9]", "");
            if (withoutNumbers.equals(string) || withoutNumbers.isEmpty()) {
                return Arrays.asList(string);
            }
            return Arrays.asList(string, withoutNumbers);
        }
    }