/*
Token is an object like:

{
"value": "text content",

Position of the index
"index": 0,

Length of the token
"offset": "text content".length
}
 Return a string ID to represent a list of tokens
487702
function tokensId(tokens, prepend) {
    if (!prepend) return null;

    return _.reduce(tokens, function(prev, token) {
        var tokId = [
            token.index,
            token.offset,
            token.value
        ].join('-');

        return [prev, tokId].join(':');
    }, prepend);
}
1357620
function getTokensLength(tokens) {
  if (typeof tokens === 'string') {
    return tokens.length;
  }
  return tokens.reduce((acc, token) => {
    return acc + (token.length ? token.length : 0);
  }, 0);
}
189904
function tokenStackLex() {
    var token;
    token = tstack.pop() || lexer.lex() || EOF;
    // if token isn't its numeric value, convert
    if (typeof token !== 'number') {
        if (token instanceof Array) {
            tstack = token;
            token = tstack.pop();
        }
        token = self.symbols_[token] || token;
    }
    return token;
}
582175
function(token) {
  if (token === undefined || token === null) {
    return {
      nextPartitionKey: undefined,
      nextRowKey:       undefined,
    };
  }
  assert(typeof token === 'string', 'Continuation token must be a string if ' +
                                     'not undefined');
  // Split at tilde (~)
  token = token.split('~');
  assert(token.length === 2, 'Expected an encoded continuation token with ' +
                             'a single tilde as separator');
  return {
    nextPartitionKey: decodeURIComponent(token[0]),
    nextRowKey:       decodeURIComponent(token[1]),
  };
}
239780
public String getIdFromToken(String token) {
        try {
            final Claims claims = getClaimsFromToken(token);
            return (String) claims.get(CLAIM_KEY_ID);
        } catch (Exception e) {
            log.error("get id from jwt token error : {}", e.getMessage());
            return null;
        }
    }
560632
function(value) {
		var ret = [];
		var linesAndNewlines = value.split(/(\n|\r\n)/);
		// Ignore the final empty token that occurs if the string ends with a new line
		if (!linesAndNewlines[linesAndNewlines.length - 1]) {
			linesAndNewlines.pop();
		}
		// Merge the content and line separators into single tokens
		for (var i = 0; i < linesAndNewlines.length; i++) {
			var line = linesAndNewlines[i];
			if (i % 2) {
				ret[ret.length - 1] += line;
			} else {
				ret.push(line);
			}
		}
		return ret;
	}
825768
private TextToken text(Token prev) {
    int startIndex = curCharIndex;

    while (curChar != '{' && curChar != EOF) {
      consume();
    }
    return new TextToken(prev, curCharIndex - startIndex);
  }
25161
function getWordToken(str) {
    var word = str.split(/\s/g)[0];
    return {text: word, type: 'WORD', length: word.length};
}
494275
function extractToken(token) {
    var regex = /^Bearer\s(\S+)$/i;
    var result = regex.exec(token);
    return result ? result[1] : false;
}
225248
function token(input) {
  let token;
  
  if (typeof input === "string") {
    token = input;
  } else if (Buffer.isBuffer(input)) {
    token = input.toString("hex");
  }

  token = token.replace(/[^0-9a-f]/gi, "");

  if (token.length === 0) {
    throw new Error("Token has invalid length");
  }

  return token;
}
1166234
function getMarkedId (text) {
  return entities.decode(text)
    .toLowerCase()
    .trim()
    .replace(/[\u2000-\u206F\u2E00-\u2E7F\\'!"#$%&()*+,./:;<=>?@[\]^`{|}~_]/g, '')
    .replace(/\s/g, '-')
}
902948
function parseEmphasis(tokens) {
  const newTokens = [];
  let i = 0;
  while (i < tokens.length) {
    if (typeof tokens[i] === 'string') {
      if (tokens[i] !== '') {
        newTokens.push(tokens[i]);
      }
      i++;
    } else {
      const currTag = tokens[i].type;
      if (INLINE_OPEN.indexOf(currTag) !== -1) {
        const closeTag = getCloseTag(currTag);
        const closePos = getClosePos(tokens, i, closeTag, tokens[i].markup);
        if (closePos !== -1) {
          let intEmphasis = getOneEmphasis(tokens, i, closePos);
          if (Array.isArray(intEmphasis) && intEmphasis.length === 0) {
            intEmphasis = '';
          }
          if (currTag === 'link_open') {
            newTokens.push(getUrlToken({ tokens, intEmphasis, num: i }));
          } else {
            newTokens.push(getEmphasisToken({ tokens, intEmphasis, currTag, num: i }));
          }
          i = closePos + 1;
          continue;
        }
      }
      newTokens.push(tokens[i]);
      i++;
    }
  }
  return newTokens;
}
854859
function Tokenizer(text) {
        this._state = TEXT;
        this._buffer = text;
        this._sectionStart = 0;
        this._sectionStartPos = {line: 0, ch: 0};
        this._index = 0;
        this._indexPos = {line: 0, ch: 0};
        this._special = 0; // 1 for script, 2 for style
        this._token = null;
        this._nextToken = null;
    }
1101970
function scanJsxToken() {
  skipComments();
  let beginIndex = index;
  switch (currentCharCode) {
    case 34:  // "
    case 39:  // '
      return scanJsxStringLiteral(beginIndex, currentCharCode);
    case 62:  // >
      next();
      return createToken(CLOSE_ANGLE, beginIndex);
    // case 123:  // {
    // case 125:  // }
  }

  if (!isIdentifierStart(currentCharCode)) {
    return scanToken();
  }
  next();
  while (isIdentifierPart(currentCharCode) || currentCharCode === 45) {  // '-'
    next();
  }
  let value = input.slice(beginIndex, index);
  return new JsxIdentifierToken(getTokenRange(beginIndex), value);
}
1317214
String tokenToString(int token)
    {
        if (Token.printTrees) {
            String name = Token.name(token);

            switch (token) {
            case Token.STRING:
            case Token.REGEXP:
            case Token.NAME:
                return name + " `" + this.string + "'";

            case Token.NUMBER:
                return "NUMBER " + this.number;
            }

            return name;
        }
        return "";
    }
634265
def get_string_index_oid(self, s):
        """"""
        return (len(self.get_bytes(s)), ) + self.get_bytes(s)
512027
function (token, k) {
        var str = "";
        for (var i = 0; i < k && i < token.length; i++)
            str += token[i];
        if (i < token.length) {
            str += "<";
            str += token[i++];
            str += ">";
            for (; i < token.length; i++)
                str += token[i];
        }
        return str;
    }