/* Tokenise an issue.
183498
function issue(eat, value, silent) {
  var self = this
  var index
  var start
  var subvalue
  var href
  var now
  var exit
  var node

  if (value.charCodeAt(0) === CC_HASH) {
    index = 1
  } else if (value.slice(0, PREFIX.length).toLowerCase() === PREFIX) {
    index = PREFIX.length
  } else {
    return
  }

  start = index
  index = issueEnd(value, index)

  if (index === -1) {
    return
  }

  /* istanbul ignore if - maybe used by plug-ins */
  if (silent) {
    return true
  }

  now = eat.now()
  href = gh(self.githubRepo) + 'issues' + C_SLASH + value.slice(start, index)
  subvalue = value.slice(0, index)

  now.column += start

  exit = self.enterLink()

  node = eat(subvalue)({
    type: 'link',
    title: null,
    url: href,
    children: self.tokenizeInline(subvalue, now)
  })

  exit()

  return node
}
1161857
private int assertToken(final StreamTokenizer tokeniser, Reader in, final String token)
            throws IOException, ParserException {
        return assertToken(tokeniser, in, token, false, false);
    }
303180
function (token, errors, message, aggressive) {
    this.addErrorWithAcorn(token, function (ast, tokens) {
      var badPlus = [];

      acornWalk.simple(ast, {
        BinaryExpression: function (node) {
          if (!badPlus.length && node.operator === '+' && (aggressive || isString(node.left) || isString(node.right))) {
            badPlus.push(utils.getNextAcornToken(tokens, node.left.end));
          }
        }
      });

      return badPlus;

      function isString(node) {
        return node.type === 'Literal' && typeof node.value === 'string';
      }
    }, errors, message);
  }
1183009
function standardSuffix (token) {

    token.replaceSuffixInRegion([
      'amentos', 'imentos', 'aço~es', 'adoras', 'adores', 'amento', 'imento',

      'aça~o', 'adora', 'ância', 'antes', 'ismos', 'istas',

      'ador', 'ante', 'ável', 'ezas', 'icas', 'icos', 'ismo', 'ista', 'ível',
      'osas', 'osos',

      'eza', 'ica', 'ico', 'osa', 'oso'

      ], '', 'r2');

    token.replaceSuffixInRegion(['logias', 'logia'], 'log', 'r2');

    // token.replaceSuffixInRegion(['uço~es', 'uça~o'], 'u', 'r1');

    token.replaceSuffixInRegion(['ências', 'ência'], 'ente', 'r2');

    token.replaceSuffixInRegion([
      'ativamente', 'icamente', 'ivamente', 'osamente', 'adamente'
    ], '', 'r2');

    token.replaceSuffixInRegion('amente', '', 'r1');

    token.replaceSuffixInRegion([
      'antemente', 'avelmente', 'ivelmente', 'mente'
    ], '', 'r2');

    token.replaceSuffixInRegion([
      'abilidades', 'abilidade',
      'icidades', 'icidade',
      'ividades', 'ividade',
      'idades', 'idade'
    ], '', 'r2');

    token.replaceSuffixInRegion([
      'ativas', 'ativos', 'ativa', 'ativo',
      'ivas', 'ivos', 'iva', 'ivo'
    ], '', 'r2');

    if (token.hasSuffix('eiras') || token.hasSuffix('eira')) {
      token.replaceSuffixInRegion(['iras', 'ira'], 'ir', 'rv');
    }

    return token;
  }
725291
void tokenize(String pat) throws javax.xml.transform.TransformerException
  {
    tokenize(pat, null);
  }
922590
function validateTokenIndent(token, desiredIndentLevel) {
            const indentation = tokenInfo.getTokenIndent(token);
            const expectedChar = indentType === "space" ? " " : "\t";

            return indentation === expectedChar.repeat(desiredIndentLevel * indentSize) ||

                // To avoid conflicts with no-mixed-spaces-and-tabs, don't report mixed spaces and tabs.
                indentation.includes(" ") && indentation.includes("\t");
        }
18839
def tokenize(self, string):
        """"""
        it = colorise.compat.ifilter(None, self._pattern.finditer(string))

        try:
            t = colorise.compat.next(it)
        except StopIteration:
            yield string, False
            return

        pos, buf, lm, escapeflag = -1, '', -1, False

        # Check if we need to yield any starting text
        if t.start() > 0:
            yield string[:t.start()], False
            pos = t.start()

        it = itertools.chain([t], it)

        for m in it:
            start = m.start()
            e, s = m.group(2) or '', m.group(3)
            escaped = e.count(self._ESCAPE) % 2 != 0

            if escaped:
                buf += string[pos:m.end(2)-1] + s
                escapeflag = True
            else:
                buf += string[pos:m.start(3)]

                if buf:
                    yield buf, escapeflag
                    buf = ''
                    escapeflag = False

                if lm == start:
                    yield '', False

                yield s, False
                lm = m.end()

            pos = m.end()

        if buf:
            yield buf, escapeflag
            escapeflag = False

        if pos < len(string):
            yield string[pos:], False
1112891
function tokenize(sOption) {
		var iAt = 1, // The token's position for error messages; we count starting with 1
			sId,
			aMatches,
			sNext = sOption,
			iOffset,
			oToken,
			aTokens = [],
			sValue;

		while (sNext.length) {
			aMatches = rToken.exec(sNext);
			iOffset = 0;
			if (aMatches) {
				sValue = aMatches[0];
				if (aMatches[7]) {
					sId = "OPTION";
				} else if (aMatches[6] || aMatches[4]) {
					sId = "VALUE";
				} else if (aMatches[5]) {
					sId = "PATH";
					if (sValue === "false" || sValue === "true" || sValue === "null") {
						sId = "VALUE";
					} else if (sValue === "not") {
						sId = "not";
						aMatches = rNot.exec(sNext);
						if (aMatches) {
							sValue = aMatches[0];
						}
					}
				} else if (aMatches[3]) { // a %-escaped delimiter
					sId = unescape(aMatches[3]);
				} else if (aMatches[2]) { // an operator
					sId = aMatches[2];
					iOffset = aMatches[1].length;
				} else { // a delimiter
					sId = aMatches[0];
				}
				if (sId === '"') {
					sId = "VALUE";
					sValue = tokenizeDoubleQuotedString(sNext, sOption, iAt);
				} else if (sId === "'") {
					sId = "VALUE";
					sValue = tokenizeSingleQuotedString(sNext, sOption, iAt);
				}
				oToken = {
					at : iAt + iOffset,
					id : sId,
					value : sValue
				};
			} else {
				throw new SyntaxError("Unknown character '" + sNext[0] + "' at " + iAt + ": "
					+ sOption);
			}
			sNext = sNext.slice(sValue.length);
			iAt += sValue.length;
			aTokens.push(oToken);
		}

		return aTokens;
	}
1156289
function tokener(value, type) {
		session.tokens.push({
			value: value,
			type:  type || value,
			start: null,
			end:   null
		});
	}
902948
function parseEmphasis(tokens) {
  const newTokens = [];
  let i = 0;
  while (i < tokens.length) {
    if (typeof tokens[i] === 'string') {
      if (tokens[i] !== '') {
        newTokens.push(tokens[i]);
      }
      i++;
    } else {
      const currTag = tokens[i].type;
      if (INLINE_OPEN.indexOf(currTag) !== -1) {
        const closeTag = getCloseTag(currTag);
        const closePos = getClosePos(tokens, i, closeTag, tokens[i].markup);
        if (closePos !== -1) {
          let intEmphasis = getOneEmphasis(tokens, i, closePos);
          if (Array.isArray(intEmphasis) && intEmphasis.length === 0) {
            intEmphasis = '';
          }
          if (currTag === 'link_open') {
            newTokens.push(getUrlToken({ tokens, intEmphasis, num: i }));
          } else {
            newTokens.push(getEmphasisToken({ tokens, intEmphasis, currTag, num: i }));
          }
          i = closePos + 1;
          continue;
        }
      }
      newTokens.push(tokens[i]);
      i++;
    }
  }
  return newTokens;
}
105851
protected void lexError(int pos, String key, Object... args) {
        log.error(pos, key, args);
        tk = TokenKind.ERROR;
        errPos = pos;
    }
282341
def accept(self, f, *args):
        """"""
        match = self.match(f, *args)
        if match is None:
            return

        self.tokenizer.skip(len(match.tokens))
        return match
253373
protected function processToken(): void
    {
        $file = $this->getFile();
        $stackPos = $this->getStackPos();
        $nextTokenContentPos = $file->findNext(
            [T_DOC_COMMENT_WHITESPACE],
            $stackPos + 1,
            null,
            true
        );

        $nextToken = $this->tokens[$nextTokenContentPos];

        if (($nextToken['code'] === T_DOC_COMMENT_TAG) && ($this->token['column'] + 2 !== $nextToken['column'])) {
            // This thing is not fixable, because the codesniffer - as you can see - does not parse the whitespace as
            // a token but handles it as "columns"
            $file->addError(
                self::MESSAGE_ERROR_REDUNDANT_WHITESPACE,
                $stackPos,
                static::CODE_ERROR_REDUNDANT_WHITESPACE
            );
        }
    }
363660
function tokenize(str, wordRegexDelimiters, phraseRegexDelimiters) {
    if (wordRegexDelimiters === void 0) { wordRegexDelimiters = ' '; }
    if (phraseRegexDelimiters === void 0) { phraseRegexDelimiters = ''; }
    /* tslint:enable */
    var regexStr = '(?:[' + phraseRegexDelimiters + '])([^' + phraseRegexDelimiters + ']+)(?:[' + phraseRegexDelimiters + '])|([^' + wordRegexDelimiters + ']+)';
    var preTokenized = str.split(new RegExp(regexStr, 'g'));
    var result = [];
    var preTokenizedLength = preTokenized.length;
    var token;
    var replacePhraseDelimiters = new RegExp('[' + phraseRegexDelimiters + ']+', 'g');
    for (var i = 0; i < preTokenizedLength; i += 1) {
        token = preTokenized[i];
        if (token && token.length && token !== wordRegexDelimiters) {
            result.push(token.replace(replacePhraseDelimiters, ''));
        }
    }
    return result;
}
1344251
function fix(text) {
    if (!text) return;
    return text.split('\n').map(function(line) {
      if (line.match(/^- /))
        line = '  ' + line;
      return line;
    }).join('\n');
  }
86031
function Tokenizer(str) {

    this.str = (str || '').toString();
    this.operatorCurrent = '';
    this.operatorExpecting = '';
    this.node = null;
    this.escaped = false;

    this.list = [];

}
517610
private void checkForTokens( String content, int startOffset, int endOffset )
  {
    while( startOffset <= endOffset )
    {
      // skip the delimiters to find the start of a new token
      while( isDelimiter( content.substring( startOffset, startOffset + 1 ) ) )
      {
        if( isError( startOffset ) )
        {
          setCharacterAttributes( startOffset, 1, _error, true );
        }
        else if( isWarning( startOffset ) )
        {
          setCharacterAttributes( startOffset, 1, _warning, true );
        }

        if( startOffset < endOffset )
        {
          startOffset++;
        }
        else
        {
          return;
        }
      }
      // Extract and process the entire token
      if( isQuoteDelimiter( content.substring( startOffset, startOffset + 1 ) ) )
      {
        startOffset = getQuoteToken( content, startOffset, endOffset );
      }
      else if( isCharQuoteDelimiter( content.substring( startOffset, startOffset + 1 ) ) )
      {
        startOffset = getQuoteToken( content, startOffset, endOffset );
      }
      else
      {
        startOffset = getOtherToken( content, startOffset, endOffset );
      }
    }
  }