将一个字符串转换成按标注序列
每列一个字或连续英文token的信息
@param inst 样本
427915
@Override
	public void addThruPipe(Instance inst) {
		String str = (String) inst.getData();
		String[][] data;
		if(isEnFilter){
			data = genSequence(str);			
		}else{
			data = new String[2][str.length()];
			for(int i = 0; i < str.length(); i++){
				data[0][i] = str.substring(i,i+1);
				data[1][i] = Chars.getStringType(data[0][i]).toString();
			}
		}
		inst.setData(data);
	}
1071211
public void visitToken(Token token) {
    for (Trivia trivia : token.getTrivia()) {
      if (!trivia.isComment()) {
        continue;
      }
      Token triviaToken = trivia.getToken();
      String comment = triviaToken.getOriginalValue();
      int line = triviaToken.getLine();
      if (indexOfIgnoreCase(comment) != -1) {
        String[] lines = EOL_PATTERN.split(comment);

        for (int i = 0; i < lines.length; i++) {
          int start = indexOfIgnoreCase(lines[i]);
          if (start != -1 && !isLetterAround(lines[i], start)) {
            check.getContext().createLineViolation(check, message, line + i);
          }
        }
      }
    }
  }
1346470
private void checkForTokens( String content, int startOffset, int endOffset ) {
        while( startOffset <= endOffset ) {
            // skip the delimiters to find the start of a new token
            while( isDelimiter(content.substring(startOffset, startOffset + 1)) ) {
                if (startOffset < endOffset)
                    startOffset++;
                else
                    return;
            }
            // Extract and process the entire token
            if (isQuoteDelimiter(content.substring(startOffset, startOffset + 1)))
                startOffset = getQuoteToken(content, startOffset, endOffset);
            else
                startOffset = getOtherToken(content, startOffset, endOffset);
        }
    }
1091794
private static String readToken(Buffer buffer) {
    try {
      long tokenSize = buffer.indexOfElement(TOKEN_DELIMITERS);
      if (tokenSize == -1L) tokenSize = buffer.size();

      return tokenSize != 0L
          ? buffer.readUtf8(tokenSize)
          : null;
    } catch (EOFException e) {
      throw new AssertionError();
    }
  }
1259291
private static List<String> parseTokens(String exp) {
        if (exp == null) {
            return null;
        }
        List<String> tokens = new ArrayList<String>();
        Character commentChar = null;
        int start = 0;
        for (int i = 0; i < exp.length(); i++) {
            if (exp.charAt(i) == '"') {
                if (commentChar == null) {
                    commentChar = '"';
                } else if (commentChar == '"') {
                    commentChar = null;
                    tokens.add(exp.substring(start, i + 1).trim());
                    start = i + 1;
                }
            } else if (exp.charAt(i) == '\'') {
                if (commentChar == null) {
                    commentChar = '\'';
                } else if (commentChar == '\'') {
                    commentChar = null;
                    tokens.add(exp.substring(start, i + 1).trim());
                    start = i + 1;
                }
            } else if (exp.charAt(i) == '[') {
                if (commentChar == null) {
                    if (start != i) {
                        tokens.add(exp.substring(start, i).trim());
                    }
                    tokens.add("[");
                    start = i + 1;
                }
            } else if (exp.charAt(i) == ']') {
                if (commentChar == null) {
                    if (start != i) {
                        tokens.add(exp.substring(start, i).trim());
                    }
                    tokens.add("]");
                    start = i + 1;
                }
            } else if (exp.charAt(i) == '.') {
                if (commentChar == null) {
                    if (start != i) {
                        tokens.add(exp.substring(start, i).trim());
                    }
                    start = i + 1;
                }
            } else if (i == exp.length() - 1) {
                tokens.add(exp.substring(start, i + 1).trim());
            }
        }
        return tokens;
    }
423553
public Token scanIndent() {
		int start = pos;
		while (pos < input.length()
				&& (input.charAt(pos) == ' ' || input.charAt(pos) == '\t')) {
			pos++;
		}
		return new Token(Token.Kind.Indent, input.substring(start, pos), start);
	}
1290863
protected void doSplitTokenImpl(Token token, ITokenAcceptor result) {
		String text = token.getText();
		int indentation = computeIndentation(text);
		if (indentation == -1 || indentation == currentIndentation) {
			// no change of indentation level detected simply process the token
			result.accept(token);
		} else if (indentation > currentIndentation) {
			// indentation level increased
			splitIntoBeginToken(token, indentation, result);
		} else if (indentation < currentIndentation) {
			// indentation level decreased
			int charCount = computeIndentationRelevantCharCount(text);
			if (charCount > 0) {
				// emit whitespace including newline
				splitWithText(token, text.substring(0, charCount), result);	
			}
			// emit end tokens at the beginning of the line
			decreaseIndentation(indentation, result);
			if (charCount != text.length()) {
				handleRemainingText(token, text.substring(charCount), indentation, result);
			}
		} else {
			throw new IllegalStateException(String.valueOf(indentation));
		}
	}
1152877
@Override
    public void trainOnInstanceImpl(Instance inst) {
        if (this.reset == true) {
            this.m_numClasses = inst.numClasses();
            double laplace = this.laplaceCorrectionOption.getValue();
            int numAttributes = inst.numAttributes();

            m_probOfClass = new double[m_numClasses];
            Arrays.fill(m_probOfClass, laplace);

            m_classTotals = new double[m_numClasses];
            Arrays.fill(m_classTotals, laplace * numAttributes);

            m_wordTotalForClass = new DoubleVector[m_numClasses];
            for (int i = 0; i< m_numClasses; i++) {
                //Arrays.fill(wordTotal, laplace);
                m_wordTotalForClass[i] = new DoubleVector();
            }
            this.reset = false;
        }
        // Update classifier
        int classIndex = inst.classIndex();
        int classValue = (int) inst.classValue();//value(classIndex);

        double w = inst.weight();
        m_probOfClass[classValue] += w;

        m_classTotals[classValue] += w * totalSize(inst);
        double total = m_classTotals[classValue];

        for (int i = 0; i < inst.numValues(); i++) {
            int index = inst.index(i);
            if (index != classIndex && !inst.isMissing(i)) {
                //m_wordTotalForClass[index][classValue] += w * inst.valueSparse(i);
                double laplaceCorrection = 0.0;
                if (m_wordTotalForClass[classValue].getValue(index)== 0) {
                    laplaceCorrection = this.laplaceCorrectionOption.getValue();
                }
                m_wordTotalForClass[classValue].addToValue(index, w * inst.valueSparse(i) + laplaceCorrection);
            }
        }
    }
427248
public static void assertToken(int expectedType, String expectedText, LexerResults lexerResults)
    {
        assertToken(expectedType, expectedText, lexerResults.getToken());
    }
254689
boolean setToken(char token[], byte tokenstring[])
    {
        if (token != null && tokenstring != null && token.length > 0 &&
            tokenstring.length > 0) {
            m_tokentable_ = token;
            m_tokenstring_ = tokenstring;
            return true;
        }
        return false;
    }
86588
private boolean tokenTextsDoNotMatch(Token token1, Token token2) {
        return !token1.getText().equals(token2.getText());
    }
536232
public Token[] tokenize(String input) 
    {
        Token[] initialTokens = innerTokenizer.tokenize(input);
        List tokens = new ArrayList();
        for (int i=0; i<initialTokens.length; i++) {
            Token tok = initialTokens[i];
            String str = "^"+tok.getValue()+"$";
            if (keepOldTokens) tokens.add( intern(str) );
            for (int lo=0; lo<str.length(); lo++) {
                for (int len=minNGramSize; len<=maxNGramSize; len++) {
                    if (lo+len<str.length()) {
                        tokens.add( innerTokenizer.intern( str.substring(lo,lo+len) )); 
                    }
                }
            }
        }
        return (Token[]) tokens.toArray(new BasicToken[tokens.size()]);
    }
657796
public static function initTokenKindToText() {
        self::$tokenKindToText = \array_flip(\array_merge(
            TokenStringMaps::OPERATORS_AND_PUNCTUATORS,
            TokenStringMaps::KEYWORDS,
            TokenStringMaps::RESERVED_WORDS
        ));
    }
1313520
private MarkToken getToken(final String in, final int pos)
    {
        final char c0 = pos > 0 ? whitespaceToSpace(in.charAt(pos - 1)) : ' ';
        final char c = whitespaceToSpace(in.charAt(pos));
        final char c1 = pos + 1 < in.length() ? whitespaceToSpace(in.charAt(pos + 1)) : ' ';
        final char c2 = pos + 2 < in.length() ? whitespaceToSpace(in.charAt(pos + 2)) : ' ';
        final char c3 = pos + 3 < in.length() ? whitespaceToSpace(in.charAt(pos + 3)) : ' ';

        switch (c)
        {
        case '*':
            if (c1 == '*')
            {
                return c0 != ' ' || c2 != ' ' ? MarkToken.STRONG_STAR : MarkToken.EM_STAR;
            }
            return c0 != ' ' || c1 != ' ' ? MarkToken.EM_STAR : MarkToken.NONE;
        case '_':
            if (c1 == '_')
            {
                return c0 != ' ' || c2 != ' ' ? MarkToken.STRONG_UNDERSCORE : MarkToken.EM_UNDERSCORE;
            }
            if (this.useExtensions)
            {
                return Character.isLetterOrDigit(c0) && c0 != '_' && Character.isLetterOrDigit(c1) ? MarkToken.NONE
                        : MarkToken.EM_UNDERSCORE;
            }
            return c0 != ' ' || c1 != ' ' ? MarkToken.EM_UNDERSCORE : MarkToken.NONE;
        case '!':
            if (c1 == '[')
            {
                return MarkToken.IMAGE;
            }
            return MarkToken.NONE;
        case '[':
            if (this.useExtensions && c1 == '[')
            {
                return MarkToken.X_LINK_OPEN;
            }
            return MarkToken.LINK;
        case ']':
            if (this.useExtensions && c1 == ']')
            {
                return MarkToken.X_LINK_CLOSE;
            }
            return MarkToken.NONE;
        case '`':
            return c1 == '`' ? MarkToken.CODE_DOUBLE : MarkToken.CODE_SINGLE;
        case '\\':
            switch (c1)
            {
            case '\\':
            case '[':
            case ']':
            case '(':
            case ')':
            case '{':
            case '}':
            case '#':
            case '"':
            case '\'':
            case '.':
            case '>':
            case '<':
            case '*':
            case '+':
            case '-':
            case '_':
            case '!':
            case '`':
            case '~':
            case '^':
                return MarkToken.ESCAPE;
            default:
                return MarkToken.NONE;
            }
        case '<':
            if (this.useExtensions && c1 == '<')
            {
                return MarkToken.X_LAQUO;
            }
            return MarkToken.HTML;
        case '&':
            return MarkToken.ENTITY;
        default:
            if (this.useExtensions)
            {
                switch (c)
                {
                case '-':
                    if (c1 == '-')
                    {
                        return c2 == '-' ? MarkToken.X_MDASH : MarkToken.X_NDASH;
                    }
                    break;
                case '^':
                    return c0 == '^' || c1 == '^' ? MarkToken.NONE : MarkToken.SUPER;
                case '>':
                    if (c1 == '>')
                    {
                        return MarkToken.X_RAQUO;
                    }
                    break;
                case '.':
                    if (c1 == '.' && c2 == '.')
                    {
                        return MarkToken.X_HELLIP;
                    }
                    break;
                case '(':
                    if (c1 == 'C' && c2 == ')')
                    {
                        return MarkToken.X_COPY;
                    }
                    if (c1 == 'R' && c2 == ')')
                    {
                        return MarkToken.X_REG;
                    }
                    if (c1 == 'T' & c2 == 'M' & c3 == ')')
                    {
                        return MarkToken.X_TRADE;
                    }
                    break;
                case '"':
                    if (!Character.isLetterOrDigit(c0) && c1 != ' ')
                    {
                        return MarkToken.X_LDQUO;
                    }
                    if (c0 != ' ' && !Character.isLetterOrDigit(c1))
                    {
                        return MarkToken.X_RDQUO;
                    }
                    break;
                }
            }
            return MarkToken.NONE;
        }
    }
71079
static void parse(String inString, IContext context, INode parent) {
        List<String> tokens = getCommaTokens(inString);
        for (String token : tokens) {
            int isFunction = token.indexOf(OPEN_PARENTHESIS);
            if (isFunction >= 0) {
                String funcName = token.substring(0, isFunction);
                INode newParent = addChild(parent, context, funcName);
                String arguments = token.substring(isFunction + 1, token.length() - 1);
                parse(arguments, context, newParent);
            } else {
                addChild(parent, context, token);
            }
        }
    }
1366255
@Override
    public boolean incrementToken() throws IOException {
        if (!input.incrementToken()) {
            return false;
        }

        final String t = myTermAttribute.toString();

        if (t != null && t.length() != 0) {
            try {
                myTermAttribute.setEmpty().append(ISO639Converter.convert(t));
            } catch (final IllegalArgumentException details) {
                if (LOGGER.isWarnEnabled()) {
                    LOGGER.warn(details.getMessage(), details);
                }
            }
        }

        return true;
    }
1132189
private void initTokenizer(StreamTokenizer tokenizer) {
        tokenizer.resetSyntax();
        tokenizer.whitespaceChars(0, ' ');
        tokenizer.wordChars(' ' + 1, '\u00FF');
        tokenizer.whitespaceChars(',', ',');
        tokenizer.commentChar('%');
        tokenizer.quoteChar('"');
        tokenizer.quoteChar('\'');
        tokenizer.ordinaryChar('{');
        tokenizer.ordinaryChar('}');
        tokenizer.eolIsSignificant(true);
    }