取回一批分词结果(需要多次调用, 直到返回 false)
974451
function get_result() {
        $off = $this->_off;
        $len = $this->_len;
        $txt = $this->_txt;
        $this->_res = array();
        while (($off < $len) && (ord($txt[$off]) <= 0x20)) {
            if ($txt[$off] == "\r" || $txt[$off] == "\n") {
                $this->_off = $off + 1;
                $this->_put_res($off, 0, 1, 'un');
                return $this->_res;
            }
            $off++;
        }
        if ($off >= $len) return false;
        // try to parse the sentence
        $this->_off = $off;
        $ch = $txt[$off];
        $cx = ord($ch);
        if ($this->_char_token($ch)) {
            $this->_off++;
            $this->_put_res($off, 0, 1, 'un');
            return $this->_res;
        }
        $clen = $this->_ztab[$cx];
        $zlen = 1;
        $pflag = ($clen > 1 ? PSCWS4_PFLAG_WITH_MB : ($this->_is_alnum($cx) ? PSCWS4_PFLAG_ALNUM : 0));
        while (($off = ($off + $clen)) < $len) {
            $ch = $txt[$off];
            $cx = ord($ch);
            if ($cx <= 0x20 || $this->_char_token($ch)) break;
            $clen = $this->_ztab[$cx];
            if (!($pflag & PSCWS4_PFLAG_WITH_MB)) {
                // pure single-byte -> multibyte (2bytes)
                if ($clen == 1) {
                    if (($pflag & PSCWS4_PFLAG_ALNUM) && !$this->_is_alnum($cx))
                        $pflag ^= PSCWS4_PFLAG_ALNUM;
                } else {
                    if (!($pflag & PSCWS4_PFLAG_ALNUM) || $zlen > 2) break;
                    $pflag |= PSCWS4_PFLAG_WITH_MB;
                }
            } else if (($pflag & PSCWS4_PFLAG_WITH_MB) && $clen == 1) {
                // mb + single-byte. allowd: alpha+num + 中文
                if (!$this->_is_alnum($cx)) break;
                $pflag &= ~PSCWS4_PFLAG_VALID;
                for ($i = $off + 1; $i < ($off + 3); $i++) {
                    $ch = $txt[$i];
                    $cx = ord($ch);
                    if (($i >= $len) || ($cx <= 0x20) || ($this->_ztab[$cx] > 1)) {
                        $pflag |= PSCWS4_PFLAG_VALID;
                        break;
                    }
                    if (!$this->_is_alnum($cx)) break;
                }
                if (!($pflag & PSCWS4_PFLAG_VALID)) break;
                $clen += ($i - $off - 1);
            }
            // hightman.070813: add max zlen limit
            if (++$zlen >= PSCWS4_MAX_ZLEN) break;
        }
        // hightman.070624: 处理半个字的问题
        if (($ch = $off) > $len)
            $off -= $clen;
        // do the real segment
        if ($off <= $this->_off) return false;
        else if ($pflag & PSCWS4_PFLAG_WITH_MB) $this->_msegment($off, $zlen);
        else if (!($pflag & PSCWS4_PFLAG_ALNUM) || (($off - $this->_off) >= PSCWS4_MAX_EWLEN)) $this->_ssegment($off);
        else {
            $zlen = $off - $this->_off;
            $this->_put_res($this->_off, 2.5 * log($zlen), $zlen, 'en');
        }
        // reutrn the result
        $this->_off = ($ch > $len ? $len : $off);
        if (count($this->_res) == 0)
            return $this->get_result();
        return $this->_res;
    }
391508
function (token) {
            if (isWhiteSpace(token)) {
                // 默认都要删除空白
                var remove = true;

                // 空白前面是换行 && 后面是注释的不删除
                if (token.prev && token.next && isLineBreak(token.prev) && isComment(token.next)) {
                    remove = false;
                }

                if (remove) {
                    removeToken(token);
                }
            }

            if (isLineComment(token)) {
                token.raw = token.raw.trimRight();
            }

            if (isBlockComment(token)) {
                var rawArray = token.raw.split(_config.lineSeparator);
                rawArray.forEach(function (line, i) {
                    rawArray[i] = line.trimRight();
                });
                token.raw = rawArray.join(_config.lineSeparator);
            }
        }
225081
function moneySmalltoBig(n) {
    if (!_.isNumber(n) || _.isNaN(n))
        return ''

    var fraction = ['角', '分'];
    var digit = ['零', '壹', '贰', '叁', '肆', '伍', '陆', '柒', '捌', '玖'];
    var unit = [['元', '万', '亿'], ['', '拾', '佰', '仟']];
    var head = n < 0 ? '欠' : '';
    n = Math.abs(n);

    var s = '';

    for (var i = 0; i < fraction.length; i++) {
        s += (digit[Math.floor(n * 10 * Math.pow(10, i)) % 10] + fraction[i]).replace(/零./, '');
    }
    s = s || '整';
    n = Math.floor(n);

    for (var i = 0; i < unit[0].length && n > 0; i++) {
        var p = '';
        for (var j = 0; j < unit[1].length && n > 0; j++) {
            p = digit[n % 10] + unit[1][j] + p;
            n = Math.floor(n / 10);
        }
        s = p.replace(/(零.)*零$/, '').replace(/^$/, '零') + unit[0][i] + s;
    }
    return head + s.replace(/(零.)*零元/, '元').replace(/(零.)+/g, '零').replace(/^整$/, '零元整');
}
864261
static boolean loadDat(String path)
    {
        try
        {
            ByteArray byteArray = ByteArray.createByteArray(path + Predefine.BIN_EXT);
            if (byteArray == null) return false;
            int size = byteArray.nextInt();
            CoreDictionary.Attribute[] attributes = new CoreDictionary.Attribute[size];
            final Nature[] natureIndexArray = Nature.values();
            for (int i = 0; i < size; ++i)
            {
                // 第一个是全部频次，第二个是词性个数
                int currentTotalFrequency = byteArray.nextInt();
                int length = byteArray.nextInt();
                attributes[i] = new CoreDictionary.Attribute(length);
                attributes[i].totalFrequency = currentTotalFrequency;
                for (int j = 0; j < length; ++j)
                {
                    attributes[i].nature[j] = natureIndexArray[byteArray.nextInt()];
                    attributes[i].frequency[j] = byteArray.nextInt();
                }
            }
            if (!trie.load(byteArray, attributes) || byteArray.hasMore()) return false;
        }
        catch (Exception e)
        {
            logger.warning("读取失败，问题发生在" + e);
            return false;
        }
        return true;
    }
1044546
private static void splitMiddleSlashFromDigitalWords(List<Vertex> linkedArray)
    {
        if (linkedArray.size() < 2)
            return;

        ListIterator<Vertex> listIterator = linkedArray.listIterator();
        Vertex next = listIterator.next();
        Vertex current = next;
        while (listIterator.hasNext())
        {
            next = listIterator.next();
//            System.out.println("current:" + current + " next:" + next);
            Nature currentNature = current.getNature();
            if (currentNature == Nature.nx && (next.hasNature(Nature.q) || next.hasNature(Nature.n)))
            {
                String[] param = current.realWord.split("-", 1);
                if (param.length == 2)
                {
                    if (TextUtility.isAllNum(param[0]) && TextUtility.isAllNum(param[1]))
                    {
                        current = current.copy();
                        current.realWord = param[0];
                        current.confirmNature(Nature.m);
                        listIterator.previous();
                        listIterator.previous();
                        listIterator.set(current);
                        listIterator.next();
                        listIterator.add(Vertex.newPunctuationInstance("-"));
                        listIterator.add(Vertex.newNumberInstance(param[1]));
                    }
                }
            }
            current = next;
        }

//        logger.trace("杠号识别后：" + Graph.parseResult(linkedArray));
    }
389834
private List<Word>[] toWords(List<Node> leaf){
        List<Word>[] result = new ArrayList[leaf.size()];
        int i = 0;
        if(LOGGER.isDebugEnabled()) {
            LOGGER.debug("全切分结果：");
        }
        for(Node node : leaf){
            result[i++] = toWords(node);
            if(LOGGER.isDebugEnabled()) {
                LOGGER.debug("\t" + i + "：" + result[i - 1]);
            }
        }
        return result;
    }
973578
private boolean isPart(PartOfSpeech pos, String word){
		if(NP.contains(pos))
			return true;
		
		if(pos == PartOfSpeech.结构助词 && word.equals("的"))
			return true;
		
		return false;
	}
1027888
function isOTag (str, oTag, index) {
    var l = oTag.length, s = str.substr(index, l), sign = str.charAt(index+l);
    // NOTE 要保证sign有值 如果当前的index已经是字符串尾部 如'foo{{' sign为空
    if (oTag === s && sign && SIGN.indexOf(sign) > -1) {
        return {
            str: s + sign,
            index: index,
            // sign: signType[str.charAt(index+l)], // TODO: delete
            sign: str.charAt(index+l),
            // ignore the last oTag charts and the sign char,
            // l(oTag.length) - 1(oTag's first char) + 1(sign's char)
            jump: l
        };
    }
}
1026815
public static int[] split(int amount, int count, int min, int max) {
        Assert.isPositive(amount, "amount必须大于0");
        Assert.isPositive(count, "count必须大于0");
        Assert.isPositive(min, "min必须大于0");
        Assert.isPositive(max, "max必须大于0");

        if (amount < count * min) {
            throw new IllegalArgumentException("参数错误，min * count 不能大于amount");
        }

        if (amount > max * count) {
            throw new IllegalArgumentException("参数错误，max * count 不能小于amount");
        }

        if (min > max) {
            throw new IllegalArgumentException("参数错误，min不能大于max");
        }

        SecureRandom secureRandom = new SecureRandom();
        int nowAmount = amount;

        int[] array = new int[count];

        for (int i = 0; i < count; i++) {
            if (i == (count - 1)) {
                array[i] = nowAmount;
            } else {
                int random = Math.abs(secureRandom.nextInt());
                int nowMax = nowAmount - ((count - i - 1) * min);
                int nowMin = nowAmount - ((count - i - 1) * max);
                //当前允许最小值
                nowMax = Math.min(nowMax, max);
                //当前允许最大值
                nowMin = Math.max(nowMin, min);
                //区间
                int mod = nowMax - nowMin;
                //当前值
                int packet;
                if (mod == 0) {
                    // 此时说明后续的红包都是最小的了
                    packet = nowMin;
                } else {
                    packet = random % mod + nowMin;
                }

                array[i] = packet;
                nowAmount -= packet;
            }
        }
        return array;
    }
595903
private static List<Integer> parseStep(String value, ValueParser parser) {
		final List<String> parts = StrUtil.split(value, StrUtil.C_SLASH);
		int size = parts.size();
		
		List<Integer> results;
		if (size == 1) {// 普通形式
			results = parseRange(value, -1, parser);
		} else if (size == 2) {// 间隔形式
			final int step = parser.parse(parts.get(1));
			if (step < 1) {
				throw new CronException("Non positive divisor for field: [{}]", value);
			}
			results = parseRange(parts.get(0), step, parser);
		} else {
			throw new CronException("Invalid syntax of field: [{}]", value);
		}
		return results;
	}
370874
public static void main(String... args) throws Exception {

        String string = "SELECT :expr1, #($expr2.class),"
                + " WHERE #if(:expr3) {e = $expr3} #else {e IS NULL}"
                + "#for(variant in $expr4.bytes) { AND c = :variant}" // NL
                + " {AND d = :expr-5}? {AND f = $expr6}?" // NL
                + " BY #!(:expr7) ASC";

        // 在输入中查找  PREFIX 字符
        Matcher matcher = PATTERN_KEYWORD.matcher(string);

        int position = 0;

        while (matcher.find(position)) {

            System.out.println("===============================");
            System.out.println("group 0: " + matcher.group(0));
            System.out.println("group 1: " + matcher.group(1));
            System.out.println("group 2: " + matcher.group(2));
            System.out.println("group 3: " + matcher.group(3));

            position = matcher.end();
        }

        // 在循环表达式中查找
        matcher = PATTERN_IN.matcher("variant in :expr5");

        if (matcher.matches()) {

            System.out.println("===============================");
            System.out.println("group 0: " + matcher.group(0));
            System.out.println("group 1: " + matcher.group(1));
            System.out.println("group 2: " + matcher.group(2));
        }

        // 编译下列语句
        ExqlPattern pattern = new ExqlCompiler(string).compile();

        ExqlContext context = new ExqlContextImpl();

        HashMap<String, Object> map = new HashMap<String, Object>();

        map.put("expr1", "expr1");
        map.put("expr2", "expr2");
        map.put("expr3", "expr3");
        map.put("expr4", "expr4");
        map.put("expr5", "expr5");
        // map.put("expr6", "expr6");
        map.put("expr7", "expr7");

        pattern.execute(context, map, map);
        System.out.println(context.flushOut());
        System.out.println(Arrays.toString(context.getArgs()));
    }
923930
@Override
    protected double scoreImpl(List<Word> words1, List<Word> words2) {
        //用词频来标注词的权重
        taggingWeightWithWordFrequency(words1, words2);
        //构造权重快速搜索容器
        Map<String, Float> weights1 = toFastSearchMap(words1);
        Map<String, Float> weights2 = toFastSearchMap(words2);
        //所有的不重复词
        Set<Word> words = new HashSet<>();
        words.addAll(words1);
        words.addAll(words2);
        //向量的维度为words的大小，每一个维度的权重是词频
        //a.b
        AtomicFloat ab = new AtomicFloat();
        //|a|的平方
        AtomicFloat aa = new AtomicFloat();
        //|b|的平方
        AtomicFloat bb = new AtomicFloat();
        //计算
        words
            .parallelStream()
            .forEach(word -> {
                Float x1 = weights1.get(word.getText());
                Float x2 = weights2.get(word.getText());
                if (x1 != null && x2 != null) {
                    //x1x2
                    float oneOfTheDimension = x1 * x2;
                    //+
                    ab.addAndGet(oneOfTheDimension);
                }
                if (x1 != null) {
                    //(x1)^2
                    float oneOfTheDimension = x1 * x1;
                    //+
                    aa.addAndGet(oneOfTheDimension);
                }
                if (x2 != null) {
                    //(x2)^2
                    float oneOfTheDimension = x2 * x2;
                    //+
                    bb.addAndGet(oneOfTheDimension);
                }
            });
        //|a|
        double aaa = Math.sqrt(aa.doubleValue());
        //|b|
        double bbb = Math.sqrt(bb.doubleValue());
        //使用BigDecimal保证精确计算浮点数
        //|a|*|b|
        //double aabb = aaa * bbb;
        BigDecimal aabb = BigDecimal.valueOf(aaa).multiply(BigDecimal.valueOf(bbb));
        //similarity=a.b/|a|*|b|
        //double cos = ab.get() / aabb.doubleValue();
        double cos = BigDecimal.valueOf(ab.get()).divide(aabb, 9, BigDecimal.ROUND_HALF_UP).doubleValue();
        return cos;
    }
601485
function countingPoint(withPointList, withoutPointList) {
        if (withPointList.length === 0 || withoutPointList.length === 0) {
            return;
        }
        if (withPointList.length > withoutPointList.length) {
            // 。ありに統一
            reportPointResult(withoutPointList, {
                shouldUsePoint: true
            });
        } else if (withPointList.length < withoutPointList.length) {
            // 。なしに統一
            reportPointResult(withPointList, {
                shouldUsePoint: false
            });
        } else {
            // 。ありに統一
            reportPointResult(withoutPointList, {
                shouldUsePoint: true
            });
        }
    }
381551
function() {
		var hostsstr = fs.readFileSync(HOSTS, 'utf-8');
		//一个domain 对应一个ip
		//先分组，再去重
		//之后去重
		var lines = hostsstr.split(EOL);
		var hostsobject = {};
		var currentName;
		var currentLog = [];
		var uniq = {};
		notes = {};
		for (var i = 0; i < lines.length; i++) {
			var line = this._parseLine(lines[i]);
			if (!line) continue;
			var type = line.type;
			var val = line.value;
			switch (type) {
			case 1:
				if (!currentName) {
					currentName = this.defaultName;
					hostsobject[currentName] = [];
					if (currentLog.length > 0) {
						notes[currentName] = currentLog.join(EOL);
						currentLog = [];
					}
				}
				val.forEach(function(item) {
					var _item = uniq[currentName + item.domain + item.ip];
					if (!_item) {
						hostsobject[currentName].push(item);
						_item = uniq[currentName + item.domain + item.ip] = item;
					}
					if (currentLog.length > 0) {
						_item.note = (_item.note ? (_item.note + EOL) : '') + currentLog.join(EOL);
					}

				});
				currentLog = [];
				break;
			case 2:
				currentLog.push(val);
				break;
			case 3:
				currentName = val;
				hostsobject[currentName] = [];
				if (currentLog.length > 0) {
					notes[currentName] = currentLog.join(EOL);
					currentLog = [];
				}
				break;
			default:
				break;
			}
		}
		if (currentLog.length > 0) {
			notes.bottom = currentLog.join(EOL);
			currentLog = [];
		}
		return hostsobject;
	}
958586
function (matrix1, matrix2) {
            matrix1 = matrix1 || [1, 0, 0, 1];
            matrix2 = matrix2 || [1, 0, 0, 1];
            // 旋转变换 4 个参数
            if (matrix1.length === 4) {
                return [
                    matrix1[0] * matrix2[0] + matrix1[2] * matrix2[1],
                    matrix1[1] * matrix2[0] + matrix1[3] * matrix2[1],
                    matrix1[0] * matrix2[2] + matrix1[2] * matrix2[3],
                    matrix1[1] * matrix2[2] + matrix1[3] * matrix2[3]
                ];
            }
            // 旋转位移变换, 6 个参数

            return [
                matrix1[0] * matrix2[0] + matrix1[2] * matrix2[1],
                matrix1[1] * matrix2[0] + matrix1[3] * matrix2[1],
                matrix1[0] * matrix2[2] + matrix1[2] * matrix2[3],
                matrix1[1] * matrix2[2] + matrix1[3] * matrix2[3],

                matrix1[0] * matrix2[4] + matrix1[2] * matrix2[5] + matrix1[4],
                matrix1[1] * matrix2[4] + matrix1[3] * matrix2[5] + matrix1[5]
            ];
        }
1014923
static function result($s, $quanpin = true, $daxie = false, $trim = false)
      {
            $pyCount = 0;
            if(is_array($quanpin) && $quanpin[1]) {
                  $pyCount = $quanpin[1];
                  unset($quanpin[1]);
                  $quanpin = $quanpin[0];
            }
            $s = preg_replace("/\s/is", "_", $s);
            $s = preg_replace("/(|\~|\`|\!|\@|\#|\$|\%|\^|\&|\*|\(|\)|\-|\+|\=|\{|\}|\[|\]|\||\\|\:|\;|\"|\'|\<|\,|\>|\.|\?|\/)/is", "", $s);
            $i = 0;
            $py = '';
            $_trim = $trim ? ' ' : '';
            // 加入这一句，自动识别UTF-8
            if (strlen("拼音") > 4)
            $s = iconv('UTF-8', 'GBK', $s);
            if ($quanpin) {
                  // 全拼
                  for ($i = 0; $i < strlen($s); $i++) {
                      if (ord($s[$i]) > 128) {
                          $char = self::asi2py(ord($s[$i]) + ord($s[$i + 1]) * 256);
                          $py.=$char.$_trim;
                          $i++;
                      } else {
                          $py.=$s[$i].$_trim;
                      }
                  }
            } else {
                  // 首字母
                  for ($i = 0; $i < strlen($s); $i++) {
                        if (ord($s[$i]) > 128) {
                              $char = self::asi2py(ord($s[$i]) + ord($s[$i + 1]) * 256);
                              $py	.=$char[0].$_trim;
                              $i++;
                        } else {
                              $py	.=$s[$i].$_trim;
                        }
                  }
            }
            $py = $trim ? trim($py, ' ') : $py;
            if($pyCount){
                  $py = substr($py, 0, $pyCount);
            }
            return ($daxie == true ? $py : strtolower($py));// 判断是否输出小写字符
      }
906610
public void SFFS() throws Exception {
		int NumTemplate = Templates.length;
		CurSet = new boolean[NumTemplate];
		LeftSet = new boolean[NumTemplate];
		for (int i = 0; i < NumTemplate; i++) {
			CurSet[i] = false;
			LeftSet[i] = true;
		}
		Evaluate(Templates, CurSet);
		boolean continueDelete = false;
		while (SetSize(CurSet) <= NumTemplate) {
			if (!continueDelete) {
				Integer theInt = BooleanSet2Integer(CurSet);
				int best = BestFeatureByAdding(CurSet, LeftSet);
				if (best != -1) {
					System.out.println("Add Best " + best);
					AlreadyAdded.add(theInt);
					CurSet[best] = true;
					LeftSet[best] = false;
					printAccuracy(CurSet);
				}
			}
			Integer theInt = BooleanSet2Integer(CurSet);
			// 被删除过的集合就不再删除了
			if (AlreadyRemoved.contains(theInt)) {
				continueDelete = false;
				continue;
			} else {
				int worst = WorstFeatureByRemoving(CurSet);
				boolean[] TestSet = CurSet.clone();
				TestSet[worst] = false;
				double accuracy = Evaluate(Templates, TestSet);
				if (accuracy >= AccuracyRecords.get(theInt)) {
					System.out.println("Remove Worst " + worst);
					AlreadyRemoved.add(theInt);
					CurSet[worst] = false;
					LeftSet[worst] = true;
					printAccuracy(CurSet);
					if (SetSize(CurSet) == 0) {
						continueDelete = false;
					} else
						continueDelete = true;
				} else {
					continueDelete = false;
					continue;
				}
			}
		}
		printAccuracyRecords(AccuracyRecords);
		ObjectOutputStream oos = new ObjectOutputStream(new FileOutputStream(
				"/home/zliu/corpus/testTemplate/SFFS.ser"));
		oos.writeObject(AccuracyRecords);
		oos.close();

	}