分词，先将原始句子split为一个数组，之后遍历每一行，调用对单行分词的函数（有两种）。
        text=True会返回分词好的字符串，为False则会返回一个二位数组方便用户做后续处理。
        函数中有一些细节处理主要是用于规范输出格式
335715
def __cutWithOutMethod(self, oiraw, cut_method, text = True):
        ''''''
        oiraw = oiraw.split('\n')
        txt = ""
        array = []
        if(text):
            for line in oiraw:
                if(self.__seg_only):
                    temp_txt = reduce(lambda x, y: x + ' ' + y if y != " " else x, cut_method(line), '') + '\n'
                else:
                    temp_txt = reduce(lambda x, y: x + ' ' + "".join(y), cut_method(line), '') + '\n'
                txt += temp_txt[1:]
            return txt[:-1]
        else:
            for line in oiraw:
                if(line):
                    if(self.__seg_only):
                        array += (reduce(lambda x, y: x + [[y, '']], cut_method(line), []))
                    else:
                        array += (reduce(lambda x, y: x + [[y[0], y[2]]], cut_method(line), []))
                array += [['\n', '']]
            return array[:-1]
1283351
function splitText(text, options) {
  const KIND_NON_CJK = "non-cjk";
  const KIND_CJ_LETTER = "cj-letter";
  const KIND_K_LETTER = "k-letter";
  const KIND_CJK_PUNCTUATION = "cjk-punctuation";

  const nodes = [];

  (options.proseWrap === "preserve"
    ? text
    : text.replace(new RegExp(`(${cjkPattern})\n(${cjkPattern})`, "g"), "$1$2")
  )
    .split(/([ \t\n]+)/)
    .forEach((token, index, tokens) => {
      // whitespace
      if (index % 2 === 1) {
        nodes.push({
          type: "whitespace",
          value: /\n/.test(token) ? "\n" : " "
        });
        return;
      }

      // word separated by whitespace

      if ((index === 0 || index === tokens.length - 1) && token === "") {
        return;
      }

      token
        .split(new RegExp(`(${cjkPattern})`))
        .forEach((innerToken, innerIndex, innerTokens) => {
          if (
            (innerIndex === 0 || innerIndex === innerTokens.length - 1) &&
            innerToken === ""
          ) {
            return;
          }

          // non-CJK word
          if (innerIndex % 2 === 0) {
            if (innerToken !== "") {
              appendNode({
                type: "word",
                value: innerToken,
                kind: KIND_NON_CJK,
                hasLeadingPunctuation: punctuationRegex.test(innerToken[0]),
                hasTrailingPunctuation: punctuationRegex.test(
                  getLast(innerToken)
                )
              });
            }
            return;
          }

          // CJK character
          appendNode(
            punctuationRegex.test(innerToken)
              ? {
                  type: "word",
                  value: innerToken,
                  kind: KIND_CJK_PUNCTUATION,
                  hasLeadingPunctuation: true,
                  hasTrailingPunctuation: true
                }
              : {
                  type: "word",
                  value: innerToken,
                  kind: kRegex.test(innerToken)
                    ? KIND_K_LETTER
                    : KIND_CJ_LETTER,
                  hasLeadingPunctuation: false,
                  hasTrailingPunctuation: false
                }
          );
        });
    });

  return nodes;

  function appendNode(node) {
    const lastNode = getLast(nodes);
    if (lastNode && lastNode.type === "word") {
      if (
        (lastNode.kind === KIND_NON_CJK &&
          node.kind === KIND_CJ_LETTER &&
          !lastNode.hasTrailingPunctuation) ||
        (lastNode.kind === KIND_CJ_LETTER &&
          node.kind === KIND_NON_CJK &&
          !node.hasLeadingPunctuation)
      ) {
        nodes.push({ type: "whitespace", value: " " });
      } else if (
        !isBetween(KIND_NON_CJK, KIND_CJK_PUNCTUATION) &&
        // disallow leading/trailing full-width whitespace
        ![lastNode.value, node.value].some(value => /\u3000/.test(value))
      ) {
        nodes.push({ type: "whitespace", value: "" });
      }
    }
    nodes.push(node);

    function isBetween(kind1, kind2) {
      return (
        (lastNode.kind === kind1 && node.kind === kind2) ||
        (lastNode.kind === kind2 && node.kind === kind1)
      );
    }
  }
}
566808
function parseChildren() {

		let ary = [], node;

		// 循环直到遇见标签闭合Token
		while ( !reader.eof() && reader.getCurrentToken().type != options.TypeTagClose ) {
			node = parseTag();
			node && ary.push(node);

			node = parseComment();
			node && ary.push(node);

			node = parseCodeBlock();
			node && ary.push(node);

			node = parseText();
			node && ary.push(node);

			node = parseExpression();
			node && ary.push(node);
		}

		return ary;
	}
376546
protected void segmentAfterRule(String sentence, String normalized, List<String> wordList)
    {
        if (!enableRuleBasedSegment)
        {
            segmenter.segment(sentence, normalized, wordList);
            return;
        }
        int start = 0;
        int end = start;
        byte preType = typeTable[normalized.charAt(end)];
        byte curType;
        while (++end < normalized.length())
        {
            curType = typeTable[normalized.charAt(end)];
            if (curType != preType)
            {
                if (preType == CharType.CT_NUM)
                {
                    // 浮点数识别
                    if ("，,．.".indexOf(normalized.charAt(end)) != -1)
                    {
                        if (end + 1 < normalized.length())
                        {
                            if (typeTable[normalized.charAt(end + 1)] == CharType.CT_NUM)
                            {
                                continue;
                            }
                        }
                    }
                    else if ("年月日时分秒".indexOf(normalized.charAt(end)) != -1)
                    {
                        preType = curType; // 交给统计分词
                        continue;
                    }
                }
                pushPiece(sentence, normalized, start, end, preType, wordList);
                start = end;
            }
            preType = curType;
        }
        if (end == normalized.length())
            pushPiece(sentence, normalized, start, end, preType, wordList);
    }
325152
public boolean contains(String group, String value, String... otherValues) {
		if (ArrayUtil.isNotEmpty(otherValues)) {
			// 需要测试多个值的情况
			final List<String> valueList = Arrays.asList(otherValues);
			valueList.add(value);
			return contains(group, valueList);
		} else {
			// 测试单个值
			final LinkedHashSet<String> valueSet = getValues(group);
			if (CollectionUtil.isEmpty(valueSet)) {
				return false;
			}

			return valueSet.contains(value);
		}
	}
1145640
function parse(text) {
	//位置: 主位置, 引入, 功能, 子功能, 功能结束符, 语句, 表达式
	let index = 0,
	    include = 0,
	    func = 0,
	    subfunc = 0,
	    end = 0,
	    script = 0,
	    sentence = 0;
	//根, 当前位置
	let root = { list: [], main: null, sub: [] },
	    current = root;
	root.main = root;
	do {
		//首先查找各起点位置
		if (include !== -1) {
			include = text.indexOf("<#", index);
		}
		if (func !== -1) {
			func = text.indexOf("<@", index);
		}
		if (subfunc !== -1) {
			subfunc = text.indexOf("<|", index);
		}
		if (end !== -1) {
			end = text.indexOf("</>", index);
		}
		if (script !== -1) {
			script = text.indexOf("<`", index);
		}
		if (sentence !== -1) {
			sentence = text.indexOf("<%", index);
		}
		//选中的类型, 新的位置
		let type = '',
		    newindex = text.length;
		//找出最靠前的起点及类型
		if (include !== -1 && include < newindex) {
			newindex = include;
			type = 'include';
		}
		if (func !== -1 && func < newindex) {
			newindex = func;
			type = 'func';
		}
		if (subfunc !== -1 && subfunc < newindex) {
			newindex = subfunc;
			type = 'subfunc';
		}
		if (end !== -1 && end < newindex) {
			newindex = end;
			type = 'end';
		}
		if (script !== -1 && script < newindex) {
			newindex = script;
			type = 'script';
		}
		if (sentence !== -1 && sentence < newindex) {
			newindex = sentence;
			type = '%';
		}
		//将中间字符加入队列
		if (index < newindex) {
			current.list.push({ type: "text", value: text.substring(index, newindex) });
		}
		index = newindex;
		switch (type) {
			case 'include':
			case 'func':
			case 'subfunc':
			case 'script':
			case '%':
				//没有数据则直接返回
				if (!text.substr(index + 2, 1)) {
					break;
				}
				//获取标签信息
				let taginfo = parseTag(text, index + 2, { 'include': '/', 'func': '', 'subfunc': '', 'script': '`', '%': '%' }[type]);
				if (!taginfo) {
					//无信息表示数据无效
					current.list.push({ type: "text", value: "<" });
					index++;
				} else if (type === 'include') {
					//包含文件
					index = taginfo.index;
					let tag = { type, value: Script(taginfo.value) };
					current.list.push(tag);
				} else if (type === '%') {
					//表达式
					//设置新下标
					index = taginfo.index;
					let tag = { type: null, value: null };
					//获取首字符，进行进一步类型判断
					let ch = taginfo.value[0];
					//输出
					if (ch === "=") {
						tag.type = 'echo';tag.value = Script(taginfo.value.substr(1));
					}
					//脚本
					else {
							tag.type = 'script';tag.value = Script(taginfo.value);
						}
					//加入列表
					current.list.push(tag);
				} else if (type === 'script') {
					//表达式
					//设置新下标
					index = taginfo.index;
					let tag = { type: null, value: null };
					//获取首字符，进行进一步类型判断
					let ch = taginfo.value[0];
					if (ch === "=") {
						//输出
						tag.type = 'echo';
						tag.value = Script(taginfo.value.substr(1));
					} else if (ch === "`") {
						//不解析脚本
						let v = taginfo.value,
						    i = 1,
						    j = v.length - 1;
						while (v[i] === '`') {
							i++;
						}
						while (v[j] === '`') {
							j--;
						}
						if (j <= i) {
							break;
						}
						tag.type = 'sentence';
						tag.value = Script(taginfo.value.substr(i, j + 1));
					} else {
						//脚本
						tag.type = 'script';
						tag.value = Script(taginfo.value);
					}
					//加入列表
					current.list.push(tag);
				} else if (type === 'func' || type === 'subfunc') {
					//功能, 子功能
					//标签解析
					let funcInfo = parseFunc(taginfo.value);
					if (!funcInfo) {
						//标签解析失败
						current.list.push({ type: "text", value: "<" });
						index++;
					} else {
						index = taginfo.index;
						if (type === 'func') {
							//创建
							let tag = {
								type: 'func', parent: current, list: [],
								name: funcInfo.name, value: funcInfo.value,
								attr: funcInfo.attr,
								main: null, sub: []
							};
							tag.main = tag;
							current.list.push(tag);
							if (!taginfo.closed) {
								current = tag;
							}
						} else if (type === 'subfunc') {
							let tag = {
								list: [], parent: current.parent,
								name: funcInfo.name, value: funcInfo.value,
								attr: funcInfo.attr, main: current.main
							};
							current.main.sub.push(tag);
							delete current.main;
							delete current.parent;
							current = tag;
							if (taginfo.closed && current !== root) {
								let parent = current.parent;
								delete current.parent;
								delete current.main;
								current = parent;
							}
						}
					}
				} else {
					index = taginfo.index;
					let tag = { type: type, value: null };
					tag.value = Script(taginfo.value);
					current.list.push(tag);
				}
				break;
			case 'end':
				if (current !== root) {
					let parent = current.parent;
					delete current.parent;
					delete current.main;
					current = parent;
				}
				index += 3;
				break;
		}
	} while (subfunc !== -1 || func !== -1 || end !== -1 || include !== -1 || script !== -1 || sentence !== -1);
	return root.list;
}
988634
public void rmLittlePathByScore() {
        int maxTo = -1;
        Term temp = null;
        for (int i = 0; i < terms.length; i++) {
            if (terms[i] == null) {
                continue;
            }
            Term maxTerm = null;
            double maxScore = 0;
            Term term = terms[i];
            // 找到自身分数对大最长的

            do {
                if (maxTerm == null || maxScore > term.score()) {
                    maxTerm = term;
                } else if (maxScore == term.score() && maxTerm.getName().length() < term.getName().length()) {
                    maxTerm = term;
                }

            } while ((term = term.next()) != null);
            term = maxTerm;
            do {
                maxTo = term.toValue();
                maxScore = term.score();
                if (maxTo - i == 1 || i + 1 == terms.length)
                    continue;
                boolean flag = true;// 可以删除
                out: for (int j = i; j < maxTo; j++) {
                    temp = terms[j];
                    if (temp == null) {
                        continue;
                    }
                    do {
                        if (temp.toValue() > maxTo || temp.score() < maxScore) {
                            flag = false;
                            break out;
                        }
                    } while ((temp = temp.next()) != null);
                }
                // 验证通过可以删除了
                if (flag) {
                    for (int j = i + 1; j < maxTo; j++) {
                        terms[j] = null;
                    }
                }
            } while ((term = term.next()) != null);
        }
    }
691333
private List<String> fullSeg(final String text, final int start) {
        List<String> result = new LinkedList<>();
        final int textLen = text.length();
        //剩下文本长度
        int len = textLen - start;
        //最大截取长度
        int interceptLength = getInterceptLength();
        if(len > interceptLength){
            len = interceptLength;
        }
        while(len > 1){
            if(getDictionary().contains(text, start, len) || RecognitionTool.recog(text, start, len)){
                result.add(text.substring(start, start + len));
            }
            len--;
        }
        if(textLen <= CHAR_IS_WORD_LENGTH_LESS_THAN || result.isEmpty()){
            //增加单字词
            result.add(text.substring(start, start + 1));
        }
        return result;
    }
369713
function parseDocToTree(doc, defaultTitle) {
  // save the parsed tree
  const tree = [];
  const lines = doc.split('\n');
  let line, m;

  // 保存一个文档区域
  let section = null;
  let parentSection = null;
  // if in a ``` block
  let inBlock = false;

  // little trick，如果第一行不是 #，则增加一个 #
  if (!/^#(?!#)/.test(lines[0])) {
    lines.unshift('# ' + defaultTitle);
  }

  while ((line = lines.shift()) !== undefined) {
    m = line && line.match(/^(#+)(?!#)(.*)$/);

    if (!inBlock && m && m.length == 3) {
      section = {
        title: m[2].trim(),
        level: m[1].length,
        parent: null,
        children: []
      };

      while (parentSection && parentSection.level >= section.level) {
        parentSection = parentSection.parent;
      }

      if (parentSection) {
        section.parent = parentSection;
        parentSection.children.push(section);
        parentSection = section;
      } else {
        tree.push(section);
        parentSection = section;
      }
    } else if (section) {
      if (line.indexOf('```') === 0) {
        inBlock = !inBlock;
      }
      section.children.push(line);
    } else {
      throw new Error(`NOT VALID LINE: ${line}`);
    }
  }

  return tree;
}
241782
private List<List<EventData>> split(List<EventData> datas) {
        List<List<EventData>> result = new ArrayList<List<EventData>>();
        if (datas == null || datas.size() == 0) {
            return result;
        } else {
            int[] bits = new int[datas.size()];// 初始化一个标记，用于标明对应的记录是否已分入某个batch
            for (int i = 0; i < bits.length; i++) {
                // 跳过已经被分入batch的
                while (i < bits.length && bits[i] == 1) {
                    i++;
                }

                if (i >= bits.length) { // 已处理完成，退出
                    break;
                }

                // 开始添加batch，最大只加入batchSize个数的对象
                List<EventData> batch = new ArrayList<EventData>();
                bits[i] = 1;
                batch.add(datas.get(i));
                for (int j = i + 1; j < bits.length && batch.size() < batchSize; j++) {
                    if (bits[j] == 0 && canBatch(datas.get(i), datas.get(j))) {
                        batch.add(datas.get(j));
                        bits[j] = 1;// 修改为已加入
                    }
                }
                result.add(batch);
            }

            return result;
        }
    }
808045
static String[] splitOnTokens(String text)
    {
        // used by wildcardMatch
        // package level so a unit test may run on this

        if (text.indexOf('?') == -1 && text.indexOf('*') == -1)
        {
            return new String[]
                    {
                        text
                    };
        }

        char[] array = text.toCharArray();
        ArrayList<String> list = new ArrayList<String>();
        StringBuilder buffer = new StringBuilder();
        for (int i = 0; i < array.length; i++)
        {
            if (array[i] == '?' || array[i] == '*')
            {
                if (buffer.length() != 0)
                {
                    list.add(buffer.toString());
                    buffer.setLength(0);
                }
                if (array[i] == '?')
                {
                    list.add("?");
                }
                else if (list.isEmpty()
                        || i > 0 && list.get(list.size() - 1).equals("*") == false)
                {
                    list.add("*");
                }
            }
            else
            {
                buffer.append(array[i]);
            }
        }
        if (buffer.length() != 0)
        {
            list.add(buffer.toString());
        }

        return list.toArray(new String[list.size()]);
    }
475713
public static List<Word> seg(String text, SegmentationAlgorithm segmentationAlgorithm){        
        List<Word> words = SegmentationFactory.getSegmentation(segmentationAlgorithm).seg(text);
        //停用词过滤
        StopWord.filterStopWords(words);
        return words;
    }
1040069
function splitAll(source, cb, rules) {
  var done = false
    , stash = '';
  do {
    splitNext( source, function(event, response) {
        response.stash = stash;
        if (event == 'end') done = true;
        else {
          
          source = response.rhs;
          stash += response.lhs + response.token;

          response.consume = function(length) {
            stash += source.substr( 0, length );
            source = source.substr( length, source.length );
          };

          response.resetStash = function() {
            stash = '';
          };
          
          response.break = function() {
            done = true; 
          };
        }
        cb( event, response );
      }, 
      rules );
  } 
  while(!done);
}
1375064
def segment(self, text, lower=True, use_stop_words=True, use_speech_tags_filter=False):
        
        jieba_result = cut_for_search(text)
        jieba_result = [w for w in jieba_result]

        # 去除特殊符号
        word_list = [w.strip() for w in jieba_result]
        word_list = [word for word in word_list if len(word) > 0]

        if use_speech_tags_filter == False:
            jieba_result = [w for w in jieba_result]

        if lower:
            word_list = [word.lower() for word in word_list]

        if use_stop_words:
            word_list = [word.strip() for word in word_list if word.strip() not in self.stop_words]

        return word_list
338920
def _run_split_on_punc(self, text):
        """"""
        chars = list(text)
        i = 0
        start_new_word = True
        output = []
        while i < len(chars):
            char = chars[i]
            if self._is_punctuation(char):
                output.append([char])
                start_new_word = True
            else:
                if start_new_word:
                    output.append([])
                start_new_word = False
                output[-1].append(char)
            i += 1

        return [''.join(x) for x in output]
734736
def split(self, text):
        '''
        
        '''
        # nesting character -> count
        counters = {"'": False, '(': 0}

        def reverse(char):

            def wrapped():
                counters[char] = not counters[char]
            return wrapped

        def increase(char):

            def wrapped():
                counters[char] += 1
            return wrapped

        def decrease(char):

            def wrapped():
                counters[char] -= 1
            return wrapped

        def is_top_level():
            return all([not x for x in counters.values()])

        def fail():
            raise BadSyntax('Syntax error processing line: %s' %\
                            self._last_line)


        temp = ""
        # end of field flag indicates that we expect next character to be
        # either whitespace of split
        eof_flag = False

        def append_char(temp, char):
            if eof_flag:
                fail()
            temp += char
            return temp

        # dictionary char -> handler
        nesters = {"'": reverse("'"), "(": increase('('), ")": decrease('(')}
        # chars to split on
        split = (',', )
        # chars to swallow
        consume = (' ', '\n')

        result = list()

        self.log("spliting: %s", text)

        for char in text:
            if char in nesters:
                nesters[char]()
                temp = append_char(temp, char)
            elif not is_top_level():
                temp = append_char(temp, char)
            elif char in consume:
                if len(temp) > 0:
                    eof_flag = True
                continue
            elif char in split:
                result.append(temp)
                temp = ""
                eof_flag = False
            else:
                temp = append_char(temp, char)
        if len(temp) > 0:
            result.append(temp)

        if not is_top_level():
            fail()
        self.log('Split returns %r', result)
        return result
955236
public static List<String> splitAsList(String text, String delimiter) {
        List<String> answer = new ArrayList<String>();
        if (text != null && text.length() > 0) {
            answer.addAll(Arrays.asList(text.split(delimiter)));
        }
        return answer;
    }