Splits punctuation on a piece of text.
338920
def _run_split_on_punc(self, text):
        """"""
        chars = list(text)
        i = 0
        start_new_word = True
        output = []
        while i < len(chars):
            char = chars[i]
            if self._is_punctuation(char):
                output.append([char])
                start_new_word = True
            else:
                if start_new_word:
                    output.append([])
                start_new_word = False
                output[-1].append(char)
            i += 1

        return [''.join(x) for x in output]
1344251
function fix(text) {
    if (!text) return;
    return text.split('\n').map(function(line) {
      if (line.match(/^- /))
        line = '  ' + line;
      return line;
    }).join('\n');
  }
1009900
function punctuation(text){

  // Dashes
  text = text.replace(/--/g, '\u2014');
  text = text.replace(/\s*\u2014\s*/g,'\u2009\u2014\u2009'); //this has thin spaces

  // Elipses
  text = text.replace(/\.\.\./g,'…');

  // Nbsp for punc with spaces
  var NBSP = '\u00a0';
  var NBSP_PUNCTUATION_START = /([«¿¡]) /g;
  var NBSP_PUNCTUATION_END = / ([!?:;.,‽»])/g;

  text = text.replace(NBSP_PUNCTUATION_START, '$1' + NBSP);
  text = text.replace(NBSP_PUNCTUATION_END, NBSP + '$1');

  return text;
}
584479
def _normalize_answer(text: str) -> str:
    """"""

    parts = [_white_space_fix(_remove_articles(_normalize_number(_remove_punc(_lower(token)))))
             for token in _tokenize(text)]
    parts = [part for part in parts if part.strip()]
    normalized = ' '.join(parts).strip()
    return normalized
672325
def split(self, text):
        """"""
        text = cleanup(text)
        return self.sent_detector.tokenize(text.strip())
82941
def tokenize(self, text, include_punc=True, **kwargs):
        
        return self.tokenizer.word_tokenize(text, include_punc, **kwargs)
15579
function stripColon(text) {
    var i = text.indexOf(":");
    if (i >= 0) {
        text = text.substr(i).trim();
        if (text[1] == " ") {
            text = text.substr(1).trim();
        }
    }
    return text;
}
765358
def indent(text, n=4):
    """"""
    _indent = ' ' * n
    return '\n'.join(_indent + line for line in text.split('\n'))
1128743
private String[] splitInTwo(String text)	{
	  Matcher matcher = LINE_SEPARATOR.matcher(text);
	  while (matcher.find()){
		  int start = matcher.start();
		  if (start > 0 && start < text.length())	{
			  String segment = text.substring(0, start);
			  if (segment.trim().length() > 0)
				  return new String[] {text.substring(0, start), text.substring(start)};
		  }
	  }
	  return new String[] {text};
  }
14054
def split(self, text):
        """"""

        import re
        if re.search('\n\n', text):
            return text.split('\n\n')
        elif re.search('\r\n\r\n', text):
            return text.split('\r\n\r\n')
        else:
            LOGGER.error("'%s' does not appear to be a 'srt' subtitle file",
                         self.filename)
            sys.exit(1)
875092
def replace(text, separator = LINE_BREAK)
      text.gsub(/([ ]+)#{separator}/, "\\1")
          .gsub(/#{separator}(?<space>[ ]+)/, "\\k<space>")
          .gsub(/#{separator}/, ' ')
    end
237370
def clean(self, text):
        """"""
        return ''.join([c for c in text if c in self.alphabet])
408801
private static void splitInSentences(HashSet<String> sentencesSet, String text) {
		
		if (inputType == inputTypes.PARAGRAPH) { // split in multiple sentences
			LinkedList<LinkedList<Word>> sentences = Splitting.tokenizeSentences(LVMorphologyReaderAndWriter.getAnalyzer(), text);
			for (LinkedList<Word> sentence : sentences) 
			{
				if(sentence != null)
				{
					String madeSentence = "";
					for(int w = 0; w < sentence.size(); w++)
					{
						madeSentence = madeSentence + " " + sentence.get(w);
					}
					madeSentence = madeSentence.trim();
					//pārbaudam, vai teikuma atšķiribas nav punkts teikuma beigas
					if(madeSentence.length() > 0 && madeSentence.lastIndexOf(".") >= 0) madeSentence = madeSentence.substring(0, madeSentence.length() - 1);
					sentencesSet.add(madeSentence.trim());
				}
			}
		} 
	}
923413
def _split(string, splitters):
    """"""
    part = ''
    for character in string:
        if character in splitters:
            yield part
            part = ''
        else:
            part += character
    yield part
1283351
function splitText(text, options) {
  const KIND_NON_CJK = "non-cjk";
  const KIND_CJ_LETTER = "cj-letter";
  const KIND_K_LETTER = "k-letter";
  const KIND_CJK_PUNCTUATION = "cjk-punctuation";

  const nodes = [];

  (options.proseWrap === "preserve"
    ? text
    : text.replace(new RegExp(`(${cjkPattern})\n(${cjkPattern})`, "g"), "$1$2")
  )
    .split(/([ \t\n]+)/)
    .forEach((token, index, tokens) => {
      // whitespace
      if (index % 2 === 1) {
        nodes.push({
          type: "whitespace",
          value: /\n/.test(token) ? "\n" : " "
        });
        return;
      }

      // word separated by whitespace

      if ((index === 0 || index === tokens.length - 1) && token === "") {
        return;
      }

      token
        .split(new RegExp(`(${cjkPattern})`))
        .forEach((innerToken, innerIndex, innerTokens) => {
          if (
            (innerIndex === 0 || innerIndex === innerTokens.length - 1) &&
            innerToken === ""
          ) {
            return;
          }

          // non-CJK word
          if (innerIndex % 2 === 0) {
            if (innerToken !== "") {
              appendNode({
                type: "word",
                value: innerToken,
                kind: KIND_NON_CJK,
                hasLeadingPunctuation: punctuationRegex.test(innerToken[0]),
                hasTrailingPunctuation: punctuationRegex.test(
                  getLast(innerToken)
                )
              });
            }
            return;
          }

          // CJK character
          appendNode(
            punctuationRegex.test(innerToken)
              ? {
                  type: "word",
                  value: innerToken,
                  kind: KIND_CJK_PUNCTUATION,
                  hasLeadingPunctuation: true,
                  hasTrailingPunctuation: true
                }
              : {
                  type: "word",
                  value: innerToken,
                  kind: kRegex.test(innerToken)
                    ? KIND_K_LETTER
                    : KIND_CJ_LETTER,
                  hasLeadingPunctuation: false,
                  hasTrailingPunctuation: false
                }
          );
        });
    });

  return nodes;

  function appendNode(node) {
    const lastNode = getLast(nodes);
    if (lastNode && lastNode.type === "word") {
      if (
        (lastNode.kind === KIND_NON_CJK &&
          node.kind === KIND_CJ_LETTER &&
          !lastNode.hasTrailingPunctuation) ||
        (lastNode.kind === KIND_CJ_LETTER &&
          node.kind === KIND_NON_CJK &&
          !node.hasLeadingPunctuation)
      ) {
        nodes.push({ type: "whitespace", value: " " });
      } else if (
        !isBetween(KIND_NON_CJK, KIND_CJK_PUNCTUATION) &&
        // disallow leading/trailing full-width whitespace
        ![lastNode.value, node.value].some(value => /\u3000/.test(value))
      ) {
        nodes.push({ type: "whitespace", value: "" });
      }
    }
    nodes.push(node);

    function isBetween(kind1, kind2) {
      return (
        (lastNode.kind === kind1 && node.kind === kind2) ||
        (lastNode.kind === kind2 && node.kind === kind1)
      );
    }
  }
}
381675
function splitWithSingleSpace(text) {
  const parts = [];

  let lastPart = undefined;
  for (const part of text.split(/( +)/g)) {
    if (part !== " ") {
      if (lastPart === " ") {
        parts.push(part);
      } else {
        parts.push((parts.pop() || "") + part);
      }
    } else if (lastPart === undefined) {
      parts.unshift("");
    }

    lastPart = part;
  }

  if (lastPart === " ") {
    parts.push((parts.pop() || "") + " ");
  }

  if (parts[0] === "") {
    parts.shift();
    parts.unshift(" " + (parts.shift() || ""));
  }

  return parts;
}
211330
def split_input_output(self, text):
        """"""
        lines = _to_lines(text)
        i = 0
        for line in lines:
            if _starts_with_regex(line, self.input_prompt_regex):
                i += 1
            else:
                break
        return lines[:i], lines[i:]