Removes leading and trailing puncutation
        Leaves contractions and most emoticons
            Does not preserve punc-plus-letter emoticons (e.g. :D)
961462
def _words_and_emoticons(self):
        
        wes = self.text.split()
        words_punc_dict = self._words_plus_punc()
        wes = [we for we in wes if len(we) > 1]
        for i, we in enumerate(wes):
            if we in words_punc_dict:
                wes[i] = words_punc_dict[we]
        return wes
1212880
function clean(cmsString) {
    if (cmsString.indexOf('\r\n') >= 0) {
        cmsString = cmsString.replace(/\r\n/g, '\n');
        cmsString = cmsString.replace(/\r/g, '\n');
    }
    cmsString = cmsString.replace(/\n{5,}/g, '\n\n\n\n'); //more than 5 line breaks breaks the parser
    return cmsString;
}
666240
function trimToMaxWidth (width, text) {
	var truncated = text.split('\n').map(function (line) {
		return line.substring(0, width);
	});
	return truncated.join('\n');
}
808715
function basicGithubId (text) {
  return text
    .toLowerCase()
    .replace(/\s+/g, '-')         // whitespaces
    .replace(/%25|%/ig, '')       // remove single % signs
    .replace(RE_ENTITIES, '')     // remove xml/html entities
    .replace(RE_SPECIALS_DOT, '') // single chars that are removed
    .replace(RE_CJK, '')          // CJK punctuations that are removed
}
527098
function getEmoticons(arr) {
  var nested = arr.map(function (str) {
    return str.match(between);
  });
  
  return flatten(nested)
    .filter(function (el) {
      return el;
    })
    .filter(emoticonsPredicate)
    .map(function (str) {
      return str.slice(1, str.length - 1);
    });
}
1179191
def remove_punctuation(text, exceptions=[]):
    

    all_but = [
        r'\w',
        r'\s'
    ]

    all_but.extend(exceptions)

    pattern = '[^{}]'.format(''.join(all_but))

    return re.sub(pattern, '', text)
15579
function stripColon(text) {
    var i = text.indexOf(":");
    if (i >= 0) {
        text = text.substr(i).trim();
        if (text[1] == " ") {
            text = text.substr(1).trim();
        }
    }
    return text;
}
923061
def remove_docstrings(tokens):
    
    prev_tok_type = None
    for index, tok in enumerate(tokens):
        token_type = tok[0]
        if token_type == tokenize.STRING:
            if prev_tok_type == tokenize.INDENT:
                # Definitely a docstring
                tokens[index][1] = '' # Remove it
                # Remove the leftover indentation and newline:
                tokens[index-1][1] = ''
                tokens[index-2][1] = ''
            elif prev_tok_type == tokenize.NL:
                # This captures whole-module docstrings:
                if tokens[index+1][0] == tokenize.NEWLINE:
                    tokens[index][1] = ''
                    # Remove the trailing newline:
                    tokens[index+1][1] = ''
        prev_tok_type = token_type
409708
def strip_caret_codes(text):
    
    ## temporarily escape out ^^
    text = text.replace('^^', '\x00')
    for token, foo in _ANSI_CODES:
        text = text.replace(token, '')
    return text.replace('\x00', '^')
565205
def clean_text(s):
    """"""
    SPACES_RE = re.compile(r'\s+')
    SPECIAL_CHARS_RE = re.compile(r'[^\w\s\.\-\(\)]')
    s = SPACES_RE.sub(' ', s)
    s = s.strip()
    s = SPECIAL_CHARS_RE.sub('', s)
    return s
1307704
def trim_punctuation
      lambda do |rec, accumulator|
        accumulator.collect! {|s| Marc21.trim_punctuation(s)}
      end
    end
698982
def other_punctuation():
    
    punc = ''.join(
        set(symbols.ALL_PUNC) -
        set(symbols.TONE_MARKS) -
        set(symbols.PERIOD_COMMA) -
        set(symbols.COLON))
    return RegexBuilder(
        pattern_args=punc,
        pattern_func=lambda x: u"{}".format(x)).regex
138545
function removeTrailingLinebreaks(text) {
    let linebreak = determineLinebreaks(text);
    if (linebreak === "") return text;

    while (text.endsWith(linebreak)) {
        text = text.slice(0, 0 - linebreak.length);
    }

    return text;
}
1111090
function stripSpace() {
      if (hasTag && !nonSpace) {
        while (spaces.length) {
          tokens.splice(spaces.pop(), 1);
        }
      } else {
        spaces = [];
      }

      hasTag = false;
      nonSpace = false;
    }
1158236
def clean_strict(text, boundary=WS):
    """"""
    # transliterate to ascii
    text = ascii_text(text)
    # replace punctuation and symbols
    text = CHARACTERS_REMOVE_RE.sub('', text)
    text = category_replace(text)
    # pad out for company type replacements
    text = ''.join((boundary, collapse_spaces(text), boundary))
    return text
703775
def strip_symbols(s, pass_symbols=(u'й', u'Й', u'\n')):
    
    result = []
    for char in s:
        # Pass these symbols without processing
        if char in pass_symbols:
            result.append(char)
            continue
        for c in unicodedata.normalize('NFKC', char):
            if unicodedata.category(c) == 'Zs':
                result.append(u' ')
                continue
            if unicodedata.category(c) not in ['So', 'Mn', 'Lo', 'Cn', 'Co', 'Cf', 'Cc']:
                result.append(c)
    return u"".join(result)
48775
def phi5_plaintext_cleanup(text, rm_punctuation=False, rm_periods=False):
    
    # This works OK, doesn't get some
    # Note: rming all characters between {} and ()
    remove_comp = regex.compile(r'-\n|«|»|\<|\>|\.\.\.|‘|’|_|{.+?}|\(.+?\)|\(|\)|“|#|%|⚔|&|=|/|\\|〚|†|『|⚖|–|˘|⚕|☾|◌|◄|►|⌐|⌊|⌋|≈|∷|≈|∞|”|[0-9]')
    text = remove_comp.sub('', text)

    new_text = None
    if rm_punctuation:
        new_text = ''
        punctuation = [',', ';', ':', '"', "'", '?', '-', '!', '*', '[', ']', '{', '}']
        if rm_periods:
            punctuation += ['.']
        for char in text:
            # rm acute combining acute accents made by TLGU
            # Could be caught by regex, tried and failed, not sure why
            if bytes(char, 'utf-8') == b'\xcc\x81':
                pass
            # second try at rming some punctuation; merge with above regex
            elif char in punctuation:
                pass
            else:
                new_text += char
    if new_text:
        text = new_text

    # replace line breaks w/ space
    replace_comp = regex.compile(r'\n')
    text = replace_comp.sub(' ', text)

    comp_space = regex.compile(r'\s+')
    text = comp_space.sub(' ', text)

    return text