remove punctuation, lowercase, stem
187917
def computePDFSimilarity(paperId, userPDF):
    if not isPDFInDb(paperId):
         return None

    userPDF.save("temp.pdf")
    # check_call is blocking
    check_call(['pdftotext', '-enc', 'UTF-8', "temp.pdf", "temp.txt"])
    os.remove("temp.pdf")
    
    a = open("temp.txt", 'r').read()
    b = open(join(dbPath, paperId) + ".txt", 'r').read()

    import nltk, string
    from sklearn.feature_extraction.text import TfidfVectorizer

    stemmer = nltk.stem.porter.PorterStemmer()
    remove_punctuation_map = dict((ord(char), None) for char in string.punctuation)

    def stem_tokens(tokens):
        return [stemmer.stem(item) for item in tokens]

    ''''''
    def normalize(text):
        return stem_tokens(nltk.word_tokenize(text.lower().translate(remove_punctuation_map)))

    vectorizer = TfidfVectorizer(tokenizer=normalize, stop_words='english')

    def cosine_sim(text1, text2):
        tfidf = vectorizer.fit_transform([text1, text2])
        return ((tfidf * tfidf.T).A)[0,1]

    similarity = cosine_sim(a, b)

    os.remove("temp.txt")

    return similarity
1250298
def remove_punctuation_dict() -> Dict[int, None]:
    
    tmp = dict((i, None) for i in range(sys.maxunicode)
               if unicodedata.category(chr(i)).startswith('P'))
    return tmp
71343
public function cleanString()
		{
			//get rid of newline holders in values
			$this->processString = str_replace('\\\\\\\\n', ' ', $this->processString);

			//replace newline holders with newline
			$this->processString = str_replace('\n', "\n", $this->processString);

			//match returns extra curly bracket at EOL, get rid of it
			$this->processString = substr($this->processString, 0, -1);

			//convert unicode encoded characters to html entities hex
			$this->processString = preg_replace('/\\\\u([0-9a-fA-F]{4})/', '&#x$1;', $this->processString);

			//get rid of escapes added for javascript
			$this->processString = stripslashes($this->processString);
		}
1307704
def trim_punctuation
      lambda do |rec, accumulator|
        accumulator.collect! {|s| Marc21.trim_punctuation(s)}
      end
    end
719249
def clean(self):
        """"""
        terms = list(self)

        for t in terms:
            self.doc.remove_term(t)
263117
def normalizer(text, exclusion=OPERATIONS_EXCLUSION, lower=True, separate_char='-', **kwargs):
    
    clean_str = re.sub(r'[^\w{}]'.format(
        "".join(exclusion)), separate_char, text.strip()) or ''
    clean_lowerbar = clean_str_without_accents = strip_accents(clean_str)

    if '_' not in exclusion:
        clean_lowerbar = re.sub(r'\_', separate_char, clean_str_without_accents.strip())

    limit_guion = re.sub(r'\-+', separate_char, clean_lowerbar.strip())

    # TODO: refactor with a regexp
    if limit_guion and separate_char and separate_char in limit_guion[0]:
        limit_guion = limit_guion[1:]

    if limit_guion and separate_char and separate_char in limit_guion[-1]:
        limit_guion = limit_guion[:-1]

    if lower:
        limit_guion = limit_guion.lower()

    return limit_guion
978118
def cleanString(someText):
    
    ret = ''
    if someText is not None:
        ret = filter(unicode.isalnum, someText.lower())
    return ret
1310866
def _remove_brackets(x, i):
    """"""

    assert x[i]['t'] == 'Cite'
    assert i > 0 and i < len(x) - 1

    # Check if the surrounding elements are strings
    if not x[i-1]['t'] == x[i+1]['t'] == 'Str':
        return

    # Trim off curly brackets
    if x[i-1]['c'].endswith('{') and x[i+1]['c'].startswith('}'):
        if len(x[i+1]['c']) > 1:
            x[i+1]['c'] = x[i+1]['c'][1:]
        else:
            del x[i+1]

        if len(x[i-1]['c']) > 1:
            x[i-1]['c'] = x[i-1]['c'][:-1]
        else:
            del x[i-1]
15579
function stripColon(text) {
    var i = text.indexOf(":");
    if (i >= 0) {
        text = text.substr(i).trim();
        if (text[1] == " ") {
            text = text.substr(1).trim();
        }
    }
    return text;
}
928216
private void removeLowerCaseHomonymsForAbbreviations(AnalyzedSentence input) {
    AnalyzedTokenReadings[] tokens = input.getTokensWithoutWhitespace();
    for (int i = 1; i < tokens.length; i++) {
      if( StringUtils.isAllUpperCase(tokens[i].getToken())
          && PosTagHelper.hasPosTagPart(tokens[i], ":abbr") ) {
        
        List<AnalyzedToken> analyzedTokens = tokens[i].getReadings();
        for(int j=analyzedTokens.size()-1; j>=0; j--) {
          AnalyzedToken analyzedToken = analyzedTokens.get(j);
          
          if( ! PosTagHelper.hasPosTagPart(analyzedToken, ":abbr") 
              && ! JLanguageTool.SENTENCE_END_TAGNAME.equals(analyzedToken) ) {
            tokens[i].removeReading(analyzedToken);
          }
        }
      }
    }    
  }
380654
def strip_punctuation_space(value):
    ""
    def strip_punctuation(string):
        replacement_list = (
            (' .',  '.'),
            (' :',  ':'),
            ('( ',  '('),
            (' )',  ')'),
        )
        for match, replacement in replacement_list:
            string = string.replace(match, replacement)
        return string
    if value == None:
        return None
    if type(value) == list:
        return [strip_punctuation(v) for v in value]
    return strip_punctuation(value)
801592
def lowercase(text_string):
    '''
    
    '''
    if text_string is None or text_string == "":
        return ""
    elif isinstance(text_string, str):
        return text_string.lower()
    else:
        raise InputError("string not passed as argument for text_string")
1084541
function cleanDoctype(name) {
  return String(name)
    .toLowerCase()
    .replace(/([a-z]+|\d+)/, ' $1 ')
    .replace(/\.0+(?!\d)/, '')
    .replace(/\.01/, '')
    .replace(/\.2/, '')
    .replace(
      /\b(xhtml|html|mathml|svg|full|basic|tiny|strict|frameset|basic)\b/g,
      character
    )
    .replace(/ t(?:ransitional)?/, '')
    .replace(/\s+/g, '')
}
738747
def text_cleanup(data, key, last_type):
    
    if key in data and last_type == STRING_TYPE:
        data[key] = data[key].strip()
    return data
515604
private void removeLowerCaseBadForUpperCaseGood(AnalyzedSentence input) {
    AnalyzedTokenReadings[] tokens = input.getTokensWithoutWhitespace();
    for (int i = 1; i < tokens.length; i++) {
      if( tokens[i].getReadings().size() > 1
          && StringTools.isCapitalizedWord(tokens[i].getToken())
          && LemmaHelper.hasLemma(tokens[i], Pattern.compile("[А-ЯІЇЄҐ][а-яіїєґ'-].*"), Pattern.compile(".*?:prop")) ) {

        String lowerLemmaToCheck = tokens[i].getAnalyzedToken(0).getLemma().toLowerCase();
        
        List<AnalyzedToken> analyzedTokens = tokens[i].getReadings();
        for(int j=analyzedTokens.size()-1; j>=0; j--) {
          AnalyzedToken analyzedToken = analyzedTokens.get(j);
          
          if( PosTagHelper.hasPosTagPart(analyzedToken, ":bad") 
              && lowerLemmaToCheck.equals(analyzedToken.getLemma()) ) {
            tokens[i].removeReading(analyzedToken);
          }
        }
      }
    }
  }
86091
def _convert_punctuation(punctuation, conversion_table):
    """"""
    if punctuation in conversion_table:
        return conversion_table[punctuation]
    return re.escape(punctuation)
258938
def remove(text, what, count=None, strip=False):
    '''
    
    '''
    return replace(text, what, '', count=count, strip=strip)