Super Cleaning for decode and strip all html tag
963285
public static function hardDecode($string) {
		$data = htmlspecialchars_decode($string);
		$data = html_entity_decode($data);
		$data = strip_tags($data);
		$data = chop(Utility::convert_smart_quotes($data));
		$data = str_replace(array("\r", "\n", "	"), "", $data);

		return ($data);
	}
449858
def clean_description desc
      strip_stopwords(
        desc.gsub(XML_TAG_FILTER,' ')
            .gsub(EMAIL_FILTER,'')
            .gsub(URL_FILTER,'')
            .gsub(GENDER_FILTER,'')
            .gsub(NEW_LINES,'')
            .gsub(SYMBOL_FILTER,' ')
            .gsub(WHITESPACE,' ')
            .gsub(WORDS_IN_BRACKETS, '\1')
            .gsub(CODE_TOKEN_FILTER,'')
            .downcase
            .strip
        )
    end
62157
private static String stripHtml(Content body) {
        String rawString = body.toString();
        // remove HTML tags
        rawString = rawString.replaceAll("\\<.*?>", " ");
        // consolidate multiple spaces between a word to a single space
        rawString = rawString.replaceAll("\\b\\s{2,}\\b", " ");
        // remove extra whitespaces
        return rawString.trim();
    }
1270904
def clean_html(self, html):
        """"""
        result_type = type(html)
        if isinstance(html, six.string_types):
            doc = html_fromstring(html)
        else:
            doc = copy.deepcopy(html)
        self(doc)
        if issubclass(result_type, six.binary_type):
            return tostring(doc, encoding='utf-8')
        elif issubclass(result_type, six.text_type):
            return tostring(doc, encoding='unicode')
        else:
            return doc
1099131
function unEscapeHtmlTags (text) {
  return text
    .replace(ampRegex, '&')
    .replace(ltRegex, '<')
    .replace(gtRegex, '>')
    .replace(quotRegex, '"')
    .replace(singleQuotRegex, '\'')
    .replace(singleQuotRegexAlt, '\'')
}
1194752
private static void clean(Element e, String tag) {
        Elements targetList = getElementsByTag(e, tag);
        boolean isEmbed = "object".equalsIgnoreCase(tag)
                       || "embed".equalsIgnoreCase(tag)
                       || "iframe".equalsIgnoreCase(tag);

        for (Element target : targetList) {
            Matcher matcher = Patterns.get(Patterns.RegEx.VIDEO).matcher(
                    target.outerHtml());
            if (isEmbed && matcher.find()) {
                continue;
            }
            target.remove();
        }
    }
854673
function cleanConditionally(e, tag) {
  var tagsList = e.getElementsByTagName(tag);
  var curTagsLength = tagsList.length;

  /**
   * Gather counts for other typical elements embedded within.
   * Traverse backwards so we can remove nodes at the same time without effecting the traversal.
   *
   * TODO: Consider taking into account original contentScore here.
   **/
  for (var i = curTagsLength - 1; i >= 0; i--) {
    var weight = getClassWeight(tagsList[i]);

    dbg("Cleaning Conditionally " + tagsList[i] + " (" + tagsList[i].className + ":" + tagsList[i].id + ")" + ((typeof tagsList[i].readability != 'undefined') ? (" with score " + tagsList[i].readability.contentScore) : ''));

    if (weight < 0) {
      tagsList[i].parentNode.removeChild(tagsList[i]);
    } else if (getCharCount(tagsList[i], ',') < 10) {
      /**
       * If there are not very many commas, and the number of
       * non-paragraph elements is more than paragraphs or other ominous signs, remove the element.
       **/

      var p = tagsList[i].getElementsByTagName("p").length;
      var img = tagsList[i].getElementsByTagName("img").length;
      var li = tagsList[i].getElementsByTagName("li").length - 100;
      var input = tagsList[i].getElementsByTagName("input").length;

      var embedCount = 0;
      var embeds = tagsList[i].getElementsByTagName("embed");
      for (var ei = 0, il = embeds.length; ei < il; ei++) {
        if (embeds[ei].src && embeds[ei].src.search(regexps.videoRe) == -1) {
          embedCount++;
        }
      }

      var linkDensity = getLinkDensity(tagsList[i]);
      var contentLength = getInnerText(tagsList[i]).length;
      var toRemove = false;

      if (img > p && img > 1) {
        toRemove = true;
      } else if (li > p && tag != "ul" && tag != "ol") {
        toRemove = true;
      } else if (input > Math.floor(p / 3)) {
        toRemove = true;
      } else if (contentLength < 25 && (img == 0 || img > 2)) {
        toRemove = true;
      } else if (weight < 25 && linkDensity > .2) {
        toRemove = true;
      } else if (weight >= 25 && linkDensity > .5) {
        toRemove = true;
      } else if ((embedCount == 1 && contentLength < 75) || embedCount > 1) {
        toRemove = true;
      }

      if (toRemove) {
        tagsList[i].parentNode.removeChild(tagsList[i]);
      }
    }
  }
}
1363516
def clean(text):
    



    text = bold_italic.sub(r'\1', text)
    text = bold.sub(r'\1', text)
    text = italic_quote.sub(r'"\1"', text)
    text = italic.sub(r'"\1"', text)
    text = quote_quote.sub(r'"\1"', text)
    # residuals of unbalanced quotes
    text = text.replace("'''", '').replace("''", '"')
    text = newlines.sub(r'\n', text)
    text = bulletlist.sub(r'', text)

    # Collect spans

    spans = []
    # Drop HTML comments
    for m in comment.finditer(text):
            spans.append((m.start(), m.end()))

    # Drop self-closing tags
    for pattern in selfClosing_tag_patterns:
        for m in pattern.finditer(text):
            spans.append((m.start(), m.end()))

    # Drop ignored tags
    for left, right in ignored_tag_patterns:
        for m in left.finditer(text):
            spans.append((m.start(), m.end()))
        for m in right.finditer(text):
            spans.append((m.start(), m.end()))

    # Bulk remove all spans
    text = dropSpans(spans, text)

    # Drop discarded elements
    for tag in discardElements:
        text = dropNested(text, r'<\s*%s\b[^>/]*>' % tag, r'<\s*/\s*%s>' % tag)

    # Expand placeholders
    for pattern, placeholder in placeholder_tag_patterns:
        index = 1
        for match in pattern.finditer(text):
            text = text.replace(match.group(), '%s_%d' % (placeholder, index))
            index += 1

    text = text.replace('<<', u'Â«').replace('>>', u'Â»')

    #############################################

    # Cleanup text
    text = text.replace('\t', ' ')
    text = spaces.sub(' ', text)
    text = dots.sub('...', text)
    text = re.sub(u' (,:\.\)\]Â»)', r'\1', text)
    text = re.sub(u'(\[\(Â«) ', r'\1', text)
    text = re.sub(r'\n\W+?\n', '\n', text, flags=re.U) # lines with only punctuations
    text = text.replace(',,', ',').replace(',.', '.')

    return text
565205
def clean_text(s):
    """"""
    SPACES_RE = re.compile(r'\s+')
    SPECIAL_CHARS_RE = re.compile(r'[^\w\s\.\-\(\)]')
    s = SPACES_RE.sub(' ', s)
    s = s.strip()
    s = SPECIAL_CHARS_RE.sub('', s)
    return s
1136431
def removeHtmlTags(self, text):
		""""""
		sb = []
		text = self.removeHtmlComments(text)
		bits = text.split(u'<')
		sb.append(bits.pop(0))
		tagstack = []
		tablestack = tagstack
		for x in bits:
			m = _tagPattern.match(x)
			if not m:
				continue
			slash, t, params, brace, rest = m.groups()
			t = t.lower()
			badtag = False
			if t in _htmlelements:
				# Check our stack
				if slash:
					# Closing a tag...
					if t in _htmlsingleonly or len(tagstack) == 0:
						badtag = True
					else:
						ot = tagstack.pop()
						if ot != t:
							if ot in _htmlsingleallowed:
								# Pop all elements with an optional close tag
								# and see if we find a match below them
								optstack = []
								optstack.append(ot)
								while True:
									if len(tagstack) == 0:
										break
									ot = tagstack.pop()
									if ot == t or ot not in _htmlsingleallowed:
										break
									optstack.append(ot)
								if t != ot:
									# No match. Push the optinal elements back again
									badtag = True
									tagstack += reversed(optstack)
							else:
								tagstack.append(ot)
								# <li> can be nested in <ul> or <ol>, skip those cases:
								if ot not in _htmllist and t in _listtags:
									badtag = True
						elif t == u'table':
							if len(tablestack) == 0:
								bagtag = True
							else:
								tagstack = tablestack.pop()
					newparams = u''
				else:
					# Keep track for later
					if t in _tabletags and u'table' not in tagstack:
						badtag = True
					elif t in tagstack and t not in _htmlnest:
						badtag = True
					# Is it a self-closed htmlpair? (bug 5487)
					elif brace == u'/>' and t in _htmlpairs:
						badTag = True
					elif t in _htmlsingleonly:
						# Hack to force empty tag for uncloseable elements
						brace = u'/>'
					elif t in _htmlsingle:
						# Hack to not close $htmlsingle tags
						brace = None
					else:
						if t == u'table':
							tablestack.append(tagstack)
							tagstack = []
						tagstack.append(t)
					newparams = self.fixTagAttributes(params, t)
				if not badtag:
					rest = rest.replace(u'>', u'&gt;')
					if brace == u'/>':
						close = u' /'
					else:
						close = u''
					sb.append(u'<')
					sb.append(slash)
					sb.append(t)
					sb.append(newparams)
					sb.append(close)
					sb.append(u'>')
					sb.append(rest)
					continue
			sb.append(u'&lt;')
			sb.append(x.replace(u'>', u'&gt;'))

		# Close off any remaining tags
		while tagstack:
			t = tagstack.pop()
			sb.append(u'</')
			sb.append(t)
			sb.append(u'>\n')
			if t == u'table':
				if not tablestack:
					break
				tagstack = tablestack.pop()

		return u''.join(sb)
608653
def strip_html(html)
      return @stripped_html if @stripped_html
      allowed = @settings[:allowed_tags]
      reg = if allowed.any?
        Regexp.new(
          %(<(?!(\\s|\\/)*(#{
            allowed.map {|tag| Regexp.escape( tag )}.join( "|" )
          })( |>|\\/|'|"|<|\\s*\\z))[^>]*(>+|\\s*\\z)),
          Regexp::IGNORECASE | Regexp::MULTILINE, 'u'
        )
      else
        /<[^>]*(>+|\s*\z)/m
      end
       @stripped_html = html.gsub(reg,'')
    end
694915
private function cleanupDOMDocument()
    {
        $expressions = array(
            '//comment()',
            '//processing-instruction()',
        );

        foreach ($expressions as $expression) {
            foreach ($this->xpath->query($expression) as $node) {
                $node->parentNode->removeChild($node);
            }
        }

        $this->dom->normalizeDocument();
    }
244300
def clean_text(self, text):
        ''''''
        if text is None:
            return ''
        text = re.sub(ILLEGAL_CHARACTERS_RE, '', text)
        if '<' in text or '&lt' in text:
            text = clean(text, tags=self.tags, strip=self.strip)

        return unescape(text)
1231009
def clean_link(self, url):
        """"""
        return self._clean_re.sub(lambda match: "%%%2x" % ord(match.group(0)), url)
1158236
def clean_strict(text, boundary=WS):
    """"""
    # transliterate to ascii
    text = ascii_text(text)
    # replace punctuation and symbols
    text = CHARACTERS_REMOVE_RE.sub('', text)
    text = category_replace(text)
    # pad out for company type replacements
    text = ''.join((boundary, collapse_spaces(text), boundary))
    return text
539343
function trimHtml(html) {
      function trimSpaces(all, s1, s2) {
        // WebKit &nbsp; meant to preserve multiple spaces but instead inserted around all inline tags,
        // including the spans with inline styles created on paste
        if (!s1 && !s2) {
          return ' ';
        }

        return '\u00a0';
      }

      html = filter(html, [
        /^[\s\S]*<body[^>]*>\s*|\s*<\/body[^>]*>[\s\S]*$/ig, // Remove anything but the contents within the BODY element
        /<!--StartFragment-->|<!--EndFragment-->/g, // Inner fragments (tables from excel on mac)
        [/( ?)<span class="Apple-converted-space">\u00a0<\/span>( ?)/g, trimSpaces],
        /<br class="Apple-interchange-newline">/g,
        /<br>$/i // Trailing BR elements
      ]);

      return html;
    }
267695
def html_clean(html_code):
    
    temp = re.sub('<script([\s\S]*?)</script>', '', html_code)
    temp = re.sub('<style([\s\S]*?)</style>', '', temp)
    html_cleaned = re.sub('(?is)<.*?>', '', temp)
    for item in html_character:
        html_cleaned = html_cleaned.replace(item, html_character[item])

    log('debug', '网页源代码预处理完成：\n【{}】'.format(html_cleaned))
    return html_cleaned