parses microformats from html
327270
function(callback){
    var options = {},//  utils.clone( this.options ),
        self = this;

    options.baseUrl = self.url;

    self.startedUFParse = new Date();
    //console.log(self.url)

    try
    {
      parser.parseHtml (self.html, options, function(err, data){

        //console.log( JSON.stringify(data) );
        if(data){

          self.profile.hCards = self.getArrayOfContentType(data, 'h-card' );
          self.profile.hResumes = self.getArrayOfContentType(data, 'hresume' );
          self.profile.xfn = self.getArrayOfContentType(data, 'xfn' );

        }
        if(callback){
          callback();
        }
      });
    }
    catch(err)
    {
      self.options.logger.warn('error parsing microformats in page: ' + self.url);
      self.error = err;
      callback();
    }

    self.endedUFParse = new Date();
    self.parseTime = self.endedUFParse.getTime() - self.startedUFParse.getTime();
    self.options.logger.log('time to parse microformats in page: ' + self.parseTime + 'ms - ' + self.url);
  }
537397
def parse_content(content)
      content = content.split(/\r?\n/) if content.is_a?(String)
      return '' if !content || content.empty?
      docstring = String.new("")

      indent = content.first[/^\s*/].length
      last_indent = 0
      orig_indent = 0
      directive = false
      last_line = ""
      tag_name = nil
      tag_buf = []

      (content + ['']).each_with_index do |line, index|
        indent = line[/^\s*/].length
        empty = (line =~ /^\s*$/ ? true : false)
        done = content.size == index

        if tag_name && (((indent < orig_indent && !empty) || done ||
            (indent == 0 && !empty)) || (indent <= last_indent && line =~ META_MATCH))
          buf = tag_buf.join("\n")
          if directive || tag_is_directive?(tag_name)
            directive = create_directive(tag_name, buf)
            if directive
              docstring << parse_content(directive.expanded_text).chomp
            end
          else
            create_tag(tag_name, buf)
          end
          tag_name = nil
          tag_buf = []
          directive = false
          orig_indent = 0
        end

        # Found a meta tag
        if line =~ META_MATCH
          directive = $1
          tag_name = $2
          tag_buf = [($3 || '')]
        elsif tag_name && indent >= orig_indent && !empty
          orig_indent = indent if orig_indent == 0
          # Extra data added to the tag on the next line
          last_empty = last_line =~ /^[ \t]*$/ ? true : false

          tag_buf << '' if last_empty
          tag_buf << line.gsub(/^[ \t]{#{orig_indent}}/, '')
        elsif !tag_name
          # Regular docstring text
          docstring << line
          docstring << "\n"
        end

        last_indent = indent
        last_line = line
      end

      docstring
    end
1089772
function parseData(data, mode) {
    	    var parsed = '';
    	    var parameters = data.split(/\n/);
    	    var regPlaceHolder = /(\{\d+\})/g;
    	    var regRepPlaceHolder = /\{(\d+)\}/g;
    	    var unicodeRE = /(\\u.{4})/ig;
            var i = 0, l = parameters.length;
    	    for (i = 0; i < l; i++) {
    		parameters[i] = parameters[i].replace(/^\s\s*/, '').replace(/\s\s*$/, ''); // trim
    		if (parameters[i].length > 0 && parameters[i].match("^#") != "#") { // skip
    										    // comments
    		    var pair = parameters[i].split('=');
    		    if (pair.length > 0) {
    			/** Process key & value */
    			var name = unescape(pair[0]).replace(/^\s\s*/, '').replace(/\s\s*$/, ''); // trim
    			var value = pair.length == 1 ? "" : pair[1];
    			// process multi-line values
    			while (value.match(/\\$/) == "\\") {
    			    value = value.substring(0, value.length - 1);
    			    value += parameters[++i].replace(/\s\s*$/, ''); // right
    									    // trim
    			}
    			// Put values with embedded '='s back together
    			for (var s = 2; s < pair.length; s++) {
    			    value += '=' + pair[s];
    			}
    			value = value.replace(/^\s\s*/, '').replace(/\s\s*$/, ''); // trim

    			/** Mode: bundle keys in a map */
    			if (mode == 'map' || mode == 'both') {
    			    // handle unicode chars possibly left out
    			    var unicodeMatches = value.match(unicodeRE);
    			    if (unicodeMatches) {
    				for (var u = 0; u < unicodeMatches.length; u++) {
    				    value = value.replace(unicodeMatches[u], unescapeUnicode(unicodeMatches[u]));
    				}
    			    }
    			    // add to map
    			    i18n.map[name] = value;
    			}

    			/* Mode: bundle keys as vars/functions */
    			if (mode == 'vars' || mode == 'both') {
    			    value = value.replace(/"/g, '\\"'); // escape quotation mark

    			    // make sure namespaced key exists (eg, some.key')
    			    checkKeyNamespace(name);

    			    // value with variable substitutions
    			    if (regPlaceHolder.test(value)) {
    				var parts = value.split(regPlaceHolder);
    				// process function args
    				var first = true;
    				var fnArgs = '';
    				var usedArgs = [];
    				for (var p = 0; p < parts.length; p++) {
    				    if (regPlaceHolder.test(parts[p]) && (usedArgs.length === 0 || usedArgs.indexOf(parts[p]) === -1)) {
    					if (!first) {
    					    fnArgs += ',';
    					}
    					fnArgs += parts[p].replace(regRepPlaceHolder, 'v$1');
    					usedArgs.push(parts[p]);
    					first = false;
    				    }
    				}
    				parsed += name + '=function(' + fnArgs + '){';
    				// process function body
    				var fnExpr = '"' + value.replace(regRepPlaceHolder, '"+v$1+"') + '"';
    				parsed += 'return ' + fnExpr + ';' + '};';

    				// simple value
    			    } else {
    				parsed += name + '="' + value + '";';
    			    }
    			} // END: Mode: bundle keys as vars/functions
    		    } // END: if(pair.length > 0)
    		} // END: skip comments
    	    }
    	    //eval(parsed);
            /* TODO: refactor */
    	}
11857
def parse_datetime(s): #source: http://stackoverflow.com/questions/2211362/how-to-parse-xsddatetime-format
    """"""
    m = re.match(r""" ^
        (?P<year>-?[0-9]{4}) - (?P<month>[0-9]{2}) - (?P<day>[0-9]{2})
        T (?P<hour>[0-9]{2}) : (?P<minute>[0-9]{2}) : (?P<second>[0-9]{2})
        (?P<microsecond>\.[0-9]{1,6})?
        (?P<tz>
        Z | (?P<tz_hr>[-+][0-9]{2}) : (?P<tz_min>[0-9]{2})
        )?
        $ """, s, re.X)
    if m is not None:
        values = m.groupdict()
        #if values["tz"] in ("Z", None):
        #    tz = 0
        #else:
        #    tz = int(values["tz_hr"]) * 60 + int(values["tz_min"])
        if values["microsecond"] is None:
            values["microsecond"] = 0
        else:
            values["microsecond"] = values["microsecond"][1:]
            values["microsecond"] += "0" * (6 - len(values["microsecond"]))
        values = dict((k, int(v)) for k, v in values.items() if not k.startswith("tz"))
        try:
            return datetime(**values) # , tz
        except ValueError:
            pass
    return None
1299698
def extract_html_urls(self, html):
        
        p = HTMLParser(tree=treebuilders.getTreeBuilder("dom"))
        dom = p.parse(html)
        urls = []

        for img in dom.getElementsByTagName('img'):
            src = img.getAttribute('src')
            if src:
                urls.append(unquote_utf8(src))

            srcset = img.getAttribute('srcset')
            if srcset:
                urls += self.extract_srcset(srcset)

        for source in dom.getElementsByTagName('source'):
            srcset = source.getAttribute('srcset')
            if srcset:
                urls += self.extract_srcset(srcset)

        for source in dom.getElementsByTagName('a'):
            href = source.getAttribute('href')
            if href:
                urls.append(unquote_utf8(href))

        return urls
208910
public void parseHtmlData(PrintWriter out, String str)
    {
        int iCurrentParsePos = 0;
        int iCurrentOutputPos = 0;
        int iStartTag, iStartVelocity;
        while (true)
        { // Done
        	iStartTag = str.indexOf('<', iCurrentParsePos);
        	iStartVelocity = str.indexOf("${", iCurrentParsePos);
        	if ((iStartTag == -1) && (iStartVelocity == -1))
    			break;	// Done
        	String strTag, strParams, strData;
        	int iNextParam, iEndCloseTag;
    		if ((iStartVelocity == -1) || (iStartVelocity > iStartTag))
    		{	// xml format param
	            iCurrentParsePos = iStartTag + 1;
	            if (iCurrentParsePos == str.length())
	                break;
	            if (str.charAt(iStartTag + 1) == '/') // Don't process end tags
	                continue;
	            int iEndTag = str.indexOf(' ', iStartTag);
	            int iEndParam = str.indexOf('>', iStartTag);
	            if (iEndParam == -1)
	                break;      // poorly formed XML
	            int iNextVelocity = str.indexOf("${", iCurrentParsePos);
	            if (iNextVelocity != -1) if (iNextVelocity < iEndParam)
	                continue;   // This handles the case: <IMG=${image}>   (used internally)
	            iNextParam = str.indexOf('<', iCurrentParsePos);
	            if (iNextParam != -1) if (iNextParam < iEndParam)
	                continue;   // This handles the case: <IMG=<image/>>   (used internally)
	            if (iNextVelocity != -1) if (iNextVelocity < iNextParam)
	            	iNextParam = iNextVelocity;
	            iEndCloseTag = iEndParam;
	            if (str.charAt(iEndParam - 1) == '/')
	                iEndParam = iEndParam - 1;
	            if ((iEndTag == -1) || (iEndTag > iEndParam))
	                iEndTag = iEndParam;
	            strTag = str.substring(iStartTag + 1, iEndTag);
	            strParams = null;
	            if (iEndTag != iEndParam)
	                strParams = str.substring(iEndTag + 1, iEndParam);
	            strData = null;
	            int iStartCloseTag = -1;
	            
	            if (iCurrentOutputPos != iStartTag)
	                out.print(str.substring(iCurrentOutputPos, iStartTag));     // Print everything up to the start of the tag.
	            iCurrentOutputPos = iStartTag;
	            
	            if (str.charAt(iEndCloseTag) != '/')
	            {   // Find the end tag (if start tag doesn't end in '/')
	                iStartCloseTag = str.indexOf("</" + strTag + '>', iEndTag);
	                if (iStartCloseTag > iEndTag)
	                {
	                    iEndCloseTag = iStartCloseTag + strTag.length() + 2;
	                    strData = str.substring(iEndParam + 1, iStartCloseTag);
	                }
	            }
        	}
    		else
    		{	// Velocity format param
    			strParams = null;
    			strData = null;
    			
	            iCurrentParsePos = iStartTag + 2;	// ${
	            if (iCurrentParsePos == str.length())
	                break;
	            iEndCloseTag = str.indexOf('}', iStartTag);
	            if (iEndCloseTag == -1)
	                break;      // poorly formed Velocity tag
    			strTag = str.substring(iStartTag + 2, iEndCloseTag);

	            int iNextVelocity = str.indexOf("${", iCurrentParsePos);
	            iNextParam = str.indexOf('<', iCurrentParsePos);
	            if (iNextVelocity != -1) if (iNextVelocity < iNextParam)
	            	iNextParam = iNextVelocity;
    		}

            // Now, I have the tag, params, and data for this tag
            boolean bProcessed = this.parseHtmlTag(out, strTag, strParams, strData);
            
            if (!bProcessed)
            {
                out.print(str.substring(iCurrentOutputPos, iNextParam));     // It wasn't processed, so output it.
                iCurrentOutputPos = iNextParam;
            }
            else
                iCurrentOutputPos = iEndCloseTag + 1;
            iCurrentParsePos = iCurrentOutputPos;
        }
        iStartTag = str.length();
        if (iCurrentOutputPos != iStartTag)
            out.println(str.substring(iCurrentOutputPos, iStartTag));
    }
641189
def parse_rsc_html(htmlstring):
    """"""
    converted = UnicodeDammit(htmlstring)
    if not converted.unicode_markup:
        raise UnicodeDecodeError('Failed to detect encoding, tried [%s]')
    root = fromstring(htmlstring, parser=HTMLParser(recover=True, encoding=converted.original_encoding))
    # Add p.otherpara tags around orphan text
    newp = None
    for child in root.get_element_by_id('wrapper'):
        if newp is not None:
            if child.tag in BLOCK_ELEMENTS or child.get('id', '').startswith('sect') or child.getnext() is None:
                child.addprevious(newp)
                newp = None
            else:
                newp.append(child)
        if newp is None and child.tag in BLOCK_ELEMENTS and child.tail and child.tail.strip():
            newp = Element('p', **{'class': 'otherpara'})
            newp.text = child.tail
            child.tail = ''
    return root
704954
def decode_html(html):
    
    if isinstance(html, unicode):
        return html

    match = CHARSET_META_TAG_PATTERN.search(html)
    if match:
        declared_encoding = match.group(1).decode("ASCII")
        # proceed unknown encoding as if it wasn't found at all
        with ignored(LookupError):
            return html.decode(declared_encoding, "ignore")

    # try to enforce UTF-8 firstly
    with ignored(UnicodeDecodeError):
        return html.decode("utf8")

    text = TAG_MARK_PATTERN.sub(to_bytes(" "), html)
    diff = text.decode("utf8", "ignore").encode("utf8")
    sizes = len(diff), len(text)

    # 99% of text is UTF-8
    if abs(len(text) - len(diff)) < max(sizes) * 0.01:
        return html.decode("utf8", "ignore")

    # try detect encoding
    encoding = "utf8"
    encoding_detector = chardet.detect(text)
    if encoding_detector["encoding"]:
        encoding = encoding_detector["encoding"]

    return html.decode(encoding, "ignore")
1159938
def parse_html(infile, xpath):
    """"""
    if not isinstance(infile, lh.HtmlElement):
        infile = lh.fromstring(infile)
    infile = infile.xpath(xpath)
    if not infile:
        raise ValueError('XPath {0} returned no results.'.format(xpath))
    return infile
239086
function parse (cstr) {
	var m, parts = [], alpha = 1, space

	if (typeof cstr === 'string') {
		//keyword
		if (names[cstr]) {
			parts = names[cstr].slice()
			space = 'rgb'
		}

		//reserved words
		else if (cstr === 'transparent') {
			alpha = 0
			space = 'rgb'
			parts = [0,0,0]
		}

		//hex
		else if (/^#[A-Fa-f0-9]+$/.test(cstr)) {
			var base = cstr.slice(1)
			var size = base.length
			var isShort = size <= 4
			alpha = 1

			if (isShort) {
				parts = [
					parseInt(base[0] + base[0], 16),
					parseInt(base[1] + base[1], 16),
					parseInt(base[2] + base[2], 16)
				]
				if (size === 4) {
					alpha = parseInt(base[3] + base[3], 16) / 255
				}
			}
			else {
				parts = [
					parseInt(base[0] + base[1], 16),
					parseInt(base[2] + base[3], 16),
					parseInt(base[4] + base[5], 16)
				]
				if (size === 8) {
					alpha = parseInt(base[6] + base[7], 16) / 255
				}
			}

			if (!parts[0]) parts[0] = 0
			if (!parts[1]) parts[1] = 0
			if (!parts[2]) parts[2] = 0

			space = 'rgb'
		}

		//color space
		else if (m = /^((?:rgb|hs[lvb]|hwb|cmyk?|xy[zy]|gray|lab|lchu?v?|[ly]uv|lms)a?)\s*\(([^\)]*)\)/.exec(cstr)) {
			var name = m[1]
			var base = name.replace(/a$/, '')
			space = base
			var size = base === 'cmyk' ? 4 : base === 'gray' ? 1 : 3
			parts = m[2].trim()
				.split(/\s*,\s*/)
				.map(function (x, i) {
					//<percentage>
					if (/%$/.test(x)) {
						//alpha
						if (i === size)	return parseFloat(x) / 100
						//rgb
						if (base === 'rgb') return parseFloat(x) * 255 / 100
						return parseFloat(x)
					}
					//hue
					else if (base[i] === 'h') {
						//<deg>
						if (/deg$/.test(x)) {
							return parseFloat(x)
						}
						//<base-hue>
						else if (baseHues[x] !== undefined) {
							return baseHues[x]
						}
					}
					return parseFloat(x)
				})

			if (name === base) parts.push(1)
			alpha = parts[size] === undefined ? 1 : parts[size]
			parts = parts.slice(0, size)
		}

		//named channels case
		else if (cstr.length > 10 && /[0-9](?:\s|\/)/.test(cstr)) {
			parts = cstr.match(/([0-9]+)/g).map(function (value) {
				return parseFloat(value)
			})

			space = cstr.match(/([a-z])/ig).join('').toLowerCase()
		}
	}

	//numeric case
	else if (!isNaN(cstr)) {
		space = 'rgb'
		parts = [cstr >>> 16, (cstr & 0x00ff00) >>> 8, cstr & 0x0000ff]
	}

	//object case - detects css cases of rgb and hsl
	else if (isObject(cstr)) {
		var r = defined(cstr.r, cstr.red, cstr.R, null)

		if (r !== null) {
			space = 'rgb'
			parts = [
				r,
				defined(cstr.g, cstr.green, cstr.G),
				defined(cstr.b, cstr.blue, cstr.B)
			]
		}
		else {
			space = 'hsl'
			parts = [
				defined(cstr.h, cstr.hue, cstr.H),
				defined(cstr.s, cstr.saturation, cstr.S),
				defined(cstr.l, cstr.lightness, cstr.L, cstr.b, cstr.brightness)
			]
		}

		alpha = defined(cstr.a, cstr.alpha, cstr.opacity, 1)

		if (cstr.opacity != null) alpha /= 100
	}

	//array
	else if (Array.isArray(cstr) || global.ArrayBuffer && ArrayBuffer.isView && ArrayBuffer.isView(cstr)) {
		parts = [cstr[0], cstr[1], cstr[2]]
		space = 'rgb'
		alpha = cstr.length === 4 ? cstr[3] : 1
	}

	return {
		space: space,
		values: parts,
		alpha: alpha
	}
}
596967
public static void parse(SimpleXMLDocHandler doc, InputStream in) throws IOException {
        byte b4[] = new byte[4];
        int count = in.read(b4);
        if (count != 4)
            throw new IOException("Insufficient length.");
        String encoding = getEncodingName(b4);
        String decl = null;
        if (encoding.equals("UTF-8")) {
            StringBuffer sb = new StringBuffer();
            int c;
            while ((c = in.read()) != -1) {
                if (c == '>')
                    break;
                sb.append((char)c);
            }
            decl = sb.toString();
        }
        else if (encoding.equals("CP037")) {
            ByteArrayOutputStream bi = new ByteArrayOutputStream();
            int c;
            while ((c = in.read()) != -1) {
                if (c == 0x6e) // that's '>' in ebcdic
                    break;
                bi.write(c);
            }
            decl = new String(bi.toByteArray(), "CP037");
        }
        if (decl != null) {
            decl = getDeclaredEncoding(decl);
            if (decl != null)
                encoding = decl;
        }
        parse(doc, new InputStreamReader(in, IanaEncodings.getJavaEncoding(encoding)));
    }
298669
function parse (headers) {
  if (headers.slice(0,4).toLowerCase() == 'http') {
    var newlineIndex = headers.indexOf('\r\n');
    headers = headers.slice(newlineIndex + 2);
  }
  return parseHeaders(headers);
}
1153020
function convertHTMLtoJSON(html) {
	var masterData = {
		sections: [],
		menu: [],
		customVars: options.customVars
	};

	var $ = cheerio.load(html);

	// Loop each section and turn into javascript object
	$('div.sg-section-' + sgUniqueIdentifier).each(function (i, elem) {
		var $section = cheerio.load($(this).html());
		var curSectionData = {
			id: '',
			section: '',
			heading: '',
			code: '',
			markup: '',
			comment: ''
		};

		$section('h1.main-section-' + sgUniqueIdentifier).each(function (i2, elem2) {
			curSectionData.section = $section(this).text().replace(/^\s+|\s+$/g, '');
		});

		$section('h1.sub-section-' + sgUniqueIdentifier).each(function (i2, elem2) {
			curSectionData.heading = $section(this).text().replace(/^\s+|\s+$/g, '');
		});

		$section('code.sg-code-' + sgUniqueIdentifier).each(function (i2, elem2) {
			curSectionData.code = $section(this).html().replace(/^\s+|\s+$/g, '');
		});

		curSectionData.markup = hl.highlightAuto(curSectionData.code).value;
		curSectionData.id = makeUrlSafe(curSectionData.section  + '-' + curSectionData.heading);

		$section('h1.main-section-' + sgUniqueIdentifier).remove();
		$section('h1.sub-section-' + sgUniqueIdentifier).remove();
		$section('code.sg-code-' + sgUniqueIdentifier).remove();
		curSectionData.comment = $section.html().replace(/^\s+|\s+$/g, '');

		masterData.sections.push(curSectionData);
	});


	// Sort
	masterData.sections.sort(function (a, b) {
		if (a.section === b.section) {
			if (a.heading > b.heading) {
				return 1;
			}
			if (a.heading < b.heading) {
				return -1;
			}
			return 0;
		} else {
			if (a.section > b.section) {
				return 1;
			}
			if (a.section < b.section) {
				return -1;
			}
			return 0;
		}
	});

	// Create menu object
	var menuObj = {};
	masterData.sections.forEach(function (section) {
		if (!menuObj.hasOwnProperty(section.section)) {
			menuObj[section.section] = [];
		}
		menuObj[section.section].push({id: section.id, name:section.heading});
	});
	var menuArr = [];
	Object.keys(menuObj).forEach(function (key) {
		menuArr.push({name: key, headings: menuObj[key]});
	});
	masterData.menu = menuArr;

	return masterData;
}
821422
def _parse_html(self):
        
        self.parsed_html = soup = BeautifulSoup(self.html, 'html.parser')
        self.scripts = [script['src'] for script in
                        soup.findAll('script', src=True)]
        self.meta = {
            meta['name'].lower():
                meta['content'] for meta in soup.findAll(
                    'meta', attrs=dict(name=True, content=True))
        }
913203
function parse(html) {
  // compatibility with the old signature
  html = arguments[arguments.length - 1];

  var items      = [];
  var parsedHTML = $.load(html);

  parsedHTML('[itemtype]').map(function (i, item) {
    processItemtype(item, items);
  });

  parsedHTML('[itemprop]').map(function (i, item) {
    processItemprop(item, items);
  });

  return exports.returnJson ? JSON.stringify(items) : items;
}
411780
function parseScript(html) {
	var $ = cheerio.load(html);
	$('script').each((i, elem) => {
		var $script = $(elem);
		var $section = $script.parents('section');
		$section.attr('data-script', encodeURIComponent($script.html()));
		$script.remove();
	})
	return $.html();

}
753942
function processHtmlToJson(files, options) {
    options = grunt.util._.defaults(options || {}, {
      separator: grunt.util.linefeed
    });
    var ret = {};

    files.map(function(filepath) {
      if (!grunt.file.isDir(filepath)) {
        var filenames = filepath.split('/');
        ret[filepath]= grunt.file.read(filepath, grunt.file.read);
      }
    });

    return ret;
  }