Parse content into a document fragment.
487356
function parseContent(window, input) {
    function nextToken() {
      // Check for end-of-string.
      if (!input) {
        return null;
      }

      // Consume 'n' characters from the input.
      function consume(result) {
        input = input.substr(result.length);
        return result;
      }

      var m = input.match(/^([^<]*)(<[^>]+>?)?/);
      // If there is some text before the next tag, return it, otherwise return
      // the tag.
      return consume(m[1] ? m[1] : m[2]);
    }

    // Unescape a string 's'.
    function unescape1(e) {
      return ESCAPE[e];
    }
    function unescape(s) {
      while ((m = s.match(/&(amp|lt|gt|lrm|rlm|nbsp);/))) {
        s = s.replace(m[0], unescape1);
      }
      return s;
    }

    function shouldAdd(current, element) {
      return !NEEDS_PARENT[element.localName] ||
             NEEDS_PARENT[element.localName] === current.localName;
    }

    // Create an element for this tag.
    function createElement(type, annotation) {
      var tagName = TAG_NAME[type];
      if (!tagName) {
        return null;
      }
      var element = window.document.createElement(tagName);
      element.localName = tagName;
      var name = TAG_ANNOTATION[type];
      if (name && annotation) {
        element[name] = annotation.trim();
      }
      return element;
    }

    var rootDiv = window.document.createElement("div"),
        current = rootDiv,
        t,
        tagStack = [];

    while ((t = nextToken()) !== null) {
      if (t[0] === '<') {
        if (t[1] === "/") {
          // If the closing tag matches, move back up to the parent node.
          if (tagStack.length &&
              tagStack[tagStack.length - 1] === t.substr(2).replace(">", "")) {
            tagStack.pop();
            current = current.parentNode;
          }
          // Otherwise just ignore the end tag.
          continue;
        }
        var ts = parseTimeStamp(t.substr(1, t.length - 2));
        var node;
        if (ts) {
          // Timestamps are lead nodes as well.
          node = window.document.createProcessingInstruction("timestamp", ts);
          current.appendChild(node);
          continue;
        }
        var m = t.match(/^<([^.\s/0-9>]+)(\.[^\s\\>]+)?([^>\\]+)?(\\?)>?$/);
        // If we can't parse the tag, skip to the next tag.
        if (!m) {
          continue;
        }
        // Try to construct an element, and ignore the tag if we couldn't.
        node = createElement(m[1], m[3]);
        if (!node) {
          continue;
        }
        // Determine if the tag should be added based on the context of where it
        // is placed in the cuetext.
        if (!shouldAdd(current, node)) {
          continue;
        }
        // Set the class list (as a list of classes, separated by space).
        if (m[2]) {
          node.className = m[2].substr(1).replace('.', ' ');
        }
        // Append the node to the current node, and enter the scope of the new
        // node.
        tagStack.push(m[1]);
        current.appendChild(node);
        current = node;
        continue;
      }

      // Text nodes are leaf nodes.
      current.appendChild(window.document.createTextNode(unescape(t)));
    }

    return rootDiv;
  }
989783
function processSingleFragment(fragment, ctx, queryMetadata) {
          // return array fragments (link[0], link[1], ... named fragments) as arrays if arrayFragments are enabled
          if (Array.isArray(fragment)) {
            if (queryMetadata.arrayFragments) {
              return fragment.map(function(subFragment) {
                return processSingleFragment(subFragment, ctx, queryMetadata);
              });
            } else {
              // only return first element of array
              if (fragment.length) {
                return processSingleFragment(fragment[0], ctx, queryMetadata);
              } else {
                // only return one element of empty array, lets say that is null
                return null;
              }
            }
          }

          // initialize Prismic fragment javascript wrapper, unless already initialized
          if (!fragment.asHtml) {
            fragment = Prismic.Fragments.initField(fragment);
          }
          var fragmentObject = {
            json: fragment
          };

          if (!queryMetadata.output || queryMetadata.output.indexOf('html') > -1) {
            fragmentObject.html = fragment.asHtml(ctx.linkResolver, ctx.htmlSerializer);
          }

          if (queryMetadata.output && queryMetadata.output.indexOf('text') > -1) {
            //  https://github.com/prismicio/javascript-kit/issues/104
            if (!(fragment instanceof Prismic.Fragments.SliceZone) && !(fragment instanceof Prismic.Fragments.SliceZone)) {
              fragmentObject.text = fragment.asText(ctx.linkResolver);
            }
          }

          // Add child fragments
          if (fragment instanceof Prismic.Fragments.DocumentLink && fragment.document.data) {
            // from document links
            fragmentObject.children = _.mapObject(fragment.document.data[fragment.type], function(subFragment) {
              if (subFragment.type) {
                return processSingleFragment(subFragment, ctx, queryMetadata);
              } else {
                // some document link subfragments aren't proper fragments
                // for example StructuredText isn't supported
                // we can't really do anything else than return the raw data
                return {
                  json: subFragment
                };
              }
            });
          } else if (fragment instanceof Prismic.Fragments.Group) {
            // from groups
            fragmentObject.children = _.map(fragment.toArray(), function(group) {
              return _.mapObject(group.fragments, function(subFragment) {
                return processSingleFragment(subFragment, ctx, queryMetadata);
              });
            });
          } else if (fragment instanceof Prismic.Fragments.SliceZone) {
            // TODO
            fragmentObject.children = _.map(fragment.value, function(sliceFragment) {
              var subFragmentObject = processSingleFragment(sliceFragment.value, ctx, queryMetadata);
              subFragmentObject.sliceType = sliceFragment.sliceType;
              subFragmentObject.sliceLabel = sliceFragment.label;
              return subFragmentObject;
            });
          }

          // Add fragment url
          if (fragment instanceof Prismic.Fragments.DocumentLink) {
            fragmentObject.url = fragment.url(ctx.linkResolver);
          } else if (fragment instanceof Prismic.Fragments.WebLink) {
            fragmentObject.url = fragment.url();
          } else if (fragment instanceof Prismic.Fragments.FileLink) {
            fragmentObject.url = fragment.url();
          } else if (fragment instanceof Prismic.Fragments.ImageLink) {
            fragmentObject.url = fragment.url();
          } else if (fragment instanceof Prismic.Fragments.Image) {
            fragmentObject.url = fragment.url;
          }

          return fragmentObject;
        }
811935
def parse(string)
      require 'rexml/document' unless defined?(REXML::Document)
      doc = REXML::Document.new(string)
      merge_element!({}, doc.root)
    end
1314715
def _docstring_parse(self, blocks):
        """"""
        result = {}
        for block, docline, doclength, key in blocks:
            doctext = "<doc>{}</doc>".format(" ".join(block))
            try:
                docs = ET.XML(doctext)
                docstart = self.parser.charindex(docline, 0, self.context)
                if not key in result:
                    result[key] = [list(docs), docstart, docstart + doclength]
                else:
                    #If there are docblocks separated by whitespace in the
                    #same element we can't easily keep track of the start and
                    #end character indices anymore.
                    result[key][0].extend(list(docs))
            except ET.ParseError:
                msg.warn(doctext)

        return result
809008
def parse_paragraph(document, par):
    

    paragraph = doc.Paragraph()
    paragraph.document = document

    for elem in par:
        if elem.tag == _name('{{{w}}}pPr'):
            parse_paragraph_properties(document, paragraph, elem)

        if elem.tag == _name('{{{w}}}r'):
            parse_text(document, paragraph, elem)

        if elem.tag == _name('{{{m}}}oMath'):
            _m = doc.Math()
            paragraph.elements.append(_m)

        if elem.tag == _name('{{{m}}}oMathPara'):
            _m = doc.Math()
            paragraph.elements.append(_m)

        if elem.tag == _name('{{{w}}}commentRangeStart'):
            _m = doc.Comment(elem.attrib[_name('{{{w}}}id')], 'start')
            paragraph.elements.append(_m)

        if elem.tag == _name('{{{w}}}commentRangeEnd'):
            _m = doc.Comment(elem.attrib[_name('{{{w}}}id')], 'end')
            paragraph.elements.append(_m)

        if elem.tag == _name('{{{w}}}hyperlink'):
            try:
                t = doc.Link(elem.attrib[_name('{{{r}}}id')])

                parse_text(document, t, elem)

                paragraph.elements.append(t)
            except:
                logger.error('Error with with hyperlink [%s].', str(elem.attrib.items()))

        if elem.tag == _name('{{{w}}}smartTag'):
            parse_smarttag(document, paragraph, elem)

    return paragraph
1272717
def _parse_doc(doc):
    
    lines = doc.split("\n")
    descriptions = list(itertools.takewhile(_checker(_KEYWORDS), lines))

    if len(descriptions) < 3:
        description = lines[0]
    else:
        description = "{0}\n\n{1}".format(
            lines[0], textwrap.dedent("\n".join(descriptions[2:])))

    args = list(itertools.takewhile(
        _checker(_KEYWORDS_OTHERS),
        itertools.dropwhile(_checker(_KEYWORDS_ARGS), lines)))
    argmap = {}
    if len(args) > 1:
        for pair in args[1:]:
            kv = [v.strip() for v in pair.split(":")]
            if len(kv) >= 2:
                argmap[kv[0]] = ":".join(kv[1:])

    return dict(headline=descriptions[0], description=description, args=argmap)
1079370
def process(self, content):
        
        log.debug('processing:\n%s', content)
        self.reset()
        if content.tag is None:
            content.tag = content.value.__class__.__name__
        document = Document()
        self.append(document, content)
        return document.root()
1349130
def parse_content(self, text):
        """"""
        match = re.search(
            self.usage_re_str.format(self.usage_name),
            text,
            flags=(re.DOTALL
                   if self.case_sensitive
                   else (re.DOTALL | re.IGNORECASE)))

        if match is None:
            return

        dic = match.groupdict()
        logger.debug(dic)
        self.raw_content = dic['raw']
        if dic['sep'] in ('\n', '\r\n'):
            self.formal_content = dic['section']
            return

        reallen = len(dic['name'])
        replace = ''.ljust(reallen)
        drop_name = match.expand('%s\\g<sep>\\g<section>' % replace)
        self.formal_content = self.drop_started_empty_lines(drop_name).rstrip()
447475
def parse_doc(doc):
    """"""
    if doc is None:
        return {}

    out = {}

    sections = re.split(r'(\w+)\n\s+-{4,50}', doc)  # obj.__doc__.split('\n\n')
    intro = sections.pop(0)
    if intro:
        content = list(map(str.strip, intro.strip().split('\n\n')))
        if len(content) == 1:
            out['title'] = content[0]
        elif len(content) == 2:
            out['title'], out['abstract'] = content

    for i in range(0, len(sections), 2):
        header, content = sections[i:i + 2]

        if header in ['Notes', 'References']:
            out[header.lower()] = content.replace('\n    ', '\n')
        elif header == 'Parameters':
            pass
        elif header == 'Returns':
            match = re.search(r'xarray\.DataArray\s*(.*)', content)
            if match:
                out['long_name'] = match.groups()[0]

    return out
160503
def process_document(self, doc):
        

        raw = doc.select_segments("$.raw_content")[0]

        doc.store(doc.extract(self.content_extractor, raw, strategy=Strategy.ALL_TEXT), "etk2_text")
        doc.store(doc.extract(self.content_extractor, raw, strategy=Strategy.MAIN_CONTENT_STRICT),
                              "etk2_content_strict")
        doc.store(doc.extract(self.content_extractor, raw, strategy=Strategy.MAIN_CONTENT_RELAXED),
                              "etk2_content_relaxed")
        doc.store(doc.extract(self.metadata_extractor,
                              raw,
                              extract_title=True,
                              extract_meta=True,
                              extract_microdata=True,
                              extract_rdfa=True,
                              ), "etk2_metadata")
        return list()
538327
def tokenize_documents(docs):
    """"""
    sep = '\n\n'
    texts = []
    for doc in docs:
        text = '\n\n'.join(['\n'.join(para[SENTENCES]) for para in doc[PARAGRAPHS]])
        doc[TEXT] = text
        del doc[PARAGRAPHS]
        texts.append(Text(doc))
    return texts
998411
def _parse_text(self, page_text):
        """"""

        # 1 sanitize: remove leading blank lines
        # 2 separate "config text" from content, store content
        # 3 convert config text + \n to obtain Meta, this is the config.

        lines = page_text.split('\n')
        i = 0
        while lines[i].strip() == '':
            i += 1
        if i > 0: # i points to the first non-blank line.  Else, i is 0, there are no leading blank lines
            lines = lines[i:]  #remove leading blank lines

        i = 0
        while lines[i].strip() != '':
            i += 1
            # i points to the first blank line
        cfg_lines = '\n'.join(lines[0:i + 1])  #config lines, plus the empty line

        md = markdown.Markdown(extensions=['meta','fenced_code', 'codehilite'],output_format="html5")
        md.convert(cfg_lines)   # need to trigger the conversion to obtain md.Meta

        self._config = md.Meta
        self._content =  '\n'.join(lines[i+1:])
701972
def _parse(self, text):
        '''
        
        '''
        # ensure unicode first
        text = str(text).strip()

        if not text.startswith(DELIMITER):
            return {}, text

        try:
            _, fm, content = BOUNDARY.split(text, 2)
        except ValueError:
            # if we can't split, bail
            return {}, text
        # loader_class = self.loader_factory(self)
        metadata = yaml.load(fm, Loader=self.loader_class)
        metadata = metadata if (isinstance(metadata, dict)) else {}
        return metadata, content
834699
def process_document(self, doc: Document):
        

        text_segments = doc.select_segments("lexisnexis.doc_description")
        for text_segment in text_segments:
            split_sentences = doc.extract(self.sentence_extractor, text_segment)
            doc.store(split_sentences, 'split_sentences')
        # for t, u in zip(text_to_be_split, units_of_text):
        #     split_sentences = doc.extract(self.sentence_extractor, t)
        #     u.store(split_sentences, "split_sentences")
        return list()
376259
def _parse_content(self, text):
        ''''''
        try:
            return super(OrphanHALNavigator, self)._parse_content(text)
        except exc.UnexpectedlyNotJSON:
            return {}
997410
function processFragment(match) {
    var hpos = match.index;
    var hlen = match[0].length;

    currentFragment._txt += srctext.slice(0, hpos);

    if (pos || hpos) {
      currentFragment = new Fragment();
      fragments.push(currentFragment);
    }

    currentFragment._hdr = noHeaders ? '' : match[0];
    currentFragment._txt = noHeaders ? match[0] : '';

    var label = u.trim(match[1]);
    if (label) { currentFragment._lbl = label; }

    srctext = srctext.slice(hpos + hlen);
    pos += hpos + hlen;
  }
363893
private ContentStream jcrBinaryContent( Document document ) {
        // pickup node properties
        Document props = document.getDocument("properties").getDocument(JcrLexicon.Namespace.URI);

        // extract binary value and content
        Binary value = props.getBinary("data");

        if (value == null) {
            return null;
        }

        byte[] content = value.getBytes();
        String fileName = props.getString("fileName");
        String mimeType = props.getString("mimeType");

        // wrap with input stream
        ByteArrayInputStream bin = new ByteArrayInputStream(content);
        bin.reset();

        // create content stream
        return new ContentStreamImpl(fileName, BigInteger.valueOf(content.length), mimeType, bin);
    }