Parses the rawtext to extract contents and references.
426358
def parse(self, parser, xml):
        """"""
        #We can only process references if the XML tag has inner-XML
        if xml.text is not None:
            matches = parser.RE_REFS.finditer(xml.text)
            if matches:
                for match in matches:
                    #Handle "special" references to this.name and param.name here.
                    self.references.append(match.group("reference"))

        #We also need to get all the XML attributes into the element
        for key in list(xml.keys()):
            self.attributes[key] = xml.get(key)
394684
def _parse(self) -> None:
        """"""
        self._in_quote = False
        self._in_top_comment = False
        while not self._at_end():
            raw_line = line = self._get_line()
            line = line.replace("from.import ", "from . import ")
            line = line.replace("\t", " ").replace('import*', 'import *')
            line = line.replace(" .import ", " . import ")
            statement_index = self.index
            skip_line = self._skip_line(line)

            if line in self._section_comments and not skip_line:
                if self.import_index == -1:
                    self.import_index = self.index - 1
                continue

            if "isort:imports-" in line and line.startswith("#"):
                section = line.split("isort:imports-")[-1].split()[0].upper()
                self.place_imports[section] = []
                self.import_placements[line] = section

            if ";" in line:
                for part in (part.strip() for part in line.split(";")):
                    if part and not part.startswith("from ") and not part.startswith("import "):
                        skip_line = True

            import_type = self._import_type(line)
            if not import_type or skip_line:
                self.out_lines.append(raw_line)
                continue

            for line in (line.strip() for line in line.split(";")):
                import_type = self._import_type(line)
                if not import_type:
                    self.out_lines.append(line)
                    continue

                if self.import_index == -1:
                    self.import_index = self.index - 1
                nested_comments = {}
                import_string, comments, new_comments = self._strip_comments(line)
                line_parts = [part for part in self._strip_syntax(import_string).strip().split(" ") if part]
                if import_type == "from" and len(line_parts) == 2 and line_parts[1] != "*" and new_comments:
                    nested_comments[line_parts[-1]] = comments[0]

                if "(" in line.split("#")[0] and not self._at_end():
                    while not line.strip().endswith(")") and not self._at_end():
                        line, comments, new_comments = self._strip_comments(self._get_line(), comments)
                        stripped_line = self._strip_syntax(line).strip()
                        if import_type == "from" and stripped_line and " " not in stripped_line and new_comments:
                            nested_comments[stripped_line] = comments[-1]
                        import_string += self.line_separator + line
                else:
                    while line.strip().endswith("\\"):
                        line, comments, new_comments = self._strip_comments(self._get_line(), comments)

                        # Still need to check for parentheses after an escaped line
                        if "(" in line.split("#")[0] and ")" not in line.split("#")[0] and not self._at_end():
                            stripped_line = self._strip_syntax(line).strip()
                            if import_type == "from" and stripped_line and " " not in stripped_line and new_comments:
                                nested_comments[stripped_line] = comments[-1]
                            import_string += self.line_separator + line

                            while not line.strip().endswith(")") and not self._at_end():
                                line, comments, new_comments = self._strip_comments(self._get_line(), comments)
                                stripped_line = self._strip_syntax(line).strip()
                                if import_type == "from" and stripped_line and " " not in stripped_line and new_comments:
                                    nested_comments[stripped_line] = comments[-1]
                                import_string += self.line_separator + line

                        stripped_line = self._strip_syntax(line).strip()
                        if import_type == "from" and stripped_line and " " not in stripped_line and new_comments:
                            nested_comments[stripped_line] = comments[-1]
                        if import_string.strip().endswith(" import") or line.strip().startswith("import "):
                            import_string += self.line_separator + line
                        else:
                            import_string = import_string.rstrip().rstrip("\\") + " " + line.lstrip()

                if import_type == "from":
                    import_string = import_string.replace("import(", "import (")
                    parts = import_string.split(" import ")
                    from_import = parts[0].split(" ")
                    import_string = " import ".join([from_import[0] + " " + "".join(from_import[1:])] + parts[1:])

                imports = [item.replace("{|", "{ ").replace("|}", " }") for item in
                           self._strip_syntax(import_string).split()]
                if "as" in imports and (imports.index('as') + 1) < len(imports):
                    while "as" in imports:
                        index = imports.index('as')
                        if import_type == "from":
                            module = imports[0] + "." + imports[index - 1]
                            self.as_map[module] = imports[index + 1]
                        else:
                            module = imports[index - 1]
                            self.as_map[module] = imports[index + 1]
                        if not self.config['combine_as_imports']:
                            self.comments['straight'][module] = comments
                            comments = []
                        del imports[index:index + 2]
                if import_type == "from":
                    import_from = imports.pop(0)
                    placed_module = self.place_module(import_from)
                    if self.config['verbose']:
                        print("from-type place_module for %s returned %s" % (import_from, placed_module))
                    if placed_module == '':
                        print(
                            "WARNING: could not place module {0} of line {1} --"
                            " Do you need to define a default section?".format(import_from, line)
                        )
                    root = self.imports[placed_module][import_type]
                    for import_name in imports:
                        associated_comment = nested_comments.get(import_name)
                        if associated_comment:
                            self.comments['nested'].setdefault(import_from, {})[import_name] = associated_comment
                            comments.pop(comments.index(associated_comment))
                    if comments:
                        self.comments['from'].setdefault(import_from, []).extend(comments)

                    if len(self.out_lines) > max(self.import_index, self._first_comment_index_end + 1, 1) - 1:
                        last = self.out_lines and self.out_lines[-1].rstrip() or ""
                        while (last.startswith("#") and not last.endswith('"""') and not last.endswith("'''") and
                               'isort:imports-' not in last):
                            self.comments['above']['from'].setdefault(import_from, []).insert(0, self.out_lines.pop(-1))
                            if len(self.out_lines) > max(self.import_index - 1, self._first_comment_index_end + 1, 1) - 1:
                                last = self.out_lines[-1].rstrip()
                            else:
                                last = ""
                        if statement_index - 1 == self.import_index:
                            self.import_index -= len(self.comments['above']['from'].get(import_from, []))

                    if import_from not in root:
                        root[import_from] = OrderedDict()
                    root[import_from].update((module, None) for module in imports)
                else:
                    for module in imports:
                        if comments:
                            self.comments['straight'][module] = comments
                            comments = None

                        if len(self.out_lines) > max(self.import_index, self._first_comment_index_end + 1, 1) - 1:

                            last = self.out_lines and self.out_lines[-1].rstrip() or ""
                            while (last.startswith("#") and not last.endswith('"""') and not last.endswith("'''") and
                                   'isort:imports-' not in last):
                                self.comments['above']['straight'].setdefault(module, []).insert(0,
                                                                                                 self.out_lines.pop(-1))
                                if len(self.out_lines) > 0 and len(self.out_lines) != self._first_comment_index_end:
                                    last = self.out_lines[-1].rstrip()
                                else:
                                    last = ""
                            if self.index - 1 == self.import_index:
                                self.import_index -= len(self.comments['above']['straight'].get(module, []))
                        placed_module = self.place_module(module)
                        if self.config['verbose']:
                            print("else-type place_module for %s returned %s" % (module, placed_module))
                        if placed_module == '':
                            print(
                                "WARNING: could not place module {0} of line {1} --"
                                " Do you need to define a default section?".format(import_from, line)
                            )
                        self.imports[placed_module][import_type][module] = None
1349130
def parse_content(self, text):
        """"""
        match = re.search(
            self.usage_re_str.format(self.usage_name),
            text,
            flags=(re.DOTALL
                   if self.case_sensitive
                   else (re.DOTALL | re.IGNORECASE)))

        if match is None:
            return

        dic = match.groupdict()
        logger.debug(dic)
        self.raw_content = dic['raw']
        if dic['sep'] in ('\n', '\r\n'):
            self.formal_content = dic['section']
            return

        reallen = len(dic['name'])
        replace = ''.ljust(reallen)
        drop_name = match.expand('%s\\g<sep>\\g<section>' % replace)
        self.formal_content = self.drop_started_empty_lines(drop_name).rstrip()
1221506
def parse(self, text):
        ''''''
        in_comment = False
        result = []
        for s in iter(text.splitlines()):
            self.log.debug('Parsing string: {}'.format(s))
            rsc = self.RE_START_COMMENT.match(s)
            if rsc:
                if in_comment:
                    self.log.debug('Double comment start!')
                in_comment = True
                comment = rsc.group(1)
                jdoc = JDOC('', '', comment)
            if in_comment:
                rcb = self.RE_COMMENT_BODY.match(s)
                rp = self.RE_PARAM.match(s)
                rec = self.RE_END_COMMENT.match(s)
                if rp:
                    if rsc:
                        self.log.debug('Comment start and param on same line!')
                    param_name = rp.group(1)
                    param_desc = rp.group(2)
                    if param_name in ['function', 'trigger', 'procedure']:
                        rso = self.RE_SCHEMA_OBJECT.match(param_desc)
                        jdoc.object_type = param_name
                        if rso:
                            jdoc.schema_name = rso.group(1)
                            if jdoc.schema_name:
                                jdoc.schema_name = jdoc.schema_name.strip(".")
                            if not jdoc.schema_name or jdoc.schema_name == "":
                                jdoc.schema_name = self.default_schema
                            jdoc.object_name = rso.group(2)
                    elif param_name in ['return',
                                        'returns',
                                        'returning',
                                        'result']:
                        jdoc.returns = param_desc
                    else:
                        jdoc.add_param(param_name, param_desc)
                    self.log.debug('Param {} = {}'.format(param_name, param_desc))
                if rcb:
                    comment = comment + ' ' + rcb.group(1)
                    jdoc.comment = comment
                if rec:
                    in_comment = False
                    result.append(copy.deepcopy(jdoc))
                    del jdoc
                    self.log.debug('Comment:' + comment)
        return result
1082733
def parse(self):
        
        fh = self._open()
        try:
            gff3Set = Gff3Set(self.fileName)
            for line in fh:
                self.lineNumber += 1
                self._parseLine(gff3Set, line[0:-1])
        finally:
            fh.close()
        gff3Set.linkChildFeaturesToParents()
        return gff3Set
858573
def parse_inline(self, text):
        
        element_list = self._build_inline_element_list()
        return inline_parser.parse(
            text, element_list, fallback=self.inline_elements['RawText']
        )
1213836
@Override
  public void parse(String contents) throws SourceMapParseException {
    SourceMapObject sourceMapObject = SourceMapObjectParser.parse(contents);
    parse(sourceMapObject, null);
  }
67414
def _parse(self, raw, next_cls):
        '''
        
        '''
        if next_cls is None:
            from switchyard.lib.packet import Ethernet
            next_cls = Ethernet

        self._headers = []
        while issubclass(next_cls, PacketHeaderBase):
            packet_header_obj = next_cls()
            raw = packet_header_obj.from_bytes(raw)
            self.add_header(packet_header_obj)
            next_cls = packet_header_obj.next_header_class()
            if next_cls is None:
                break
        if raw:
            self.add_header(RawPacketContents(raw))
569732
def extract(text, default=UNKNOWN):
        """"""
        if not text:
            return default
        found = CALLING_CONVENTION_TYPES.pattern.match(text)
        if found:
            return found.group('cc')

        return default
701972
def _parse(self, text):
        '''
        
        '''
        # ensure unicode first
        text = str(text).strip()

        if not text.startswith(DELIMITER):
            return {}, text

        try:
            _, fm, content = BOUNDARY.split(text, 2)
        except ValueError:
            # if we can't split, bail
            return {}, text
        # loader_class = self.loader_factory(self)
        metadata = yaml.load(fm, Loader=self.loader_class)
        metadata = metadata if (isinstance(metadata, dict)) else {}
        return metadata, content
1221554
def inline_parse(text, verbose=False):
    """"""
    if isinstance(text, str):
        text = text.decode('utf-8')

    parsed = parse(text, verbose=verbose)

    shift = 0
    for quantity in parsed:
        index = quantity.span[1] + shift
        to_add = u' {' + unicode(quantity) + u'}'
        text = text[0:index] + to_add + text[index:]
        shift += len(to_add)

    return text
993265
def refs(soup):
    """"""
    tags = raw_parser.ref_list(soup)
    refs = []
    position = 1

    article_doi = doi(soup)

    for tag in tags:
        ref = {}

        ref['ref'] = ref_text(tag)

        # ref_id
        copy_attribute(tag.attrs, "id", ref)

        # article_title
        if raw_parser.article_title(tag):
            ref['article_title'] = node_text(raw_parser.article_title(tag))
            ref['full_article_title'] = node_contents_str(raw_parser.article_title(tag))

        if raw_parser.pub_id(tag, "pmid"):
            ref['pmid'] = node_contents_str(first(raw_parser.pub_id(tag, "pmid")))

        if raw_parser.pub_id(tag, "isbn"):
            ref['isbn'] = node_contents_str(first(raw_parser.pub_id(tag, "isbn")))

        if raw_parser.pub_id(tag, "doi"):
            ref['reference_id'] = node_contents_str(first(raw_parser.pub_id(tag, "doi")))
            ref['doi'] = doi_uri_to_doi(node_contents_str(first(raw_parser.pub_id(tag, "doi"))))

        uri_tag = None
        if raw_parser.ext_link(tag, "uri"):
            uri_tag = first(raw_parser.ext_link(tag, "uri"))
        elif raw_parser.uri(tag):
            uri_tag = first(raw_parser.uri(tag))
        if uri_tag:
            set_if_value(ref, "uri", uri_tag.get('xlink:href'))
            set_if_value(ref, "uri_text", node_contents_str(uri_tag))
        # look for a pub-id tag if no uri yet
        if not ref.get('uri') and raw_parser.pub_id(tag, "archive"):
            pub_id_tag = first(raw_parser.pub_id(tag, pub_id_type="archive"))
            set_if_value(ref, "uri", pub_id_tag.get('xlink:href'))

        # accession, could be in either of two tags
        set_if_value(ref, "accession", node_contents_str(first(raw_parser.object_id(tag, "art-access-id"))))
        if not ref.get('accession'):
            set_if_value(ref, "accession", node_contents_str(first(raw_parser.pub_id(tag, pub_id_type="accession"))))
        if not ref.get('accession'):
            set_if_value(ref, "accession", node_contents_str(first(raw_parser.pub_id(tag, pub_id_type="archive"))))

        if(raw_parser.year(tag)):
            set_if_value(ref, "year", node_text(raw_parser.year(tag)))
            set_if_value(ref, "year-iso-8601-date", raw_parser.year(tag).get('iso-8601-date'))

        if(raw_parser.date_in_citation(tag)):
            set_if_value(ref, "date-in-citation", node_text(first(raw_parser.date_in_citation(tag))))
            set_if_value(ref, "iso-8601-date", first(raw_parser.date_in_citation(tag)).get('iso-8601-date'))

        if(raw_parser.patent(tag)):
            set_if_value(ref, "patent", node_text(first(raw_parser.patent(tag))))
            set_if_value(ref, "country", first(raw_parser.patent(tag)).get('country'))

        set_if_value(ref, "source", node_text(first(raw_parser.source(tag))))
        set_if_value(ref, "elocation-id", node_text(first(raw_parser.elocation_id(tag))))
        if raw_parser.element_citation(tag):
            copy_attribute(first(raw_parser.element_citation(tag)).attrs, "publication-type", ref)
        if "publication-type" not in ref and raw_parser.mixed_citations(tag):
            copy_attribute(first(raw_parser.mixed_citations(tag)).attrs, "publication-type", ref)

        # authors
        person_group = raw_parser.person_group(tag)
        authors = []

        for group in person_group:

            author_type = None
            if "person-group-type" in group.attrs:
                author_type = group["person-group-type"]

            # Read name or collab tag in the order they are listed
            for name_or_collab_tag in extract_nodes(group, ["name", "string-name", "collab"]):
                author = {}

                # Shared tag attribute
                set_if_value(author, "group-type", author_type)

                # name tag attributes
                if name_or_collab_tag.name in ["name", "string-name"]:
                    set_if_value(author, "surname", node_text(first(raw_parser.surname(name_or_collab_tag))))
                    set_if_value(author, "given-names", node_text(first(raw_parser.given_names(name_or_collab_tag))))
                    set_if_value(author, "suffix", node_text(first(raw_parser.suffix(name_or_collab_tag))))

                # collab tag attribute
                if name_or_collab_tag.name == "collab":
                    set_if_value(author, "collab", node_contents_str(name_or_collab_tag))

                if len(author) > 0:
                    authors.append(author)

            # etal for the person group
            if first(raw_parser.etal(group)):
                author = {}
                author['etal'] = True
                set_if_value(author, "group-type", author_type)
                authors.append(author)

        # Check for collab tag not wrapped in a person-group for backwards compatibility
        if len(person_group) == 0:
            collab_tags = raw_parser.collab(tag)
            for collab_tag in collab_tags:
                author = {}
                set_if_value(author, "group-type", "author")
                set_if_value(author, "collab", node_contents_str(collab_tag))

                if len(author) > 0:
                    authors.append(author)

        if len(authors) > 0:
            ref['authors'] = authors

        set_if_value(ref, "volume", node_text(first(raw_parser.volume(tag))))
        set_if_value(ref, "issue", node_text(first(raw_parser.issue(tag))))
        set_if_value(ref, "fpage", node_text(first(raw_parser.fpage(tag))))
        set_if_value(ref, "lpage", node_text(first(raw_parser.lpage(tag))))
        set_if_value(ref, "collab", node_text(first(raw_parser.collab(tag))))
        set_if_value(ref, "publisher_loc", node_text(first(raw_parser.publisher_loc(tag))))
        set_if_value(ref, "publisher_name", node_text(first(raw_parser.publisher_name(tag))))
        set_if_value(ref, "edition", node_contents_str(first(raw_parser.edition(tag))))
        set_if_value(ref, "version", node_contents_str(first(raw_parser.version(tag))))
        set_if_value(ref, "chapter-title", node_contents_str(first(raw_parser.chapter_title(tag))))
        set_if_value(ref, "comment", node_text(first(raw_parser.comment(tag))))
        set_if_value(ref, "data-title", node_contents_str(first(raw_parser.data_title(tag))))
        set_if_value(ref, "conf-name", node_text(first(raw_parser.conf_name(tag))))

        # If not empty, add position value, append, then increment the position counter
        if(len(ref) > 0):
            ref['article_doi'] = article_doi

            ref['position'] = position

            refs.append(ref)
            position += 1

    return refs
1039436
def parse(contents, tokens=None):
    """"""
    # Shortcut for users who are interested in tokens
    if tokens is None:
        tokens = [t for t in tokenize(contents)]

    token_index, body = _ast_worker(tokens, len(tokens), 0, None)

    assert token_index == len(tokens)
    assert body.arguments == []

    return ToplevelBody(statements=body.statements)
291500
def parse(text):
    
    ret = Docstring()
    if not text:
        return ret

    text = inspect.cleandoc(text)
    match = re.search('^:', text, flags=re.M)
    if match:
        desc_chunk = text[:match.start()]
        meta_chunk = text[match.start():]
    else:
        desc_chunk = text
        meta_chunk = ''

    parts = desc_chunk.split('\n', 1)
    ret.short_description = parts[0] or None
    if len(parts) > 1:
        long_desc_chunk = parts[1] or ''
        ret.blank_after_short_description = long_desc_chunk.startswith('\n')
        ret.blank_after_long_description = long_desc_chunk.endswith('\n\n')
        ret.long_description = long_desc_chunk.strip() or None

    for match in re.finditer(
            r'(^:.*?)(?=^:|\Z)', meta_chunk, flags=re.S | re.M
    ):
        chunk = match.group(0)
        if not chunk:
            continue
        try:
            args_chunk, desc_chunk = chunk.lstrip(':').split(':', 1)
        except ValueError:
            raise ParseError(
                'Error parsing meta information near "{}".'.format(chunk)
            )
        args = args_chunk.split()
        desc = desc_chunk.strip()
        if '\n' in desc:
            first_line, rest = desc.split('\n', 1)
            desc = first_line + '\n' + inspect.cleandoc(rest)
        ret.meta.append(DocstringMeta(args, description=desc, type=None))

    return ret
1001898
def parse(self, contents):
        
        i = 0
        for text in contents.split(self.delim):
            if not len(text.strip()):
                continue
            words = text.split()
            char_offsets = [0] + [
                int(_) for _ in np.cumsum([len(x) + 1 for x in words])[:-1]
            ]
            text = " ".join(words)
            yield {
                "text": text,
                "words": words,
                "pos_tags": [""] * len(words),
                "ner_tags": [""] * len(words),
                "lemmas": [""] * len(words),
                "dep_parents": [0] * len(words),
                "dep_labels": [""] * len(words),
                "char_offsets": char_offsets,
                "abs_char_offsets": char_offsets,
            }
            i += 1
778079
def referencesFinder(text):
    
    references = referencesRegEx.finditer(text)
    count = 0
    refs = []
    spans = []
    for i in references:
        refs.append(i.group())
        spans.append(i.span())
        count += 1
    done = set()
    nameRegEx = re.compile(r"""(name=["']*.*?["']*)(\s|/|>)""")

    for index, obj in enumerate(refs):

        if obj.startswith('<ref name='):
                nameTag = re.escape(nameRegEx.search(obj).group(1))

                if nameTag not in done:
                    nameTag = re.escape(nameRegEx.search(obj).group(1))
                    indeces = [i for i, x in enumerate(refs) if re.search(nameTag, x)]
                    matches = [refs[i] for i in indeces]
                    full = max(matches, key=len)

                    for i in indeces:
                        refs[i] = full

                    done.add(nameTag)

    #eliminate <ref tag or other rudiments from the ref string

    for i in range(len(refs)):
        #print('SIIT', refs[i])
        lastindex = refs[i].rindex('<')
        firstindex = refs[i].index('>')+1
        refs[i]=refs[i][firstindex:lastindex]

    #Handle cite-references

    for i in range(len(refs)):
        if 'cite' in refs[i].lower():
            newText = ''
            values = refs[i].split('|')
            for j in values:
                if '=' in j:
                    first = j.index('=')
                    newText += j[first+1:].strip() + ';'
            refs[i] = newText
    #a ref string:position int dictionary
    refspos = {}
    c = 0
    for i in refs:
        if i not in refspos.keys():
            refspos[i] = c
            c +=1
        else:
            continue

    #print(refspos)

    #eliminate old, bad <ref> tags and insert clean  ones <ref 1..2..3/> to the same spot.
    newText = ''
    assert len(spans) == len(refs)
    #Could happen... havent yet.
    next = 0
    for i in range(len(spans)):
        start = spans[i][0]
        newText+=text[next:start]+'<ref '+str(refspos[refs[i]])+'/>'
        next = spans[i][1]

    newText+=text[next:]

    #switch keys:values in the dictionary for use in sectionsParser
    #positiontag:ref
    newDict  = {y:x for x,y in refspos.items()}

    return newText, newDict
354411
public List<Node> parseWrapper(String raw) {
    // holds literal text as we parse
    StringBuilder buf = new StringBuilder();

    // indicates we're inside a quote.
    boolean inquote = false;

    // indicates we're inside a {x} tag.
    boolean intag = false;

    // length of input pattern
    int length = raw.length();

    // current input pattern index
    int i = 0;

    List<Node> nodes = new ArrayList<>();

    while (i < length) {
      char ch = raw.charAt(i);
      switch (ch) {
        case '{':
          if (buf.length() > 0) {
            nodes.add(new Text(buf.toString()));
            buf.setLength(0);
          }
          intag = true;
          break;

        case '}':
          intag = false;
          break;

        case '\'':
          if (inquote) {
            inquote = false;
          } else {
            inquote = true;
          }
          break;

        default:
          if (intag) {
            nodes.add(new Field(ch, 0));
          } else {
            buf.append(ch);
          }
      }

      i++;
    }

    if (buf.length() > 0) {
      nodes.add(new Text(buf.toString()));
    }
    return nodes;
  }