Get the relations between the each pattern in the spacy rule and the matches
        Args:
            span_doc: doc
            r: List

        Returns: Dict
546386
def _find_relation(self, span_doc: doc, r: List) -> Dict:
        

        rule = r[1][0]
        span_pivot = 0
        relation = {}
        for e_id, element in enumerate(rule):
            if not span_doc[span_pivot:]:
                for extra_id, _, in enumerate(rule[e_id:]):
                    relation[e_id+extra_id] = None
                break
            new_doc = self._tokenizer.tokenize_to_spacy_doc(span_doc[span_pivot:].text)
            if "OP" not in element:
                relation[e_id] = (span_pivot, span_pivot+1)
                span_pivot += 1
            else:
                if e_id < len(rule)-1:
                    tmp_rule_1 = [rule[e_id]]
                    tmp_rule_2 = [rule[e_id+1]]
                    tmp_matcher = Matcher(self._nlp.vocab)
                    tmp_matcher.add(0, None, tmp_rule_1)
                    tmp_matcher.add(1, None, tmp_rule_2)
                    tmp_matches = sorted([x for x in tmp_matcher(new_doc) if x[1] != x[2]], key=lambda a: a[1])

                    if not tmp_matches:
                        relation[e_id] = None
                    else:
                        matches_1 = [x for x in tmp_matches if x[0] == 0 and x[1] == 0]
                        if not matches_1:
                            relation[e_id] = None
                        else:
                            _, s1, e1 = matches_1[0]
                            matches_2 = [x for x in tmp_matches if x[0] == 1]
                            if not matches_2:
                                relation[e_id] = (span_pivot, span_pivot + e1)
                                span_pivot += e1
                            else:
                                _, s2, e2 = matches_2[0]
                                if e1 <= s2:
                                    relation[e_id] = (span_pivot, span_pivot + e1)
                                    span_pivot += e1
                                else:
                                    relation[e_id] = (span_pivot, span_pivot + s2)
                                    span_pivot += s2
                else:
                    relation[e_id] = (span_pivot, len(span_doc))

        return relation
953878
def extract_object_names_from_docs(filename):
    """"""
    text = split_code_and_text_blocks(filename)[1]
    text = '\n'.join(t[1] for t in text if t[0] == 'text')
    regex = re.compile(r':(?:'
                       r'func(?:tion)?|'
                       r'meth(?:od)?|'
                       r'attr(?:ibute)?|'
                       r'obj(?:ect)?|'
                       r'class):`(\S*)`'
                       )
    return [(x, x) for x in re.findall(regex, text)]
361928
def get_visual_aligned_lemmas(mention):
    
    spans = _to_spans(mention)
    for span in spans:
        sentence = span.sentence
        doc = sentence.document
        # cache features for the entire document
        _preprocess_visual_features(doc)

        for aligned_lemma in sentence._aligned_lemmas:
            yield aligned_lemma
311722
def _docs(self):
        
        iterdocs = iter(self.docs())
        first = next(iterdocs)
        needs_parsing = False
        if isinstance(first, six.string_types):
            needs_parsing = True
        elif isinstance(first, dict):
            pass
        else:
            raise RuntimeError('Document must be either JSON strings or dict.')
        for doc in itertools.chain([first], iterdocs):
            if needs_parsing:
                doc = json.loads(doc)
            if '_index' not in doc:
                doc['_index'] = self.index
            if '_type' not in doc:
                doc['_type'] = self.doc_type
            yield doc
582998
def _get_span(self, m):
        
        return (m.sentence.id, m.char_start, m.char_end)
1097111
def get_relations(self, cursor, table_name):
        
        # pylint:disable=global-statement,too-many-locals,too-many-nested-blocks,unused-argument
        def table2model(table_name):
            return SfProtectName(table_name).title().replace(' ', '').replace('-', '')
        global last_introspected_model, last_read_only, last_refs
        global last_with_important_related_name  # pylint:disable=invalid-name
        result = {}
        reverse = {}
        last_with_important_related_name = []
        last_read_only = {}
        last_refs = {}
        for _, field in enumerate(self.table_description_cache(table_name)['fields']):
            if field['type'] == 'reference' and field['referenceTo']:
                reference_to_name = SfProtectName(field['referenceTo'][0])
                relationship_order = field['relationshipOrder']
                if relationship_order is None:
                    relationship_tmp = set()
                    for rel in field['referenceTo']:
                        for chld in self.table_description_cache(rel)['childRelationships']:
                            if chld['childSObject'] == table_name and chld['field'] == field['name']:
                                relationship_tmp.add(chld['cascadeDelete'])
                    assert len(relationship_tmp) <= 1
                    if True in relationship_tmp:
                        relationship_order = '*'
                last_refs[field['name']] = (field['referenceTo'], relationship_order)
                result[field['name']] = ('Id', reference_to_name)
                reverse.setdefault(reference_to_name, []).append(field['name'])
                if not field['updateable'] or not field['createable']:
                    sf_read_only = (0 if field['updateable'] else 1) | (0 if field['createable'] else 2)
                    # use symbolic names NOT_UPDATEABLE, NON_CREATABLE, READ_ONLY instead of 1, 2, 3
                    last_read_only[field['name']] = reverse_models_names[sf_read_only]
        for ref, ilist in reverse.items():
            # Example of back_collision: a class Aaa has a ForeignKey to a class
            # Bbb and the class Bbb has any field with the name 'aaa'.
            back_name_collisions = [
                x['name'] for x in self.table_description_cache(ref)['fields']
                if re.sub('Id$' if x['type'] == 'reference' else '', '',
                          re.sub('__c$', '', x['name'])
                          ).replace('_', '').lower() == table2model(table_name).lower()]
            # add `related_name` only if necessary
            if len(ilist) > 1 or back_name_collisions:
                last_with_important_related_name.extend(ilist)
        last_introspected_model = table2model(table_name)
        return result
1135284
def overlap_tokens(doc, other_doc):
    
    overlap = []
    other_tokens = [token.text for token in other_doc]
    for token in doc:
        if token.text in other_tokens:
            overlap.append(token)
    return overlap
867341
def _get_tokens_for_line_func(self, cli, document):
        
        # Cache using `document.text`.
        def get_tokens_for_line():
            return self.lexer.lex_document(cli, document)

        return self._token_cache.get(document.text, get_tokens_for_line)
379959
def _docstring_getblocks(self):
        
        #If there are no lines to look at, we have nothing to do here.
        if self.ibuffer[0] == self.ibuffer[1]:
            return []

        lines = self.context.bufferstr[self.ibuffer[0]:self.ibuffer[1]]
        docblock = []
        result = []
        self._doclines = []

        #We need to keep track of the line number for the start of the
        #documentation strings.
        docline = 0
        doclength = 0

        first = self.docparser.RE_DOCS.match(lines[0])
        if first is not None:
            docblock.append(first.group("docstring"))
            docline = self.ibuffer[0]
            self._doclines.append(docline)
            doclength += len(lines[0]) + 1 # + 1 for \n removed by split.

            #We need to search backwards in the main buffer string for
            #additional tags to add to the block
            i = self.ibuffer[0] - 1
            while i > 0:
                current = self.context.bufferstr[i]
                docmatch = self.docparser.RE_DOCS.match(current)
                if docmatch is not None:
                    docblock.append(docmatch.group("docstring"))
                    docline = i
                    doclength += len(current) + 1
                else:
                    break
                i -= 1

        #Reverse the docblock list since we were going backwards and appending.
        if len(docblock) > 0:
            docblock.reverse()

        #Now handle the lines following the first line. Also handle the
        #possibility of multiple, separate blocks that are still valid XML.
        #We have to keep going until we have exceed the operational changes
        #or found the decorating element.
        i = self.ibuffer[0] + 1
        while (i < len(self.context.bufferstr) and 
               (i < self.ibuffer[1] or len(docblock) > 0)):
            line = self.context.bufferstr[i]
            docmatch = self.docparser.RE_DOCS.match(line)
            if docmatch is not None:
                docblock.append(docmatch.group("docstring"))
                doclength += len(line)
                if docline == 0:
                    docline = i
                #Only track actual documentation lines that are within the 
                #operations list of lines.
                if i < self.ibuffer[1]:
                    self._doclines.append(i)

            elif len(docblock) > 0:
                key = self._docstring_key(line)
                result.append((docblock, docline, doclength, key))
                docblock = []
                docline = 0
                doclength = 0

            #We need to exit the loop if we have exceeded the length of
            #the operational changes
            if len(docblock) == 0 and i > self.ibuffer[1]:
                break
            i += 1

        return result
901742
def rintersects(self, span):
        
        if isinstance(span, list):
            return [sp for sp in span if self._rintersects(sp)]

        return self._rintersects(span)
348593
def get_arg_descriptions_from_docstring(cls, obj):
    """"""

    ret = OrderedDict()
    name = ''
    doc = obj.__doc__ or ''
    lines = [s.strip() for s in doc.split('\n')]
    stanza_first_line_re = cls._get_stanza_first_line_re()
    for line in lines:
      m = stanza_first_line_re.match(line)
      if m and m.group(1) == 'param':
        # If first line of a parameter description, set name and description.
        name, description = m.group(3, 4)
        ret[name] = description
      elif m and m.group(1) != 'param':
        # If first line of a description of an item other than a parameter, clear name.
        name = ''
      elif name and line:
        # If subsequent line of a parameter description, add to existing description (if any) for
        # that parameter.
        ret[name] += (' ' + line) if ret[name] else line
      # Ignore subsequent lines of descriptions of items other than parameters.
    return ret
305735
def tokenize(s):
    
    return [m.group(m.lastindex) for m in _tdl_re.finditer(s)]
20043
def get_relation_by_type_list(parser, token):
    
    tokens = token.contents.split()
    if len(tokens) not in (6, 7):
        raise template.TemplateSyntaxError(
            "%r tag requires 6 arguments" % tokens[0]
        )

    if tokens[2] != 'for':
        raise template.TemplateSyntaxError(
            "Third argument in %r tag must be 'for'" % tokens[0]
        )

    if tokens[4] != 'as':
        raise template.TemplateSyntaxError(
            "Fifth argument in %r tag must be 'as'" % tokens[0]
        )

    direction = 'forward'
    if len(tokens) == 7:
        direction = tokens[6]

    return RelationByTypeListNode(
        name=tokens[1], obj=tokens[3], as_var=tokens[5], direction=direction
    )
1200388
def at_span(self, span, predicate=None, index=None):
        
        starting = self._index_dict[span[0]]
        ending = self._index_dict[span[1] - 1]

        merged = list(starting)
        for marker in ending:
            if marker not in merged:
                merged.append(marker)

        return filter_index(merged, predicate, index)
1121399
def list_relations(self):
        '''  '''
        _ = self._execute('select * from relations').fetchall()
        for i in _:
            #print(i)
            src, name, dst = i
            src = self.deserialize(
                next(self._execute('select code from objects where id=?',(src,)))[0]
            )
            dst = self.deserialize(
                next(self._execute('select code from objects where id=?',(dst,)))[0]
            )
            yield src, name, dst
1292907
def _nodes_to_regs(self):
        
        def get_tuples():
            for r, re_match in self._re_matches:
                for group_name, group_index in r.groupindex.items():
                    if group_name != _INVALID_TRAILING_INPUT:
                        reg = re_match.regs[group_index]
                        node = self._group_names_to_nodes[group_name]
                        yield (node, reg)

        return list(get_tuples())
388046
def process_document(self, doc):
        

        raw = doc.select_segments("$.raw_content")[0]
        extractions = doc.extract(self.inferlink_extractor, raw)
        doc.store(extractions, "inferlink_extraction")
        return list()