Given a text message, extract all the URLs found in the message, along
    with their surrounding context.  The output is a list of sequences of Chunk
    objects, corresponding to the contextual regions extracted from the string.
496370
def extracturls(mesg):
    
    lines = NLRE.split(mesg)

    # The number of lines of context above to provide.
    # above_context = 1
    # The number of lines of context below to provide.
    # below_context = 1

    # Plan here is to first transform lines into the form
    # [line_fragments] where each fragment is a chunk as
    # seen by parse_text_urls.  Note that this means that
    # lines with more than one entry or one entry that's
    # a URL are the only lines containing URLs.

    linechunks = [parse_text_urls(l) for l in lines]

    return extract_with_context(linechunks,
                                lambda chunk: len(chunk) > 1 or
                                (len(chunk) == 1 and chunk[0].url is not None),
                                1, 1)
1361313
def extract(self, text: str) -> List[Extraction]:

        
        
        content = BeautifulSoup(text, 'html5lib')
        subject = content.find('h1').text.strip()
        recip = self.mailing_list_name
        
        navbar = content.find(id='navbar')
        if navbar == None:
            info = self.old_format(content)
        else:
            info = self.new_format(navbar, content)
        for i in info[0:3]:
            if i == 'None':
                print('missed something important')
        sender = info[0]
        date   = info[1]
        body   = info[2]
        nxt    = info[3]
        rep_to = info[4]
        pub = 'SeeSat_Obs'
        dRec = datetime.datetime.now().isoformat()
        
        msg_obj = { 
            'url' : self.email_url,
            '@context' : {
                '@vocab' : 'schema.org'
            },
            'subject' : subject,
            'recip' : recip,
            'sender' : sender
        }
        if date != 'None':
            msg_obj['dateReceived'] = date
        if body != 'None':
            msg_obj['body'] = body
        if nxt != 'None':
            msg_obj['nxt'] = nxt
        if rep_to != 'None':
            msg_obj['replyToMessage'] = rep_to
        return Extraction(value=msg_obj, extractor_name=self.name)
291500
def parse(text):
    
    ret = Docstring()
    if not text:
        return ret

    text = inspect.cleandoc(text)
    match = re.search('^:', text, flags=re.M)
    if match:
        desc_chunk = text[:match.start()]
        meta_chunk = text[match.start():]
    else:
        desc_chunk = text
        meta_chunk = ''

    parts = desc_chunk.split('\n', 1)
    ret.short_description = parts[0] or None
    if len(parts) > 1:
        long_desc_chunk = parts[1] or ''
        ret.blank_after_short_description = long_desc_chunk.startswith('\n')
        ret.blank_after_long_description = long_desc_chunk.endswith('\n\n')
        ret.long_description = long_desc_chunk.strip() or None

    for match in re.finditer(
            r'(^:.*?)(?=^:|\Z)', meta_chunk, flags=re.S | re.M
    ):
        chunk = match.group(0)
        if not chunk:
            continue
        try:
            args_chunk, desc_chunk = chunk.lstrip(':').split(':', 1)
        except ValueError:
            raise ParseError(
                'Error parsing meta information near "{}".'.format(chunk)
            )
        args = args_chunk.split()
        desc = desc_chunk.strip()
        if '\n' in desc:
            first_line, rest = desc.split('\n', 1)
            desc = first_line + '\n' + inspect.cleandoc(rest)
        ret.meta.append(DocstringMeta(args, description=desc, type=None))

    return ret
664272
async def _parse_message_text(self, message, parse_mode):
        
        if parse_mode is ():
            parse_mode = self._parse_mode
        else:
            parse_mode = utils.sanitize_parse_mode(parse_mode)

        if not parse_mode:
            return message, []

        message, msg_entities = parse_mode.parse(message)
        for i in reversed(range(len(msg_entities))):
            e = msg_entities[i]
            if isinstance(e, types.MessageEntityTextUrl):
                m = re.match(r'^@|\+|tg://user\?id=(\d+)', e.url)
                if m:
                    user = int(m.group(1)) if m.group(1) else e.url
                    is_mention = await self._replace_with_mention(msg_entities, i, user)
                    if not is_mention:
                        del msg_entities[i]
            elif isinstance(e, (types.MessageEntityMentionName,
                                types.InputMessageEntityMentionName)):
                is_mention = await self._replace_with_mention(msg_entities, i, e.user_id)
                if not is_mention:
                    del msg_entities[i]

        return message, msg_entities
584405
private List<List<Term>> seg2sentence(String text) {
        List<String> sentenceList = toSentenceList(text);
        List<List<Term>> resultList = new ArrayList<List<Term>>(sentenceList.size());
        for (String sentence : sentenceList) {
            resultList.add(analysis.parseStr(sentence).recognition(sr).getTerms());
        }
        return resultList;
    }
1053359
@Benchmark
  @BenchmarkMode(Mode.SampleTime)
  @OutputTimeUnit(TimeUnit.NANOSECONDS)
  public SpanContext extract(Data data) throws SpanContextParseException {
    return data.textFormatBase.extract(data.spanContextHeaders);
  }
282176
public static <T> String get(Message<T> text, T context) {
        for (AlternativeUiTextProvider p : all()) {
            String s = p.getText(text, context);
            if (s!=null)
                return s;
        }
        return null;
    }
681094
private List<String> extractWords(@NotNull String text, Language language) {

        List<String> words = textUtils.extractWords(text, language);

        List<String> wordList = words.stream()
                                     .flatMap(word -> {
                                         return textUtils.getNormalForms(word.toLowerCase(), language)
                                                         .stream()
                                                         .filter(element -> textUtils.isNormalWord(element, language));
                                     })
                                     .collect(Collectors.toList());

        return wordList;
    }
1233053
function(text, root, callback) {
      var matches = this.extractUrls(text, root);

      // every call to process returns all the text this loader has ever received
      var done = callback.bind(null, this.map);
      this.fetch(matches, done);
    }
135687
def complete_use(self, text, *_):
        
        return [t + " " for t in REGIONS if t.startswith(text)]
1129214
def parse(self, text):
        '''
        '''
        results = []
        if isinstance(text, str):
            if six.PY2:
                text = unicode(text, 'utf-8')
        self.clean_text = self._normalize_string(text)

        # get addresses
        addresses = set(self._get_addresses(self.clean_text))
        if addresses:
            # append parsed address info
            results = list(map(self._parse_address, addresses))

        return results
701036
def extract(self, text: str) -> List[Extraction]:
        

        doc = self._tokenizer.tokenize_to_spacy_doc(text)
        self._load_matcher()

        matches = [x for x in self._matcher(doc) if x[1] != x[2]]
        pos_filtered_matches = []
        neg_filtered_matches = []
        for idx, start, end in matches:
            span_doc = self._tokenizer.tokenize_to_spacy_doc(doc[start:end].text)
            this_spacy_rule = self._matcher.get(idx)
            relations = self._find_relation(span_doc, this_spacy_rule)
            rule_id, _ = self._hash_map[idx]
            this_rule = self._rule_lst[rule_id]
            if self._filter_match(doc[start:end], relations, this_rule.patterns):
                value = self._form_output(doc[start:end], this_rule.output_format, relations, this_rule.patterns)
                if this_rule.polarity:
                    pos_filtered_matches.append((start, end, value, rule_id, relations))
                else:
                    neg_filtered_matches.append((start, end, value, rule_id, relations))

        return_lst = []
        if pos_filtered_matches:
            longest_lst_pos = self._get_longest(pos_filtered_matches)
            if neg_filtered_matches:
                longest_lst_neg = self._get_longest(neg_filtered_matches)
                return_lst = self._reject_neg(longest_lst_pos, longest_lst_neg)
            else:
                return_lst = longest_lst_pos

        extractions = []
        for (start, end, value, rule_id, relation) in return_lst:
            this_extraction = Extraction(value=value,
                                         extractor_name=self.name,
                                         start_token=start,
                                         end_token=end,
                                         start_char=doc[start].idx,
                                         end_char=doc[end-1].idx+len(doc[end-1]),
                                         rule_id=rule_id.split("rule_id##")[0],
                                         match_mapping=relation)
            extractions.append(this_extraction)

        return extractions
782796
def Main(url):
    '''
    
    '''
    # The object of Web-Scraping.
    web_scrape = WebScraping()
    # Execute Web-Scraping.
    document = web_scrape.scrape(url)
    # The object of NLP.
    nlp_base = NlpBase()
    # Set tokenizer. This is japanese tokenizer with MeCab.
    nlp_base.tokenizable_doc = MeCabTokenizer()

    sentence_list = nlp_base.listup_sentence(document)

    batch_size = 10
    if len(sentence_list) < batch_size:
        raise ValueError("The number of extracted sentences is insufficient.")

    all_token_list = []
    for i in range(len(sentence_list)):
        nlp_base.tokenize(sentence_list[i])
        all_token_list.extend(nlp_base.token)
        sentence_list[i] = nlp_base.token
    
    vectorlizable_sentence = LSTMRTRBM()
    vectorlizable_sentence.learn(
        sentence_list=sentence_list, 
        token_master_list=list(set(all_token_list)),
        hidden_neuron_count=1000,
        batch_size=batch_size,
        learning_rate=1e-03,
        seq_len=5
    )
    test_list = sentence_list[:batch_size]
    feature_points_arr = vectorlizable_sentence.vectorize(test_list)

    print("Feature points (Top 5 sentences):")
    print(feature_points_arr)
696273
protected String extractTextBetweenTags(String stringContainingText, String tagname) {

    String openTag = "<" + tagname.toUpperCase() + ">";
    String closeTag = "</" + tagname.toUpperCase() + ">";
    StringBuilder buf = new StringBuilder();
    boolean record = false;
    for (int i = 0; i < stringContainingText.length() - openTag.length(); i++) {
      if (stringContainingText.substring(i, i + closeTag.length()).equalsIgnoreCase(closeTag)) {
        record = false;
        break;
      }

      if (record) {
        buf.append(stringContainingText.charAt(i));
      }
      if (stringContainingText.substring(i, i + openTag.length()).equalsIgnoreCase(openTag)) {
        buf = new StringBuilder();
        i += openTag.length() - 1;
        record = true;
      }

    }

    return buf.toString().trim();
  }
1143857
def _extract_links_from_asset_tags_in_text(self, text):
        
        # Extract asset tags from instructions text
        asset_tags_map = self._extract_asset_tags(text)
        ids = list(iterkeys(asset_tags_map))
        if not ids:
            return {}

        # asset tags contain asset names and ids. We need to make another
        # HTTP request to get asset URL.
        asset_urls = self._extract_asset_urls(ids)

        supplement_links = {}

        # Build supplement links, providing nice titles along the way
        for asset in asset_urls:
            title = clean_filename(
                asset_tags_map[asset['id']]['name'],
                self._unrestricted_filenames)
            extension = clean_filename(
                asset_tags_map[asset['id']]['extension'].strip(),
                self._unrestricted_filenames)
            url = asset['url'].strip()
            if extension not in supplement_links:
                supplement_links[extension] = []
            supplement_links[extension].append((url, title))

        return supplement_links
16919
function extractToText(strings)
			{
				var $htmlContent, $htmlPattern, $htmlReplace;

				// first remove to unnecessary gaps
				$htmlContent = strings.replace(/\n/gim,'').replace(/\r/gim,'').replace(/\t/gim,'').replace(/&nbsp;/gim,' ');

				$htmlPattern =  [
					/\<span(|\s+.*?)><span(|\s+.*?)>(.*?)<\/span><\/span>/gim, // trim nested spans
					/<(\w*[^p])\s*[^\/>]*>\s*<\/\1>/gim, // remove empty or white-spaces tags (ignore paragraphs (<p>) and breaks (<br>))
					/\<div(|\s+.*?)>(.*?)\<\/div>/gim, // convert div to p
					/\<strong(|\s+.*?)>(.*?)\<\/strong>/gim, // convert strong to b
					/\<em(|\s+.*?)>(.*?)\<\/em>/gim // convert em to i
				];

				$htmlReplace = [
					'<span$2>$3</span>',
					'',
					'<p$1>$2</p>',
					'<b$1>$2</b>',
					'<i$1>$2</i>'
				];
				
				// repeat the cleaning process 5 times
				for(c=0; c<5; c++)
				{
					// create loop as the number of pattern
					for(var i = 0; i < $htmlPattern.length; i++)
					{
						$htmlContent = $htmlContent.replace($htmlPattern[i], $htmlReplace[i]);
					}
				}

				// if paragraph is false (<p>), convert <p> to <br>
				if(!vars.p)
					$htmlContent = $htmlContent.replace(/\<p(|\s+.*?)>(.*?)\<\/p>/ig, '<br/>$2');

				// if break is false (<br>), convert <br> to <p>
				if(!vars.br)
				{
					$htmlPattern =  [
						/\<br>(.*?)/ig,
						/\<br\/>(.*?)/ig
					];

					$htmlReplace = [
						'<p>$1</p>',
						'<p>$1</p>'
					];

					// create loop as the number of pattern (for breaks)
					for (var i = 0; i < $htmlPattern.length; i++) {
						$htmlContent = $htmlContent.replace($htmlPattern[i], $htmlReplace[i]);
					}
				}

				// if paragraph and break is false (<p> && <br>), convert <p> to <div>
				if(!vars.p && !vars.br)
					$htmlContent = $htmlContent.replace(/\<p>(.*?)\<\/p>/ig, '<div>$1</div>');

				return $htmlContent;
			}
565770
def extract_emails(text):
    """"""
    text = text.replace(u'\u2024', '.')
    emails = []
    for m in EMAIL_RE.findall(text):
        emails.append(m[0])
    return emails