Extract news data from a feed item
427917
def extract_data item, text_limit: 1200
      link = item.link
      desc = item.description
      title = item.title.force_encoding('UTF-8')

      # Extract main image
      case
      when desc.match(/\|.*\|/m)
        img, terms, text = desc.split('|')
      when desc.include?('|')
        img, text = desc.split('|')
        terms = nil
      when desc.match(/&lt;img.*?src="(.*?)\"/)
        img = desc.match(/&lt;img.*?src="(.*?)\"/)[1]
        text = desc
        terms = nil
      else
        img, terms, text = nil, nil, desc
      end

      # Extract categories
      categories =
      if terms
        terms.split(/term_group: .*\n/).
          select {|g| g.match /taxonomy: category/}.
          map {|c| c.match( /name: (.*)\n/)[1]}
      else
        []
      end

      # Crop text
      short_text = strip_tags(text)[0..text_limit]

      # Normalize newlines
      short_text = short_text.gsub(/\A\s+/,"").gsub(/\n+/,"\n\n")

      # Right-strip to end of last paragraph
      short_text = short_text[0..short_text.rindex(/\.\n/)]

      [title, link, img, short_text, categories]
    end
171617
def get_feed_posts(self) -> Iterator[Post]:
        

        data = self.context.graphql_query("d6f4427fbe92d846298cf93df0b937d3", {})["data"]

        while True:
            feed = data["user"]["edge_web_feed_timeline"]
            yield from (Post(self.context, edge["node"]) for edge in feed["edges"]
                        if not edge["node"]["__typename"] == "GraphSuggestedUserFeedUnit")
            if not feed["page_info"]["has_next_page"]:
                break
            data = self.context.graphql_query("d6f4427fbe92d846298cf93df0b937d3",
                                              {'fetch_media_item_count': 12,
                                               'fetch_media_item_cursor': feed["page_info"]["end_cursor"],
                                               'fetch_comment_count': 4,
                                               'fetch_like': 10,
                                               'has_stories': False})["data"]
337531
def __item_data(self, model, index):
        """"""
        item = model.items[index]

        data = {
            "name": item.name,
            "data": item.data,
            "doc": getattr(item, "doc", None),
            "path": getattr(item, "path", None),
        }

        return data
555977
def _parseMessage(self, content):
        """"""
        self._seq = content.get("seq", "0")

        if "lb_info" in content:
            self._sticky = content["lb_info"]["sticky"]
            self._pool = content["lb_info"]["pool"]

        if "batches" in content:
            for batch in content["batches"]:
                self._parseMessage(batch)

        if "ms" not in content:
            return

        for m in content["ms"]:
            mtype = m.get("type")
            try:
                # Things that directly change chat
                if mtype == "delta":
                    self._parseDelta(m)
                # Inbox
                elif mtype == "inbox":
                    self.onInbox(
                        unseen=m["unseen"],
                        unread=m["unread"],
                        recent_unread=m["recent_unread"],
                        msg=m,
                    )

                # Typing
                elif mtype == "typ" or mtype == "ttyp":
                    author_id = str(m.get("from"))
                    thread_id = m.get("thread_fbid")
                    if thread_id:
                        thread_type = ThreadType.GROUP
                        thread_id = str(thread_id)
                    else:
                        thread_type = ThreadType.USER
                        if author_id == self._uid:
                            thread_id = m.get("to")
                        else:
                            thread_id = author_id
                    typing_status = TypingStatus(m.get("st"))
                    self.onTyping(
                        author_id=author_id,
                        status=typing_status,
                        thread_id=thread_id,
                        thread_type=thread_type,
                        msg=m,
                    )

                # Delivered

                # Seen
                # elif mtype == "m_read_receipt":
                #
                #     self.onSeen(m.get('realtime_viewer_fbid'), m.get('reader'), m.get('time'))

                elif mtype in ["jewel_requests_add"]:
                    from_id = m["from"]
                    self.onFriendRequest(from_id=from_id, msg=m)

                # Happens on every login
                elif mtype == "qprimer":
                    self.onQprimer(ts=m.get("made"), msg=m)

                # Is sent before any other message
                elif mtype == "deltaflow":
                    pass

                # Chat timestamp
                elif mtype == "chatproxy-presence":
                    statuses = dict()
                    for id_, data in m.get("buddyList", {}).items():
                        statuses[id_] = ActiveStatus._from_chatproxy_presence(id_, data)
                        self._buddylist[id_] = statuses[id_]

                    self.onChatTimestamp(buddylist=statuses, msg=m)

                # Buddylist overlay
                elif mtype == "buddylist_overlay":
                    statuses = dict()
                    for id_, data in m.get("overlay", {}).items():
                        old_in_game = None
                        if id_ in self._buddylist:
                            old_in_game = self._buddylist[id_].in_game

                        statuses[id_] = ActiveStatus._from_buddylist_overlay(
                            data, old_in_game
                        )
                        self._buddylist[id_] = statuses[id_]

                    self.onBuddylistOverlay(statuses=statuses, msg=m)

                # Unknown message type
                else:
                    self.onUnknownMesssageType(msg=m)

            except Exception as e:
                self.onMessageError(exception=e, msg=m)
281091
def _create_feed_dict(self, data):
        
        return {
            self.input_data: data,
            self.hrand: np.random.rand(data.shape[0], self.num_hidden),
            self.vrand: np.random.rand(data.shape[0], data.shape[1])
        }
443829
public static function parse( $feed, $url ) {
		$items     = array();
		$rss_items = $feed->get_items();
		$title     = $feed->get_title();
		foreach ( $rss_items as $item ) {
			$items[] = self::get_item( $item, $title );
		}
		return array_filter(
			array(
				'type'       => 'feed',
				'_feed_type' => self::get_type( $feed ),
				'summary'    => $feed->get_description(),
				'author'     => self::get_author( $feed->get_author() ),
				'name'       => $title,
				'url'        => $feed->get_permalink(),
				'photo'      => $feed->get_image_url(),
				'items'      => $items,
			)
		);
	}
784858
def get_newsitem_model(model_string):
    
    try:
        NewsItem = apps.get_model(model_string)
        assert issubclass(NewsItem, AbstractNewsItem)
    except (ValueError, LookupError, AssertionError):
        raise ValueError('Invalid news item model string'.format(model_string))
    return NewsItem
1305572
def get_box_newsfeeds(self, box_key, detail_level = None):
		'''
		'''
		uri = '/'.join([
						self.api_uri,
						self.boxes_suffix,
						box_key,
						self.newsfeed_suffix
						])
		return self._get_newsfeeds(uri, detail_level)
591198
def news_item(name, options)
      after_global_configure do |config|
        config.news.item(name, options) if config.news
      end
    end
1111960
def get_news_content(self, url):
        
        res = WyuNews.__wyu_news_content(url)
        soup = BeautifulSoup(res, from_encoding='utf-8')
        tag_table = soup.find(self.__get_tag_table)

        tr = tag_table.findAll('tr')[4]

        return tr.decode()
626668
def extract_feed(
    inpath: str, outpath: str, view: View, config: nx.DiGraph = None
) -> str:
    """"""
    config = default_config() if config is None else config
    config = remove_node_attributes(config, "converters")
    feed = load_feed(inpath, view, config)
    return write_feed_dangerously(feed, outpath)
254521
function processFeed(feed) {

	var feedName = ('nitro-'+feed.name).toCamelCase();
	var s = '/**\n';
	s += '* '+feed.title+'\n';
	s += '*/\n';
	fs.appendFileSync(apijs, s+'const '+feedName+" = '"+feed.href+"';\n", 'utf8');
	cache.push(feedName+' : '+feedName+',\n');

	var pathname = feed.href.replace('/nitro/api','');
	var path = openapi.paths[pathname] = {};
	path.get = {};
	path.get.description = feed.title;
	path.get.tags = ['feeds'];
	path.get.summary = feed.title;

	// a long time ago, in a galaxy far, far away, the API had longer descriptions. Some taken from https://developer.bbc.co.uk/content/
	if (feed.name == 'Programmes') {
		path.get.description = 'Fetch metadata about Programmes (brands, series, episodes, clips). By applying different filter restrictions this feed can be used in many ways, for example to retrieve all series belonging to a brand, all the episodes and/or clips for a specific series, or any TLEO objects for a masterbrand. Other filters permit restricting to specific formats and/or genres, and you can request specific versions (for example Signed or Audio-Described). Parameters may be combined in any way suitable for your application.';
	}
	else if (feed.name == 'Broadcasts') {
		path.get.description = 'Fetch metadata about linear Broadcasts and Services, allowing the generation of Television and Radio schedules and other datasets for broadcast items. Use /schedules instead of this feed as it is more efficient. Broadcasts will be deprecated in the future.';
	}
	else if (feed.name == 'Schedules') {
		path.get.description = 'Dates, Times, Schedules: when and where are programmes being shown?';
	}
	else if (feed.name == 'Versions') {
		path.get.description = 'The versions feed exposes editorial "Versions" of programmes. These are concepts used to capture different presentations of an overall programme: for example, versions of a programme may include one with sign language, one with audio description, one edited for content and more. Versions are also important to understand for broadcasts: a linear broadcast or an ondemand is always of a specific version, not merely of a programme.';
	}
	else if (feed.name == 'Services') {
		path.get.description = 'The services feed exposes the linear broadcast "services" from PIPs. These are the actual services which broadcast programmes (eg bbc_one_oxford is the service for BBC One in Oxford).';
	}
	else if (feed.name == 'People') {
		path.get.description = 'The People feed allows you to search for the people and groups that contribute to programmes. This is the starting point for cast and crew credits, as well as finding contributors using external IDs (such as Wikipedia URLs)';
	}
	else if (feed.name == 'Availabilities') {
		path.get.description = 'For advanced users only: Use the availabilities feed to discover details of on-demand availability for programmes and their versions.';
	}
	else if (feed.name == 'Images') {
		path.get.description = 'Find metadata for images, particularly those in galleries';
	}
	else if (feed.name == 'Promotions') {
		path.get.description = 'Details of short-term editorially curated "promotions", for instance those programmes featured on iPlayer today';
	}
	else if (feed.name == 'Groups') {
		path.get.description = 'Long-lived curated collections of programmes and more, including Collections, Seasons, Franchises and Galleries';
	}

	path.get.operationId = 'list'+feed.name;
	params = path.get.parameters = [];

	path.get.responses = {};
	path.get.responses['200'] = {};
	path.get.responses['200'].description = 'Nitro response';
	path.get.responses['200'].content = {};
	path.get.responses['200'].content['application/json'] = {};
	path.get.responses['200'].content['application/json'].schema = {};
	path.get.responses['200'].content['application/json'].schema.$ref = '#/components/schemas/nitro';
	path.get.responses['200'].content['application/xml'] = clone(path.get.responses['200'].content['application/json']);

	path.get.responses.default = {};
	path.get.responses.default.description = 'Unexpected error';
	path.get.responses.default.content = {};
	path.get.responses.default.content['application/json'] = {};
	path.get.responses.default.content['application/json'].schema = {};
	path.get.responses.default.content['application/json'].schema.$ref = '#/components/schemas/ErrorModel';
	path.get.responses.default.content['application/xml'] = clone(path.get.responses.default.content['application/json']);

	if (feed.release_status == 'deprecated') {
		path.get.deprecated = true;
		console.log('! Warning, feed '+feed.name+' is '+feed.release_status+deprecationInfo(feed,feed.name,'feed',feed.deprecated_since))
	}

	if (feed.sorts) {
		feed.sorts.sort = toArray(feed.sorts.sort);
		if (feed.sorts.sort instanceof Array) {
			for (var i in feed.sorts.sort) {
				var sort = feed.sorts.sort[i];
				var sortName = ('s-'+feed.name+'-'+sort.name).toCamelCase();
				processSort(feed,sort,sortName);
			}
		}
	}

	if (feed.mixins) {
		feed.mixins.mixin = toArray(feed.mixins.mixin);
		for (var i in feed.mixins.mixin) {
			var mixin = feed.mixins.mixin[i];
			var mixinName = ('m-'+feed.name+'-'+mixin.name).toCamelCase();
			processMixin(feed,mixin,mixinName,true);
		}
		if (feed.mixins.unstable_mixins) {
			feed.mixins.unstable_mixins.mixin = toArray(feed.mixins.unstable_mixins.mixin);
			for (var i in feed.mixins.unstable_mixins.mixin) {
				var mixin = feed.mixins.unstable_mixins.mixin[i];
				var mixinName = ('m-'+feed.name+'-'+mixin.name).toCamelCase();
				processMixin(feed,mixin,mixinName,false);
			}
		}
	}

	if (feed.filters) {
		feed.filters.filter = toArray(feed.filters.filter);
		for (var i in feed.filters.filter) {
			var filter = feed.filters.filter[i];
			var filterName = ('f-'+feed.name+'-'+filter.name).toCamelCase();
			processFilter(feed,filter,filterName);
		}
	}

	additionalParameters(feed.name);

}
1361313
def extract(self, text: str) -> List[Extraction]:

        
        
        content = BeautifulSoup(text, 'html5lib')
        subject = content.find('h1').text.strip()
        recip = self.mailing_list_name
        
        navbar = content.find(id='navbar')
        if navbar == None:
            info = self.old_format(content)
        else:
            info = self.new_format(navbar, content)
        for i in info[0:3]:
            if i == 'None':
                print('missed something important')
        sender = info[0]
        date   = info[1]
        body   = info[2]
        nxt    = info[3]
        rep_to = info[4]
        pub = 'SeeSat_Obs'
        dRec = datetime.datetime.now().isoformat()
        
        msg_obj = { 
            'url' : self.email_url,
            '@context' : {
                '@vocab' : 'schema.org'
            },
            'subject' : subject,
            'recip' : recip,
            'sender' : sender
        }
        if date != 'None':
            msg_obj['dateReceived'] = date
        if body != 'None':
            msg_obj['body'] = body
        if nxt != 'None':
            msg_obj['nxt'] = nxt
        if rep_to != 'None':
            msg_obj['replyToMessage'] = rep_to
        return Extraction(value=msg_obj, extractor_name=self.name)
1306776
def get_articles(self, issue=''):
        	
		
		
	soup = get_soup() # get soup of all articles 
	issues = soup.find_all('ul')
		
	# validating and assigning default value for issue
	if not type(issue) is int or issue < 0 :
		issue = 1
	if issue > len(issues):
		issue = len(issues)		
		
	# considering latest article is last element 		
	articles = issues[len(issues)-issue].find_all('a') 
	mArticles = []
	for article in articles:
		mArticle = {}
		mArticle['link'] = article.get('href')[1:]
		mArticle['title'] = article.find('li').contents[0].strip()
		mArticle['author'] = article.find('span').contents[0].encode('utf8')
		mArticles.append(mArticle)
	return mArticles
1281683
def feed(date = nil)
      date = Time.now if date.nil?
      @client.request('getProcessedUserFeedData',
                      userID: @id,
                      day: date.strftime('%Y%m%d'))
    end
587869
def from_url(url, timeout=None):
        
        articles = NewsPlease.from_urls([url], timeout=timeout)
        if url in articles.keys():
            return articles[url]
        else:
            return None
1183986
function extractContent(desc) {
  /*
  // search all possible matches, returning first find
  var founds = sites.findMatch(desc.uri);

  if (founds.length) {
    var $ = cheerio.load(desc.content);
  // use first found extractor
    founds.forEach(function(found) {
      if ($(found.selector)) {
        var content = $(found.selector).html();
        return { selector: found.selector, content: content };
      }
    });
  }
  // ok, let's try unfluff
  var uf = unfluff(desc.content);
  return { selector: 'body', content: desc.content, text: uf.text, title: uf.title };
  */
  var title = getTitle(desc.content, '<title') || getTitle(desc.content, '<TITLE');

  return { selector: 'body', content: desc.content, text: 'NOTEXTRACTED', title: title };
}