Get the stream of tweets for the search term and cache to the database
190398
public static function getStream()
    {
        $tweetStore = Store::get('Tweet');

        $consumerKey = Setting::getSetting('twitter', 'consumer_key');
        $consumerSecret = Setting::getSetting('twitter', 'consumer_secret');
        $accessToken = Setting::getSetting('twitter', 'access_token');
        $accessTokenSecret = Setting::getSetting('twitter', 'access_token_secret');
        $streamSearch = Setting::getSetting('twitter', 'stream_search');

        $twitter = new Twitter($consumerKey, $consumerSecret, $accessToken, $accessTokenSecret);

        if (self::canCallAPI()) {
            self::updateLastAPICall();

            $stream = $twitter->search($streamSearch);

            foreach ($stream as $s) {
                if (!$tweetStore->getByTwitterIdForScope($s->id, 'stream')) {
                    $t = new Tweet;
                    $t->setTwitterId($s->id);
                    $t->setText($s->text);
                    $t->setHtmlText(Twitter::clickable($s));
                    $t->setScreenname($s->user->screen_name);
                    $t->setPosted($s->created_at);
                    $t->setCreatedDate(new \DateTime());
                    $t->setScope('stream');
                    $tweetStore->insert($t);
                }
            }
        }
    }
1191486
def search(self, search_phrase, limit=None):
        

        # SQLite FTS can't find terms with `-`, therefore all hyphens replaced with underscore before save.
        # Now to make proper query we need to replace all hyphens in the search phrase.
        # See http://stackoverflow.com/questions/3865733/how-do-i-escape-the-character-in-sqlite-fts3-queries
        search_phrase = search_phrase.replace('-', '_')
        terms = SearchTermParser().parse(search_phrase)
        from_year = terms.pop('from', None)
        to_year = terms.pop('to', None)

        query, query_params = self._make_query_from_terms(terms)

        self._parsed_query = (query, query_params)

        connection = self.backend.library.database.connection

        connection.connection.create_function('rank', 1, _make_rank_func((1., .1, 0, 0)))

        # SQLite FTS implementation does not allow to create indexes on FTS tables.
        # see https://sqlite.org/fts3.html 1.5. Summary, p 1:
        # ... it is not possible to create indices ...
        #
        # So, filter years range here.

        results = connection.execute(query, query_params).fetchall()

        for result in results:
            vid, dataset_vid, score, db_from_year, db_to_year = result
            if from_year and from_year < db_from_year:
                continue
            if to_year and to_year > db_to_year:
                continue
            yield PartitionSearchResult(
                vid=vid, dataset_vid=dataset_vid, score=score)
115464
def tweet_list_handler(request, tweet_list_builder, msg_prefix=""):

    
    # tweet_list_builder is a function that takes a unique identifier and returns a list of things to say
    tweets = tweet_list_builder(request.access_token())
    print (len(tweets), 'tweets found')
    if tweets:
        twitter_cache.initialize_user_queue(user_id=request.access_token(),
                                            queue=tweets)
        text_to_read_out = twitter_cache.user_queue(request.access_token()).read_out_next(MAX_RESPONSE_TWEETS)        
        message = msg_prefix + text_to_read_out + ", say 'next' to hear more, or reply to a tweet by number."
        return alexa.create_response(message=message,
                                     end_session=False)
    else:
        return alexa.create_response(message="Sorry, no tweets found, please try something else", 
                                 end_session=False)
405700
def dump(result):
        """"""
        if isinstance(result, dict):
            # Result is a search result.
            statuses = result['results']
        else:
            # Result is a lookup result.
            statuses = result
        status_str_list = []
        for status in statuses:
            status_str_list.append(textwrap.dedent(u"""
                @{screen_name} -- https://twitter.com/{screen_name}
                {text}
            """).strip().format(
                screen_name=status['actor']['preferredUsername'],
                text=status['body']))
        return u'\n\n'.join(status_str_list)
1350322
function getFieldsFromTweetLike(tweetLike) {
  var fields = fieldsToCheck.map(function(fieldName) {
    return _.get(tweetLike, fieldName);
  });

  arrayFieldsToCheck.forEach(function(fieldDescription) {
    var arr = _.get(tweetLike, fieldDescription.arrayAt);
    if (_.isArray(arr)) {
      arr.forEach(function(element) {
        fields.push(_.get(element, fieldDescription.inEach));
      });
    }
  });

  return fields.filter(function(field) {
    return typeof field === 'string' && field.trim().length !== 0;
  }).map(function(field) {
    return field.toLowerCase();
  });
}
459785
@NonNull
    @Override
    public TweetWithUser mapFromCursor(@NonNull StorIOSQLite storIOSQLite, @NonNull Cursor cursor) {
        final Tweet tweet = Tweet.newTweet(
                cursor.getLong(cursor.getColumnIndexOrThrow(Relations.QUERY_COLUMN_TWEET_ID)),
                cursor.getString(cursor.getColumnIndexOrThrow(Relations.QUERY_COLUMN_TWEET_AUTHOR)),
                cursor.getString(cursor.getColumnIndexOrThrow(Relations.QUERY_COLUMN_TWEET_CONTENT))
        );

        final User user = User.newUser(
                cursor.getLong(cursor.getColumnIndexOrThrow(Relations.QUERY_COLUMN_USER_ID)),
                cursor.getString(cursor.getColumnIndexOrThrow(Relations.QUERY_COLUMN_USER_NICK))
        );

        return new TweetWithUser(tweet, user);
    }
1373942
def stream_info(self):
        
        accum = {}
        for key in self.keys:
            accum[key] = self.database.xinfo_stream(key)
        return accum
1169535
def store_snow_tweets_from_disk_to_mongodb(snow_tweets_folder):
    
    client = pymongo.MongoClient("localhost", 27017)

    db = client["snow_tweet_storage"]
    collection = db["tweets"]

    for tweet in extract_all_snow_tweets_from_disk_generator(snow_tweets_folder):
        collection.insert(tweet)
937412
def focused_on_tweet(request):
    
    slots = request.get_slot_map()
    if "Index" in slots and slots["Index"]:
        index = int(slots['Index'])

    elif "Ordinal" in slots and slots["Index"]:
        parse_ordinal = lambda inp : int("".join([l for l in inp if l in string.digits]))
        index = parse_ordinal(slots['Ordinal'])
    else:
        return False
        
    index = index - 1 # Going from regular notation to CS notation
    user_state = twitter_cache.get_user_state(request.access_token())
    queue = user_state['user_queue'].queue()
    if index < len(queue):
        # Analyze tweet in queue
        tweet_to_analyze = queue[index]
        user_state['focus_tweet'] = tweet_to_analyze
        return index + 1 # Returning to regular notation
        twitter_cache.serialize()
    return False
1142618
def get(self):
        
        self.log.info('starting the ``get`` method')

        # SPLIT THE LIST OF NAMES INTO BATCHES
        self.theseBatches, self.theseBatchParams = self._split_incoming_queries_into_batches(
            sources=self.uplist,
            searchParams=self.searchParams
        )

        # PERFORM NAME QUERIES AGAINST NED
        self._build_api_url_and_download_results()
        self.results, self.headers = self._parse_the_ned_list_results()
        self._output_results()

        self.log.info('completed the ``get`` method')
        return self.results
1119826
function (tweets, search, includeHighlighted) {
        var updated = [], tmp, i = 0;

        if (typeof search == 'string') {
          search = this.format(search);
        }

        for (i = 0; i < tweets.length; i++) {
          if (this.match(tweets[i], search, includeHighlighted)) {
            updated.push(tweets[i]);
          }
        }

        return updated;
      }
613237
def process(self, tup):
        
        status_seq = self.iter_using_shelf(tup.values[2], self.tweet_shelf)
        # This could be more efficient by passing the result from twitter
        # straight through to the producer, instead of deserializing and
        # reserializing json.
        self.producer.produce(json.dumps(status) for status in status_seq)
461831
def tweets_for_user(screen_name, limit=1e10):
    
    qu = Queue()
    p = Thread(target=_tweets_for_user, args=(qu, screen_name, limit))
    p.start()
    p.join(910)
    if p.is_alive():
        sys.stderr.write('no results after 15 minutes for %s. Aborting.' % screen_name)
        return []
    else:
        return qu.get()
617594
def parse_binlinks_from_tweet(tweetdict):
    

    urls = []

    if "user" in tweetdict.keys():
        urls.append(tweetdict["user"]["profile_image_url_https"])
        urls.append(tweetdict["user"]["profile_background_image_url_https"])

    if "extended_entities" in tweetdict.keys():
        urls.extend(parse_extended_entities(tweetdict["extended_entities"]))
    return urls
1223634
def _parse(res, params, n, api, **kwds):
    """"""
    cursor = "cursor" in params
    if not cursor:
        start = params["start"]
    if n == 0:
        return ""
    _json = res.get('search-results', {}).get('entry', [])
    # Download the remaining information in chunks
    while n > 0:
        n -= params["count"]
        if cursor:
            pointer = res['search-results']['cursor'].get('@next')
            params.update({'cursor': pointer})
        else:
            start += params["count"]
            params.update({'start': start})
        res = download(url=URL[api], params=params, accept="json", **kwds).json()
        _json.extend(res.get('search-results', {}).get('entry', []))
    return _json
1041860
def listen_for_dweets_from(thing_name, timeout=900, key=None, session=None):
    
    url = BASE_URL + '/listen/for/dweets/from/{0}'.format(thing_name)
    session = session or requests.Session()
    if key is not None:
        params = {'key': key}
    else:
        params = None

    start = datetime.datetime.utcnow()
    while True:
        request = requests.Request("GET", url, params=params).prepare()
        resp = session.send(request, stream=True, timeout=timeout)
        try:
            for x in _listen_for_dweets_from_response(resp):
                yield x
                _check_stream_timeout(start, timeout)
        except (ChunkedEncodingError, requests.exceptions.ConnectionError, requests.exceptions.ReadTimeout):
            pass
        _check_stream_timeout(start, timeout)
62113
private void findStream(QueryKraken query, 
                          Object []args, 
                          ResultStream<Cursor> result)
  {
    try {
      TableKraken table = query.table();
      TableKelp tableKelp = table.getTableKelp();
      TablePod tablePod = table.getTablePod();

      if (query.isStaticNode()) {
        RowCursor cursor = tableKelp.cursor();

        query.fillKey(cursor, args);

        int hash = query.calculateHash(cursor);

        //ShardPod node = tablePod.getPodNode(hash);

        if (tablePod.getNode(hash).isSelfCopy() || true) {
          query.findStream(result, args);
          return;
        }
        else {
          //tablePod.get(cursor.getKey(), new FindGetResult(result, query, args));
          result.ok();
          return;
        }
      }

      query.findStream(result, args);
    } catch (Exception e) {
      result.fail(e);
    }
  }