Parse media file URL:s form tweet data

    :extended_entities_dict:
    :returns: list of media file urls
976250
def parse_extended_entities(extended_entities_dict):
    
    urls = []

    if "media" in extended_entities_dict.keys():
        for item in extended_entities_dict["media"]:

            # add static image
            urls.append(item["media_url_https"])

            # add best quality video file
            if "video_info" in item.keys():
                max_bitrate = -1  # handle twitters occasional bitrate=0
                video_url = None
                for video in item["video_info"]["variants"]:
                    if "bitrate" in video.keys() and "content_type" in video.keys():
                        if video["content_type"] == "video/mp4":
                            if int(video["bitrate"]) > max_bitrate:
                                max_bitrate = int(video["bitrate"])
                                video_url = video["url"]

                if not video_url:
                    print("Error: No bitrate / content_type")
                    print(item["video_info"])
                else:
                    urls.append(video_url)

    return urls
909625
def get_entity_meta(self, datastream):
        
        url = '/datastream/' + str(datastream) + '/entityMeta'
        response = self.http.get(url)
        entityMetaList = []
        for entityMeta in response:
            entityMetaList.append(Schemas.EntityMeta(entityMeta=entityMeta))
        return entityMetaList
948548
def urlize_tweet(tweet):
    
    if tweet['retweeted']:
        text = 'RT {user}: {text}'.format(
            user=TWITTER_USERNAME_URL.format(screen_name=tweet['user']['screen_name']),
            text=tweet['text'])
    else:
        text = tweet['text']

    for hashtag in tweet['entities']['hashtags']:
        text = text.replace(
            '#%s' % hashtag['text'],
            TWITTER_HASHTAG_URL.format(hashtag=hashtag['text']))

    for mention in tweet['entities']['user_mentions']:
        text = text.replace(
            '@%s' % mention['screen_name'],
            TWITTER_USERNAME_URL.format(screen_name=mention['screen_name']))

    urls = tweet['entities']['urls']
    for url in urls:
        text = text.replace(
            url['url'], TWITTER_URL.format(
                url=url['expanded_url'], display_url=url['display_url']))

    if 'media' in tweet['entities']:
        for media in tweet['entities']['media']:
            text = text.replace(
                media['url'],
                TWITTER_MEDIA_URL.format(
                    url=media['expanded_url'],
                    display_url=media['display_url']))

    return mark_safe(text)
817031
def parse_url(track_url):
    
    global arguments
    item = get_item(track_url)
    logger.debug(item)
    if not item:
        return
    elif item['kind'] == 'track':
        logger.info('Found a track')
        download_track(item)
    elif item['kind'] == 'playlist':
        logger.info('Found a playlist')
        download_playlist(item)
    elif item['kind'] == 'user':
        logger.info('Found a user profile')
        if arguments['-f']:
            download(item, 'favorites', 'likes')
        elif arguments['-C']:
            download(item, 'commented', 'commented tracks')
        elif arguments['-t']:
            download(item, 'tracks', 'uploaded tracks')
        elif arguments['-a']:
            download(item, 'all', 'tracks and reposts')
        elif arguments['-p']:
            download(item, 'playlists', 'playlists')
        elif arguments['-m']:
            download(item, 'playlists-liked', 'my and liked playlists')
        else:
            logger.error('Please provide a download type...')
    else:
        logger.error('Unknown item type {0}'.format(item['kind']))
520062
def read_json(directory, data_files='data/js/tweets/*.js'):
    '''
    
    '''
    files = path.join(directory, data_files)

    for fname in iglob(files):
        with open(fname, 'r') as f:
            # Twitter's JSON first line is bogus
            data = f.readlines()[1:]
            data = "".join(data)
            tweetlist = json.loads(data)

        for tweet in tweetlist:
            yield tweet
701469
def _get_media(media_types):
    """"""
    get_mapped_media = (lambda x: maps.VIRTUAL_MEDIA_TYPES_MAP[x]
                        if x in maps.VIRTUAL_MEDIA_TYPES_MAP else None)
    return list(map(get_mapped_media, media_types))
337420
def load_tlds():
    
    file = os.path.join(os.path.dirname(__file__),
                        'assets',
                        'tlds-alpha-by-domain.txt')
    with open(file) as fobj:
        return [elem for elem in fobj.read().lower().splitlines()[1:]
                if "--" not in elem]
627101
def medias(self, timeframe=None):
        
        info = self.info
        if timeframe is not None:
            start, end = TimedMediasIterator.get_times(timeframe)
            timestamp = info.get("taken_at_timestamp") or info["media"]
            if not (start >= timestamp >= end):
                raise StopIteration
        yield info
1015434
def reverse_media_url(target_type, url_string, *args, **kwargs):
    '''
    
    '''
    args_str = '<%s>' % '><'.join(args)
    kwargs_str = '<%s>' % '><'.join('%s:%s' % pair for pair in kwargs.items())
    url_str = ''.join([url_string, args_str, kwargs_str])
    normalized_url = str(ResourceURL(url_str))

    query_tuples = []

    if singletons.settings.SECURITY and 'Sha1' in singletons.settings.SECURITY:
        secret = singletons.settings.HMAC_SECRET
        digest = get_hmac_sha1_digest(secret, normalized_url, target_type)
        query_tuples.append(('digest', digest))

    # Add in URL as last querystring argument
    query_tuples.append(('url', normalized_url))

    querystring = urlencode(query_tuples)
    scheme = singletons.settings.EXTERNAL_SCHEME
    host = singletons.settings.EXTERNAL_HOST
    port = singletons.settings.EXTERNAL_PORT
    if not host:
        host = singletons.settings.HOST
    if not port:
        port = singletons.settings.PORT

    port_suffix = ':%s' % port if port != 80 else ''

    typestring_normalized = str(TypeString(target_type))
    return '%s://%s%s/media/%s/?%s' % (
        scheme,
        host,
        port_suffix,
        typestring_normalized,
        querystring,
    )
867612
def _parse(self):
        
        stream = self.stream
        media_type = self.content_type

        if stream is None or media_type is None:
            empty_data = QueryDict('', encoding=self._request._encoding)
            empty_files = MultiValueDict()
            return (empty_data, empty_files)

        parser = self.negotiator.select_parser(self, self.parsers)

        if not parser:
            raise exceptions.UnsupportedMediaType(media_type)

        try:
            parsed = parser.parse(stream, media_type, self.parser_context)
        except:
            # If we get an exception during parsing, fill in empty data and
            # re-raise.  Ensures we don't simply repeat the error when
            # attempting to render the browsable renderer response, or when
            # logging the request or similar.
            self._data = QueryDict('', encoding=self._request._encoding)
            self._files = MultiValueDict()
            self._full_data = self._data
            raise

        # Parser classes may return the raw data, or a
        # DataAndFiles object.  Unpack the result as required.
        try:
            return (parsed.data, parsed.files)
        except AttributeError:
            empty_files = MultiValueDict()
            return (parsed, empty_files)
1108970
def fetch_list_members(list_url):
    
    match = re.match(r'.+twitter\.com\/(.+)\/lists\/(.+)', list_url)
    if not match:
        print('cannot parse list url %s' % list_url)
        return []
    screen_name, slug = match.groups()
    print('collecting list %s/%s' % (screen_name, slug))
    return twutil.collect.list_members(slug, screen_name)
41595
def cat_tweets(filename='all_tweets.json.gz', path=DATA_PATH, ext='.json', save_tmp=False, verbosity=1, numtweets=10000000, ignore_suspicious=True):
    """"""
    log.info('Finding {} files in {}...'.format(ext, path))
    meta_files = find_files(path=path, ext=ext)
    meta_files = [meta for meta in meta_files
                  if re.match(r'^[-#@a-z ]*201[5-6]-[0-9]{2}-[0-9]{2}.*', meta['name'])]
    #  '\s[0-9]{2}[:][0-9]{2}[:][0-9]{2}[.][0-9]+[.]json(.gz)?$', meta['name'])]
    log.info('Found {} files that look like tweetget dumps.'.format(len(meta_files)))
    print([mf['name'] for mf in meta_files])
    total_size = sum([meta['size'] for meta in meta_files])
    if verbosity > 0:
        pbar = progressbar.ProgressBar(maxval=(total_size + 1.) / 1e6)
        pbar.start()
    else:
        pbar = None

    loaded_size = 0
    df_all = pd.DataFrame()
    for meta in meta_files:
        with (gzip.open(meta['path']) if ext.endswith('.gz') else open(meta['path'])) as fin:
            js = pd.json.load(fin)
        if not len(js):
            continue
        df = pd.io.json.json_normalize(js)
        # json entries were dumped in reverse time order (most recent first)
        df.drop_duplicates(['id'], keep='first', inplace=True)
        df.set_index('id', drop=True, inplace=True)
        if 'geo.coordinates' in df.columns:
            latlon = np.array([(ll[0], ll[1]) if isinstance(ll, list) else (np.nan, np.nan) for ll in df['geo.coordinates']])
            for i, ll in enumerate(latlon):
                try:
                    latlon[i] = float(ll[0]), float(ll[1])
                except ValueError:
                    latlon[i] = np.nan, np.nan
            ll = list(zip(*latlon))
            df['lat'] = ll[0]
            df['lon'] = ll[1]
            df_all = df_all.append(df)
        else:
            df['lat'] = np.nan * np.ones(len(df))
            df['lon'] = np.nan * np.ones(len(df))
            if ignore_suspicious:
                log.info('\nOddly the DataFrame in {} didnt have a geo.coordinates column.'.format(meta['path']))
                log.warn('\nSkipping {} suspicious tweets.'.format(len(df)))
            else:
                df_all = df_all.append(df)
        # this would be a good time to incrementally save these rows to disc
        del df
        loaded_size += meta['size']
        if len(df_all) >= numtweets:
            # FIXME use average rate of 400 tweets/MB to calculate better progressbar size at start
            break
        if save_tmp:
            save_tweets(df_all, path=path, filename='tmp.csv')
        if pbar:
            pbar.update(loaded_size / 1e6)
    print(len(df_all))
    bigger_better_cols = [c for c in df_all.columns if c.endswith('_at') or '_count' in c]
    print(bigger_better_cols)
    print(all(c in df_all.columns for c in bigger_better_cols))
    df_all = df_all.sort_values(by=bigger_better_cols, inplace=False)
    hashable_cols = [c for c in df_all.columns if df_all[c].dtype not in (list,)]
    df_all = df_all.drop_duplicates(subset=hashable_cols, inplace=False)
    if pbar:
        pbar.finish()
    log.info('Loaded {} unique tweets.'.format(len(df_all)))
    return df_all
274161
def to_array(self):
        
        array = super(InputMediaDocument, self).to_array()
        # 'type' given by superclass
        # 'media' given by superclass
        if self.thumb is not None:
            if isinstance(self.thumb, InputFile):
                array['thumb'] = None  # type InputFile
            elif isinstance(self.thumb, str):
                array['thumb'] = u(self.thumb)  # py2: type unicode, py3: type str
            else:
                raise TypeError('Unknown type, must be one of InputFile, str.')
            # end if
        # 'caption' given by superclass
        # 'parse_mode' given by superclass
        return array
1024372
def parse_oembed_data(oembed_data, data):
    
    data.update({
        'oembed': oembed_data,
    })
    _type = oembed_data.get('type')
    provider_name = oembed_data.get('provider_name')
    if not _type:
        return data

    if oembed_data.get('title'):
        data.update({
            'title': oembed_data.get('title'),
        })

    if _type == 'video':
        try:
            item = {
                'width': convert_to_int(oembed_data.get('width')),
                'height': convert_to_int(oembed_data.get('height'))
            }
            if provider_name in ['YouTube', ]:
                item['src'] = HYPERLINK_PATTERN.search(oembed_data.get('html')).group(0)

            data['videos'].append(item)
        except Exception:
            pass

        if oembed_data.get('thumbnail_url'):
            item = {
                'width': convert_to_int(oembed_data.get('thumbnail_width')),
                'height': convert_to_int(oembed_data.get('thumbnail_height')),
                'src': oembed_data.get('thumbnail_url')
            }

            data['images'].append(item)

    return data
478092
def parse_args(args):
    
    parser = argparse.ArgumentParser(
        description="Concatenate and preprocess tweet json files dumped by tweetget")
    parser.add_argument(
        '-v',
        '--verbose',
        help='Whether to show progress messages on stdout',
        action='store_true')
    parser.add_argument(
        '--version',
        help='print twip package version and exit.',
        action='version',
        version='twip {ver}'.format(ver=__version__))
    parser.add_argument(
        '-p',
        '--path',
        help='path to data dir (json input and csv output files)',
        default=DATA_PATH)
    parser.add_argument(
        '-t',
        '--tweetfile',
        help='file to store normalized table (csv) of tweets and features (metadata)',
        default='all_tweets.csv')
    parser.add_argument(
        '-n',
        '--numtweets',
        help='maximum number of tweets to process',
        type=int,
        default=1000000)
    parser.add_argument(
        '-g',
        '--geofile',
        help="file to store tweet lat/lon information (few use it)",
        default='geo_tweets.csv')
    return parser.parse_args(args)
447337
def _parse_image_multilogs_string(config, ret, repo):
    '''
    
    '''
    image_logs, infos = [], None
    if ret and ret.strip().startswith('{') and ret.strip().endswith('}'):
        pushd = 0
        buf = ''
        for char in ret:
            buf += char
            if char == '{':
                pushd += 1
            if char == '}':
                pushd -= 1
            if pushd == 0:
                try:
                    buf = json.loads(buf)
                except Exception:
                    pass
                image_logs.append(buf)
                buf = ''
        image_logs.reverse()
        # search last layer grabbed
        for l in image_logs:
            if isinstance(l, dict):
                if l.get('status') == 'Download complete' and l.get('id'):
                    infos = _get_image_infos(config, repo)
                    break
    return image_logs, infos
108936
def ParseDownloadsRow(
      self, parser_mediator, query, row, **unused_kwargs):
    
    query_hash = hash(query)

    event_data = FirefoxDownloadEventData()
    event_data.full_path = self._GetRowValue(query_hash, row, 'target')
    event_data.mime_type = self._GetRowValue(query_hash, row, 'mimeType')
    event_data.name = self._GetRowValue(query_hash, row, 'name')
    event_data.offset = self._GetRowValue(query_hash, row, 'id')
    event_data.query = query
    event_data.received_bytes = self._GetRowValue(query_hash, row, 'currBytes')
    event_data.referrer = self._GetRowValue(query_hash, row, 'referrer')
    event_data.temporary_location = self._GetRowValue(
        query_hash, row, 'tempPath')
    event_data.total_bytes = self._GetRowValue(query_hash, row, 'maxBytes')
    event_data.url = self._GetRowValue(query_hash, row, 'source')

    timestamp = self._GetRowValue(query_hash, row, 'startTime')
    if timestamp:
      date_time = dfdatetime_posix_time.PosixTimeInMicroseconds(
          timestamp=timestamp)
      event = time_events.DateTimeValuesEvent(
          date_time, definitions.TIME_DESCRIPTION_START)
      parser_mediator.ProduceEventWithEventData(event, event_data)

    timestamp = self._GetRowValue(query_hash, row, 'endTime')
    if timestamp:
      date_time = dfdatetime_posix_time.PosixTimeInMicroseconds(
          timestamp=timestamp)
      event = time_events.DateTimeValuesEvent(
          date_time, definitions.TIME_DESCRIPTION_END)
      parser_mediator.ProduceEventWithEventData(event, event_data)