Get the links of a FEBRL dataset.
183069
def _febrl_links(df):
    """"""

    index = df.index.to_series()
    keys = index.str.extract(r'rec-(\d+)', expand=True)[0]

    index_int = numpy.arange(len(df))

    df_helper = pandas.DataFrame({
        'key': keys,
        'index': index_int
    })

    # merge the two frame and make MultiIndex.
    pairs_df = df_helper.merge(
        df_helper, on='key'
    )[['index_x', 'index_y']]
    pairs_df = pairs_df[pairs_df['index_x'] > pairs_df['index_y']]

    return pandas.MultiIndex(
        levels=[df.index.values, df.index.values],
        labels=[pairs_df['index_x'].values, pairs_df['index_y'].values],
        names=[None, None],
        verify_integrity=False
    )
1343378
def generate_item_urls(self):
        """"""
        item_urls = {}
        for i in self.dataset.identifiers:
            relpath = self.dataset.item_properties(i)["relpath"]
            url = self.generate_url("data/" + relpath)
            item_urls[i] = url
        return item_urls
493073
def get_data(url, gallery_dir):
    """"""

    # shelve keys need to be str in python 2
    if sys.version_info[0] == 2 and isinstance(url, unicode):
        url = url.encode('utf-8')

    cached_file = os.path.join(gallery_dir, 'searchindex')
    search_index = shelve.open(cached_file)
    if url in search_index:
        data = search_index[url]
    else:
        data = _get_data(url)
        search_index[url] = data
    search_index.close()

    return data
416224
def resource_to_rdf(resource, dataset=None, graph=None):
    '''
    
    '''
    graph = graph or Graph(namespace_manager=namespace_manager)
    if dataset and dataset.id:
        id = URIRef(url_for('datasets.show_redirect', dataset=dataset.id,
                            _external=True,
                            _anchor='resource-{0}'.format(resource.id)))
    else:
        id = BNode(resource.id)
    permalink = url_for('datasets.resource', id=resource.id, _external=True)
    r = graph.resource(id)
    r.set(RDF.type, DCAT.Distribution)
    r.set(DCT.identifier, Literal(resource.id))
    r.add(DCT.title, Literal(resource.title))
    r.add(DCT.description, Literal(resource.description))
    r.add(DCAT.downloadURL, URIRef(resource.url))
    r.add(DCAT.accessURL, URIRef(permalink))
    r.add(DCT.issued, Literal(resource.published))
    r.add(DCT.modified, Literal(resource.modified))
    if dataset and dataset.license:
        r.add(DCT.rights, Literal(dataset.license.title))
        if dataset.license.url:
            r.add(DCT.license, URIRef(dataset.license.url))
    if resource.filesize is not None:
        r.add(DCAT.bytesSize, Literal(resource.filesize))
    if resource.mime:
        r.add(DCAT.mediaType, Literal(resource.mime))
    if resource.format:
        r.add(DCT.term('format'), Literal(resource.format))
    if resource.checksum:
        checksum = graph.resource(BNode())
        checksum.set(RDF.type, SPDX.Checksum)
        algorithm = 'checksumAlgorithm_{0}'.format(resource.checksum.type)
        checksum.add(SPDX.algorithm, getattr(SPDX, algorithm))
        checksum.add(SPDX.checksumValue, Literal(resource.checksum.value))
        r.add(SPDX.checksum, checksum)
    return r
460016
def results(self, dataset_name, index_by, timeframe):
        
        url = "{0}/{1}/results".format(self._cached_datasets_url, dataset_name)

        index_by = index_by if isinstance(index_by, str) else json.dumps(index_by)
        timeframe = timeframe if isinstance(timeframe, str) else json.dumps(timeframe)

        query_params = {
            "index_by": index_by,
            "timeframe": timeframe
        }

        return self._get_json(
            HTTPMethods.GET, url, self._get_read_key(), params=query_params
        )
364072
def links(cls, page):
    """"""
    for match in cls.HREF_RE.finditer(page):
      yield cls.href_match_to_url(match)
6765
def _get_keywords(self, **kwargs):
        """"""
        if self.jurisdiction and 'jurisdiction_id' not in kwargs:
            kwargs['jurisdiction_id'] = self.jurisdiction
        if 'count' in kwargs:
            kwargs['page_size'] = kwargs.pop('count')
        if 'start' in kwargs:
            start = kwargs.pop('start')
            if 'end' in kwargs:
                end = kwargs.pop('end')
            else:
                end = date.today().strftime('%m-%d-%Y')
            start, end = self._format_dates(start, end)
            kwargs['start_date'] = start
            kwargs['end_date'] = end
        elif 'between' in kwargs:
            start, end = kwargs.pop('between')
            start, end = self._format_dates(start, end)
            kwargs['start_date'] = start
            kwargs['end_date'] = end
        return kwargs
1161311
def links(self):
    '''
    
    '''

    if not getattr(self, '_links', False):
      self._links = [
        link['title']
        for link in self.__continued_query({
          'prop': 'links',
          'plnamespace': 0,
          'pllimit': 'max'
        })
      ]

    return self._links
467584
def output_links(self):
        """"""
        output_links = []

        pm_dict = self.get_dict()
        output_links.append('surface_sample')
        if pm_dict['target_volume'] != 0.0:
            output_links.append('cell')

        return output_links
310150
def get(dataset = None, include_metadata = False, mnemonics = None, **dim_values):
    """"""

    if not dataset and not mnemonics:
        raise ValueError('Dataset id is not specified')

    if mnemonics and dim_values:
        raise ValueError('The function does not support specifying mnemonics and selection in a single call')

    config = ApiConfig()
    client = ApiClient(config.host, config.app_id, config.app_secret)
    client.check_correct_host()

    ds = client.get_dataset(dataset) if dataset else None
    reader =  MnemonicsDataReader(client, mnemonics) if mnemonics else StreamingDataReader(client, dim_values) if ds.type == 'Regular' else PivotDataReader(client, dim_values)
    reader.include_metadata = include_metadata
    reader.dataset = ds

    return reader.get_pandasframe()
1250965
def get_link_pages(links):
        
        link_pages = []
        i = 0
        while i < len(links):
            link_page = []
            while i < len(links) and len(link_page) < 10:
                link_page.append(links[i])
                i += 1
            link_pages.append(link_page)
        return link_pages
971209
def get_dataset(self, dataset_id, ds_info, xslice=slice(None), yslice=slice(None)):
        """"""
        var_path = ds_info.get('file_key', '{}'.format(dataset_id.name))
        metadata = self.get_metadata(dataset_id, ds_info)
        shape = metadata['shape']
        file_shape = self[var_path + '/shape']
        if isinstance(shape, tuple) and len(shape) == 2:
            # 2D array
            if xslice.start is not None:
                shape = (shape[0], xslice.stop - xslice.start)
            if yslice.start is not None:
                shape = (yslice.stop - yslice.start, shape[1])
        elif isinstance(shape, tuple) and len(shape) == 1 and yslice.start is not None:
            shape = ((yslice.stop - yslice.start) / yslice.step,)
        metadata['shape'] = shape

        valid_min = self[var_path + '/attr/valid_min']
        valid_max = self[var_path + '/attr/valid_max']
        # no need to check fill value since we are using valid min/max
        scale_factor = self.get(var_path + '/attr/scale_factor')
        add_offset = self.get(var_path + '/attr/add_offset')

        if isinstance(file_shape, tuple) and len(file_shape) == 3:
            data = self[var_path][0, yslice, xslice]
        elif isinstance(file_shape, tuple) and len(file_shape) == 2:
            data = self[var_path][yslice, xslice]
        elif isinstance(file_shape, tuple) and len(file_shape) == 1:
            data = self[var_path][yslice]
        else:
            data = self[var_path]
        data = data.where((data >= valid_min) & (data <= valid_max))
        if scale_factor is not None:
            data = data * scale_factor + add_offset

        if ds_info.get('cloud_clear', False):
            # clear-sky if bit 15-16 are 00
            clear_sky_mask = (self['l2p_flags'][0] & 0b1100000000000000) != 0
            data = data.where(~clear_sky_mask)

        data.attrs.update(metadata)
        return data
232241
def download_urls(self):
    """"""
    urls = {
        tfds.Split.TRAIN: ["train_clean100"],
        tfds.Split.VALIDATION: ["dev_clean"],
        tfds.Split.TEST: ["test_clean"],
    }
    if self.data in ["all", "clean360"]:
      urls[tfds.Split.TRAIN].append("train_clean360")
    if self.data == "all":
      urls[tfds.Split.TRAIN].extend(["train_clean360", "train_other500"])
      urls[tfds.Split.VALIDATION].append("dev_other")
      urls[tfds.Split.TEST].append("test_other")

    urls = {
        split: [_DL_URLS[name] for name in names
               ] for split, names in urls.items()
    }
    return urls
959487
def scrape_links(self, text, context=False):
        ''''''
        return self.iter_processed_links(io.StringIO(text), context=context)
824859
def get_new_links(self, url, resp):
        """"""
        links_on_page = resp.xpath('//a/@href')
        links = [utils.clean_url(u, url) for u in links_on_page]

        # Remove non-links through filtering by protocol
        links = [x for x in links if utils.check_protocol(x)]

        # Restrict new URLs by the domain of the input URL
        if not self.args['nonstrict']:
            domain = utils.get_domain(url)
            links = [x for x in links if utils.get_domain(x) == domain]

        # Filter URLs by regex keywords, if any
        if self.args['crawl']:
            links = utils.re_filter(links, self.args['crawl'])
        return links
343264
def dataset(self):
        """"""
        data = tablib.Dataset()
        data.headers = self.keys()

        row = _reduce_datetimes(self.values())
        data.append(row)

        return data
1160165
def links(self) -> _Links:
        """"""

        def gen():
            for link in self.find('a'):

                try:
                    href = link.attrs['href'].strip()
                    if href and not (href.startswith('#') and self.skip_anchors) and not href.startswith(('javascript:', 'mailto:')):
                        yield href
                except KeyError:
                    pass

        return set(gen())