Load one or more CADA CSV files matching patterns
488863
def load(patterns, full_reindex):
    '''
    
    '''
    header('Loading CSV files')
    for pattern in patterns:
        for filename in iglob(pattern):
            echo('Loading {}'.format(white(filename)))
            with open(filename) as f:
                reader = csv.reader(f)
                # Skip header
                reader.next()
                for idx, row in enumerate(reader, 1):
                    try:
                        advice = csv.from_row(row)
                        skipped = False
                        if not full_reindex:
                            index(advice)
                        echo('.' if idx % 50 else white(idx), nl=False)
                    except Exception:
                        echo(cyan('s') if idx % 50 else white('{0}(s)'.format(idx)), nl=False)
                        skipped = True
                if skipped:
                    echo(white('{}(s)'.format(idx)) if idx % 50 else '')
                else:
                    echo(white(idx) if idx % 50 else '')
                success('Processed {0} rows'.format(idx))
    if full_reindex:
        reindex()
1224244
def recursive_load(search_root):
    """"""
    for root, dirs, files in os.walk(search_root):
        dir_name = os.path.basename(root)
        if dir_name == 'fixtures':
            for file_name in files:
                fixture_path = os.path.join(root, file_name)
                execute_manage('loaddata', fixture_path)
1338418
def load_results(result_files, options, run_set_id=None, columns=None,
                 columns_relevant_for_diff=set()):
    """"""
    return parallel.map(
        load_result,
        result_files,
        itertools.repeat(options),
        itertools.repeat(run_set_id),
        itertools.repeat(columns),
        itertools.repeat(columns_relevant_for_diff))
64723
def load_datafile(name, search_path, codecs=get_codecs(), **kwargs):
    
    return munge.load_datafile(name, search_path, codecs, **kwargs)
1339473
def ImportPatterns(self, s):
    ""
    for line, strli, direction, seq, station_id in  \
    ReadCSV(s, ['LI_NR', 'STR_LI_VAR', 'LI_RI_NR', 'LI_LFD_NR', 'ORT_NR']):
      pattern_id = u'Pat.%s.%s.%s' % (line, strli, direction)
      pattern = self.patterns.get(pattern_id, None)
      if not pattern:
        pattern = Pattern()
        pattern.id = pattern_id
        pattern.stops = []
        pattern.stoptimes = {}
        self.patterns[pattern_id] = pattern
      seq = int(seq) - 1
      if len(pattern.stops) <= seq:
        pattern.stops.extend([None] * (seq - len(pattern.stops)  + 1))
      pattern.stops[seq] = station_id
271180
def catCSVs(folder, ouputFileName, removeDups = False) :
	""""""
	strCmd = r"""cat %s/*.csv > %s""" %(folder, ouputFileName)
	os.system(strCmd)

	if removeDups :
		removeDuplicates(ouputFileName, ouputFileName)
66203
def load_csvf(fpath, fieldnames, encoding):
    
    with open(fpath, mode='r', encoding=encoding) as f:
        snippet = f.read(8192)
        f.seek(0)

        dialect = csv.Sniffer().sniff(snippet)
        dialect.skipinitialspace = True
        return list(csv.DictReader(f, fieldnames=fieldnames, dialect=dialect))
473029
def _load(self, files_in, files_out, urlpath, meta=True):
        """"""
        import dask
        out = []
        outnames = []
        for file_in, file_out in zip(files_in, files_out):
            cache_path = file_out.path
            outnames.append(cache_path)

            # If `_munge_path` did not find a match we want to avoid
            # writing to the urlpath.
            if cache_path == urlpath:
                continue

            if not os.path.isfile(cache_path):
                logger.debug("Caching file: {}".format(file_in.path))
                logger.debug("Original path: {}".format(urlpath))
                logger.debug("Cached at: {}".format(cache_path))
                if meta:
                    self._log_metadata(urlpath, file_in.path, cache_path)
                ddown = dask.delayed(_download)
                out.append(ddown(file_in, file_out, self.blocksize,
                                 self.output))
        dask.compute(*out)
        return outnames
477684
def import_csv(csv_file, **kwargs):
    """"""
    records = get_imported_data(csv_file, **kwargs)
    _check_required_columns(csv_file, records.results)
    return records
48534
def _get_purecn_dx_files(paired, out):
    
    out_base = "%s-dx" % utils.splitext_plus(out["rds"])[0]
    all_files = []
    for key, ext in [[("mutation_burden",), "_mutation_burden.csv"],
                     [("plot", "signatures"), "_signatures.pdf"],
                     [("signatures",), "_signatures.csv"]]:
        cur_file = "%s%s" % (out_base, ext)
        out = tz.update_in(out, key, lambda x: cur_file)
        all_files.append(os.path.basename(cur_file))
    return out_base, out, all_files
202878
def load_matches(self):
        
        if not self.exists():
            return []

        matches = []
        with open(self.location(), 'r') as f:
            for line in f.readlines():
                match_obj = json.loads(line.strip("\n"))
                if match_obj['processed']:
                    continue

                matches.append(match_obj)

        return matches
1133945
def load(self):
        
        # read file, keep all values as strings
        df = pd.read_csv(self.input_file,
                         sep=',',
                         quotechar='"',
                         encoding='utf-8',
                         dtype=object)

        # wer are only interested in the NUTS code and description, rename them also
        df = df[['NUTS-Code', 'Description']]
        df.columns = ['key', 'name']

        # we only want NUTS2 regions (4-digit codes)
        df = df[df['key'].str.len() == 4]

        # drop 'Extra Regio' codes ending in 'ZZ'
        df = df[df['key'].str[2:] != 'ZZ']

        return df
636428
def download(csvpath, asset_manager_id, data_id_type, data_id_list):
        """"""
        interface = interface_direct_csvpath(csvpath)
        logging.config.dictConfig(DEFAULT_LOGGING)
        logger = logging.getLogger(__name__)
        objs = []
        for data_id in data_id_list:
            Dict = dict()
            Dict[data_id_type] = data_id
            objs.append(interface.retrieve(asset_manager_id=asset_manager_id, **Dict))
        return objs
134993
def _match_with_pandas(filtered, matcher):
    """"""

    import pandas

    data = [fl.to_dict() for fl in filtered]

    if not data:
        return []

    df = pandas.DataFrame(data)
    df = df.sort_values(['uuid'])

    cdfs = []
    criteria = matcher.matching_criteria()

    for c in criteria:
        cdf = df[['id', 'uuid', c]]
        cdf = cdf.dropna(subset=[c])
        cdf = pandas.merge(cdf, cdf, on=c, how='left')
        cdf = cdf[['uuid_x', 'uuid_y']]
        cdfs.append(cdf)

    result = pandas.concat(cdfs)
    result = result.drop_duplicates()
    groups = result.groupby(by=['uuid_x'],
                            as_index=True, sort=True)

    matched = _calculate_matches_closures(groups)

    return matched
502420
def file_matches(filename, patterns):
    """"""
    return any(fnmatch.fnmatch(filename, pat)
               or fnmatch.fnmatch(os.path.basename(filename), pat)
               for pat in patterns)
1057233
def load_lincs_csv(url):
    """"""
    resp = requests.get(url, params={'output_type': '.csv'}, timeout=120)
    resp.raise_for_status()
    if sys.version_info[0] < 3:
        csv_io = BytesIO(resp.content)
    else:
        csv_io = StringIO(resp.text)
    data_rows = list(read_unicode_csv_fileobj(csv_io, delimiter=','))
    headers = data_rows[0]
    return [{header: val for header, val in zip(headers, line_elements)}
            for line_elements in data_rows[1:]]
47018
def super_glob(pattern):
    ''
    pieces = pattern.split('/')
    if '**' in pieces:
        prefix = '/'.join(pieces[:pieces.index('**')])
        postfix = '/'.join(pieces[pieces.index('**') + 1:])
        roots = [dirname
                 for dirname, dirnames, filenames in os.walk(prefix)]
        patterns = [root + '/' + postfix for root in roots]
    else:
        patterns = ['/'.join(pieces)]
    return chain.from_iterable(glob(pattern) for pattern in patterns)