parse csv file with one line per file. It will merge
    all files that have the same description name
485574
def _get_samples_to_process(fn, out_dir, config, force_single, separators):
    """"""
    out_dir = os.path.abspath(out_dir)
    samples = defaultdict(list)
    with open(fn) as handle:
        for l in handle:
            if l.find("description") > 0:
                logger.info("Skipping header.")
                continue
            cols = l.strip().split(",")
            if len(cols) > 0:
                if len(cols) < 2:
                    raise ValueError("Line needs 2 values: file and name.")
                if utils.file_exists(cols[0]) or is_gsm(cols[0]) or is_srr(cols[0]):
                    if cols[0].find(" ") > -1:
                        new_name = os.path.abspath(cols[0].replace(" ", "_"))
                        logger.warning("Space finds in %s. Linked to %s." % (cols[0], new_name))
                        logger.warning("Please, avoid names with spaces in the future.")
                        utils.symlink_plus(os.path.abspath(cols[0]), new_name)
                        cols[0] = new_name
                    samples[cols[1]].append(cols)
                else:
                    logger.info("skipping %s, File doesn't exist." % cols[0])
    for sample, items in samples.items():
        if is_fastq(items[0][0], True):
            fn = "fq_merge"
            ext = ".fastq.gz"
        elif is_bam(items[0][0]):
            fn = "bam_merge"
            ext = ".bam"
        elif is_gsm(items[0][0]):
            fn = "query_gsm"
            ext = ".fastq.gz"
        elif is_srr(items[0][0]):
            fn = "query_srr"
            ext = ".fastq.gz"
        files = [os.path.abspath(fn_file[0]) if utils.file_exists(fn_file[0]) else fn_file[0] for fn_file in items]
        samples[sample] = [{'files': _check_paired(files, force_single, separators),
                            'out_file': os.path.join(out_dir, sample + ext),
                            'fn': fn, 'anno': items[0][2:], 'config': config,
                            'name': sample, 'out_dir': out_dir}]
    return [samples[sample] for sample in samples]
1117397
def _get_data_from_csv_files(self):
    """"""

    all_df = []
    for file_name in self._input_csv_files:
      with _util.open_local_or_gcs(file_name, mode='r') as f:
        all_df.append(pd.read_csv(f, names=self._headers))
    df = pd.concat(all_df, ignore_index=True)
    return df
854005
def parse(self, filenames):
        
        filenames = list_strings(filenames)

        read_ok = []
        for fname in filenames:
            try:
                fh = open(fname)
            except IOError:
                logger.warning("Cannot open file %s" % fname)
                continue

            try:
                self._read(fh, fname)
                read_ok.append(fname)

            except self.Error as e:
                logger.warning("exception while parsing file %s:\n%s" % (fname, str(e)))
                continue

            finally:
                fh.close()

        # Add read_ok to the list of files that have been parsed.
        self._filenames.extend(read_ok)
        return read_ok
894843
def read_csv(filename):
    

    with open(filename) as file:
        table = list(csv.reader(file))
    m = from_table(table)
    return m
1369439
def csv_to_dict(file_name, file_location):
    
    file = __os.path.join(file_location, file_name)
    try:
        csv_read = open(file, "r")
    except Exception as e:
        LOGGER.critical('Function csv_to_dict Error {error} ignoring any errors'.format(error=e))
        print('Error {error} ignoring any errors'.format(error=e))
        csv_read = open(file, "r", errors='ignore')
    data_row = __csv.DictReader(csv_read, dialect="excel")
    dict_key = 1
    temp_dict = dict()
    for row in data_row:
        temp_dict[dict_key] = row
        dict_key += 1
    csv_read.close()
    return temp_dict
526057
def csv_to_dict(csv_filepath, **kwargs):
    
    callbacks = {'to_list': csv_tolist,
                 'row_csv_limiter': row_csv_limiter,
                 'csv_row_cleaner': csv_row_cleaner,
                 'row_headers_count': row_headers_count,
                 'get_col_header': get_csv_col_headers,
                 'get_row_headers': get_row_headers,
                 'populate_headers': populate_headers,
                 'csv_column_header_cleaner': csv_column_header_cleaner,
                 'csv_column_cleaner': csv_column_cleaner,
                 'retrieve_csv_data': retrieve_csv_data}

    callbacks.update(kwargs.get('alt_callbacks', {}))
    rows = kwargs.get('rows', [])

    if not rows:
        # csv_tolist of rows
        rows = callbacks.get('to_list')(csv_filepath, **kwargs)

        if not rows:
            msg = 'Empty rows obtained from {}'.format(csv_filepath)
            logger.warning(msg)
            raise ValueError(msg)

    # apply limits
    rows = callbacks.get('row_csv_limiter')(
        rows, kwargs.get('limits', [None, None]))

    # apply row cleaner
    rows = callbacks.get('csv_row_cleaner')(rows)

    # apply column cleaner
    rows = callbacks.get('csv_column_cleaner')(rows)

    # count raw headers
    num_row_headers = callbacks.get('row_headers_count')(rows)

    # take colum_headers
    c_headers_raw = callbacks.get('get_col_header')(rows, num_row_headers)

    # get row_headers
    r_headers = callbacks.get('get_row_headers')(
        rows, num_row_headers, len(c_headers_raw))

    # format colum_headers
    c_headers_dirty = callbacks.get('populate_headers')(
        c_headers_raw) if len(c_headers_raw) > 1 else c_headers_raw[0]

    # Clean csv column headers of empty values.
    c_headers = callbacks.get('csv_column_header_cleaner')(c_headers_dirty)

    # take data
    csv_data = callbacks.get('retrieve_csv_data')(
        rows,
        column_header=len(c_headers_raw),
        row_header=num_row_headers,
        limit_column=len(c_headers) - len(c_headers_dirty) or None)

    # Check column headers validation
    if csv_data:
        assert len(c_headers) == len(csv_data[0])

    # Check row headers validation
    if r_headers:
        assert len(r_headers) == len(csv_data)

    # Transform rows into dict zipping the headers.
    kwargs.pop('rows', None)
    result = csv_format(csv_data, c_headers, r_headers, rows, **kwargs)

    return result
200565
def _merge_out_from_infiles(in_files):
    
    fname = os.path.commonprefix([os.path.basename(f) for f in in_files])
    while fname.endswith(("-", "_", ".")):
        fname = fname[:-1]
    ext = os.path.splitext(in_files[0])[-1]
    dirname = os.path.dirname(in_files[0])
    while dirname.endswith(("split", "merge")):
        dirname = os.path.dirname(dirname)
    return os.path.join(dirname, "%s%s" % (fname, ext))
1141451
def _parsecsv(x):
        
        for line in x:
            # decode as utf-8, whitespace-strip and split on delimiter
            yield line.decode('utf-8').strip().split(config.DELIMITER)
567988
def iter_csv_stream(input_stream, fieldnames=None, sniff=False, *args, **kwargs):
    '''  '''
    if 'dialect' not in kwargs and sniff:
        kwargs['dialect'] = csv.Sniffer().sniff(input_stream.read(1024))
        input_stream.seek(0)
    if 'quoting' in kwargs and kwargs['quoting'] is None:
        kwargs['quoting'] = csv.QUOTE_MINIMAL
    if fieldnames:
        # read csv using dictreader
        if isinstance(fieldnames, bool):
            reader = csv.DictReader(input_stream, *args, **kwargs)
        else:
            reader = csv.DictReader(input_stream, *args, fieldnames=fieldnames, **kwargs)
        for row in reader:
            yield row
    else:
        csvreader = csv.reader(input_stream, *args, **kwargs)
        for row in csvreader:
            yield row
962254
def fix(csvfile):
    ''''''
    header('Apply fixes from {}', csvfile.name)
    bads = []
    reader = csv.reader(csvfile)
    reader.next()  # Skip header
    for id, _, sources, dests in reader:
        advice = Advice.objects.get(id=id)
        sources = [s.strip() for s in sources.split(',') if s.strip()]
        dests = [d.strip() for d in dests.split(',') if d.strip()]
        if not len(sources) == len(dests):
            bads.append(id)
            continue
        for source, dest in zip(sources, dests):
            echo('{0}: Replace {1} with {2}', white(id), white(source), white(dest))
            advice.subject = advice.subject.replace(source, dest)
            advice.content = advice.content.replace(source, dest)
        advice.save()
        index(advice)
    for id in bads:
        echo('{0}: Replacements length not matching', white(id))
    success('Done')
791657
def csv_to_list(csv_file, delimiter=','):
    
    with open_csv(csv_file) as csv_con:
        if len(delimiter) > 1:
            dialect = csv.Sniffer().sniff(csv_con.read(1024),
                                          delimiters=delimiter)
            csv_con.seek(0)
            reader = csv.reader(csv_con, dialect)
        else:
            reader = csv.reader(csv_con, delimiter=delimiter)
        return list(reader)
588179
def read_csv_file(self, file_name):
        
        result = []
        with open(os.path.join(self.__path(), os.path.basename(file_name)),
                  'rt') as csvfile:
            headers_reader = csv.reader(csvfile, delimiter=',', quotechar='|')
            for type_row in headers_reader:
                for t in type_row:
                    result.append(t)
        return result
256805
def DoxyfileParse(file_contents):
   
   data = {}

   import shlex
   lex = shlex.shlex(instream = file_contents, posix = True)
   lex.wordchars += "*+./-:"
   lex.whitespace = lex.whitespace.replace("\n", "")
   lex.escape = ""

   lineno = lex.lineno
   token = lex.get_token()
   key = token   # the first token should be a key
   last_token = ""
   key_token = False
   next_key = False
   new_data = True

   def append_data(data, key, new_data, token):
      if new_data or len(data[key]) == 0:
         data[key].append(token)
      else:
         data[key][-1] += token

   while token:
      if token in ['\n']:
         if last_token not in ['\\']:
            key_token = True
      elif token in ['\\']:
         pass
      elif key_token:
         key = token
         key_token = False
      else:
         if token == "+=":
            if not data.has_key(key):
               data[key] = list()
         elif token == "=":
            if key == "TAGFILES" and data.has_key(key):
               append_data( data, key, False, "=" )
               new_data=False
            else:
               data[key] = list()
         else:
            append_data( data, key, new_data, token )
            new_data = True

      last_token = token
      token = lex.get_token()

      if last_token == '\\' and token != '\n':
         new_data = False
         append_data( data, key, new_data, '\\' )

   # compress lists of len 1 into single strings
   for (k, v) in data.items():
      if len(v) == 0:
         data.pop(k)

      # items in the following list will be kept as lists and not converted to strings
      if k in ["INPUT", "FILE_PATTERNS", "EXCLUDE_PATTERNS", "TAGFILES"]:
         continue

      if len(v) == 1:
         data[k] = v[0]

   return data
13861
def _split_lines(self):
        ''''''
        parsed_lines = {}
        for rt in all_record_types:
            parsed_lines[rt] = []
        parsed_lines[0] = []

        for line in self.lines:
            linetype = line[0:6]
            if linetype in all_record_types:
                parsed_lines[linetype].append(line)
            else:
                parsed_lines[0].append(line)

        self.parsed_lines = parsed_lines
        self._update_structure_lines()
1234473
def split_file_info(fileinfo):
    
    current_year = datetime.datetime.now().strftime('%Y')
    files = []
    for line in fileinfo:
        parts = re.split(
            r'^([\-dbclps])' +                  # Directory flag [1]
            r'([\-rwxs]{9})\s+' +               # Permissions [2]
            r'(\d+)\s+' +                       # Number of items [3]
            r'([a-zA-Z0-9_-]+)\s+' +            # File owner [4]
            r'([a-zA-Z0-9_-]+)\s+' +            # File group [5]
            r'(\d+)\s+' +                       # File size in bytes [6]
            r'(\w{3}\s+\d{1,2})\s+' +           # 3-char month and 1/2-char day of the month [7]
            r'(\d{1,2}:\d{1,2}|\d{4})\s+' +     # Time or year (need to check conditions) [+= 7]
            r'(.+)$',                           # File/directory name [8]
            line
        )

        date = parts[7]
        time = parts[8] if ':' in parts[8] else '00:00'
        year = parts[8] if ':' not in parts[8] else current_year
        dt_obj = parser.parse("%s %s %s" % (date, year, time))

        files.append({
            'directory': parts[1],
            'perms': parts[2],
            'items': parts[3],
            'owner': parts[4],
            'group': parts[5],
            'size': int(parts[6]),
            'date': date,
            'time': time,
            'year': year,
            'name': parts[9],
            'datetime': dt_obj
        })
    return files
359196
def filelines(fname,strip=False):
    ''''''
    with open(fname,'r') as f:
        lines = f.readlines();
    if strip:
        lines[:] = [line.strip() for line in lines]
    return lines;
1208741
def read_csv(fpath):
    
    import csv
    import utool as ut
    #csvfile = open(fpath, 'rb')
    with open(fpath, 'rb') as csvfile:
        row_iter = csv.reader(csvfile, delimiter=str(','), quotechar=str('|'))
        row_list = [ut.lmap(ut.ensure_unicode, row) for row in row_iter]
    return row_list