Reads generic key-value pairs from input files
543857
def read_targets(targets):
    """"""
    results = {}
    for target, regexer in regexer_for_targets(targets):
        with open(target) as fh:
            results.update(extract_keypairs(fh.readlines(), regexer))
    return results
1056258
def _ctype_key_value(keys, vals):
    
    if isinstance(keys, (tuple, list)):
        assert(len(keys) == len(vals))
        c_keys = []
        c_vals = []
        use_str_keys = None
        for key, val in zip(keys, vals):
            c_key_i, c_val_i, str_keys_i = _ctype_key_value(key, val)
            c_keys += c_key_i
            c_vals += c_val_i
            use_str_keys = str_keys_i if use_str_keys is None else use_str_keys
            assert(use_str_keys == str_keys_i), "inconsistent types of keys detected."
        c_keys_arr = c_array(ctypes.c_char_p, c_keys) if use_str_keys \
                     else c_array(ctypes.c_int, c_keys)
        c_vals_arr = c_array(ctypes.c_void_p, c_vals)
        return (c_keys_arr, c_vals_arr, use_str_keys)

    assert(isinstance(keys, (int,) + string_types)), \
           "unexpected type for keys: " + str(type(keys))
    use_str_keys = isinstance(keys, string_types)
    if isinstance(vals, NDArray):
        c_keys = c_str_array([keys]) if use_str_keys \
                 else c_array_buf(ctypes.c_int, array('i', [keys]))
        return (c_keys, c_handle_array([vals]), use_str_keys)
    else:
        for value in vals:
            assert(isinstance(value, NDArray))
        c_keys = c_str_array([keys] * len(vals)) if use_str_keys \
                 else c_array_buf(ctypes.c_int, array('i', [keys] * len(vals)))
        return (c_keys, c_handle_array(vals), use_str_keys)
1030626
def internal_reader(self, input_stream):
        
        for input_line in input_stream:
            yield list(map(self.deserialize, input_line.split("\t")))
62858
def _read_data(file_obj, fo_encoding, value_count, bit_width):
    
    vals = []
    if fo_encoding == parquet_thrift.Encoding.RLE:
        seen = 0
        while seen < value_count:
            values = encoding.read_rle_bit_packed_hybrid(file_obj, bit_width)
            if values is None:
                break  # EOF was reached.
            vals += values
            seen += len(values)
    elif fo_encoding == parquet_thrift.Encoding.BIT_PACKED:
        raise NotImplementedError("Bit packing not yet supported")

    return vals
1292914
def _to_key_val_pairs(defs):
    

    if isinstance(defs, STRING_TYPES):
        # Convert 'a' to [('a', None)], or 'a.b.c' to [('a', 'b.c')]
        return [defs.split('.', 1) if '.' in defs else (defs, None)]
    else:
        pairs = []

        # Convert collections of strings or lists as above; break dicts into component items
        pairs.extend(p for s in defs if isinstance(s, STRING_TYPES) for p in _to_key_val_pairs(s))
        pairs.extend(p for l in defs if isinstance(l, list) for p in _to_key_val_pairs(l))
        pairs.extend(p for d in defs if isinstance(d, dict) for p in iteritems(d))

        return pairs
1237483
def read_dict_from_properties(desired_type: Type[dict], file_object: TextIOBase,
                              logger: Logger, conversion_finder: ConversionFinder, **kwargs) -> Dict[str, Any]:
    

    # right now jprops relies on a byte stream. So we convert back our nicely decoded Text stream to a unicode
    # byte stream ! (urgh)
    class Unicoder:
        def __init__(self, file_object):
            self.f = file_object

        def __iter__(self):
            return self

        def __next__(self):
            line = self.f.__next__()
            return line.encode(encoding='utf-8')

    res = jprops.load_properties(Unicoder(file_object))

    # first automatic conversion of strings > numbers
    res = {key: try_parse_num_and_booleans(val) for key, val in res.items()}

    # further convert if required
    return ConversionFinder.convert_collection_values_according_to_pep(res, desired_type, conversion_finder, logger, 
                                                                       **kwargs)
148214
def read_file(input_filename):
        ''''''
        with open(input_filename) as f:
            g = GreenGenesTaxonomy.read(f)
        return g
126319
def process_inputs(self):
        
        ret = []
        files = self.args.files
        if files is None:
            return ret

        for fname in files:
            if fname == '-':
                if self.args.base64stdin:
                    for line in sys.stdin:
                        data = base64.b64decode(line)
                        ret.append(self.process_file(data, fname))

                    continue
                else:
                    fh = sys.stdin

            elif fname.endswith('.tar') or fname.endswith('.tar.gz'):
                sub = self.process_tar(fname)
                ret.append(sub)
                continue

            elif not os.path.isfile(fname):
                sub = self.process_dir(fname)
                ret.append(sub)
                continue

            else:
                fh = open(fname, 'rb')

            with fh:
                data = fh.read()
                sub = self.process_file(data, fname)
                ret.append(sub)

        return ret
931272
def parse_file(self, sourcepath):
        """"""

        # Open input file and read JSON array:
        with open(sourcepath, 'r') as logfile:
            jsonstr = logfile.read()

        # Set our attributes for this entry and add it to data.entries:
        data = {}
        data['entries'] = json.loads(jsonstr)
        if self.tzone:
            for e in data['entries']:
                e['tzone'] = self.tzone

        # Return the parsed data
        return data
946801
def _calculate_mapping_reads(items, work_dir, input_backs=None):
    
    out_file = os.path.join(work_dir, "mapping_reads.txt")
    if not utils.file_exists(out_file):
        lines = []
        for data in items:
            count = 0
            for line in subprocess.check_output([
                "samtools", "idxstats", dd.get_align_bam(data)]).decode().split("\n"):
                if line.strip():
                    count += int(line.split("\t")[2])
            lines.append("%s\t%s" % (dd.get_sample_name(data), count))
        with file_transaction(items[0], out_file) as tx_out_file:
            with open(tx_out_file, "w") as out_handle:
                out_handle.write("\n".join(lines) + "\n")
                if input_backs:
                    for input_back in input_backs:
                        with open(input_back) as in_handle:
                            for line in in_handle:
                                if len(line.split()) == 2:
                                    out_handle.write(line)
    return out_file
424677
def parse_files(self, req, name, field):
        """"""
        return core.get_value(req.FILES, name, field)
673652
def GenericPump(filenames, use_jppy=False, name="GenericPump", **kwargs):
    """"""
    if isinstance(filenames, str):
        filenames = [filenames]

    try:
        iter(filenames)
    except TypeError:
        log.critical("Don't know how to iterate through filenames.")
        raise TypeError("Invalid filenames.")

    extensions = set(os.path.splitext(fn)[1] for fn in filenames)

    if len(extensions) > 1:
        log.critical("Mixed filetypes, please use only files of the same type")
        raise IOError("Mixed filetypes.")

    extension = list(extensions)[0]

    io = {
        '.evt': EvtPump,
        '.h5': HDF5Pump,
        '.root': EventPump if use_jppy else AanetPump,
        '.dat': DAQPump,
        '.dqd': CLBPump,
    }

    if extension not in io:
        log.critical(
            "No pump found for file extension '{0}'".format(extension)
        )
        raise ValueError("Unknown filetype")

    missing_files = [fn for fn in filenames if not os.path.exists(fn)]
    if missing_files:
        if len(missing_files) == len(filenames):
            message = "None of the given files could be found."
            log.critical(message)
            raise SystemExit(message)
        else:
            log.warning(
                "The following files are missing and ignored: {}".format(
                    ', '.join(missing_files)
                )
            )

    input_files = set(filenames) - set(missing_files)

    if len(input_files) == 1:
        return io[extension](filename=filenames[0], name=name, **kwargs)
    else:
        return io[extension](filenames=filenames, name=name, **kwargs)
728922
def _parse(fileobj):
    """"""
    fileobj.seek(0)
    try:
        part = fileobj.read(2)
    except UnicodeDecodeError:
        part = ""

    if part == "#!":
        shebang = shlex.split(fileobj.readline().strip())
        if (platform.system() == "Windows" and
                len(shebang) and
                os.path.basename(shebang[0]) == "env"):
            return shebang[1:]

        return shebang

    return []
243338
def get_files(input_file):
  
  file_index = {}
  base_dir = None
  if os.path.isfile(input_file):
    file_index[input_file] = None
    base_dir = os.path.dirname(input_file)
  elif os.path.isdir(input_file):
    base_dir = input_file
    for inf in glob.glob(input_file + s.SBP_EXTENSION):
      file_index[os.path.abspath(inf)] = None
    for inf in glob.glob(input_file + '/*'):
      base, index = get_files(os.path.abspath(inf))
      z = file_index.copy()
      z.update(index)
      file_index = z
  return (base_dir, file_index)
13981
def read_csv(fname):
    
    values = defaultdict(list)
    with open(fname) as f:
        reader = csv.DictReader(f)
        for row in reader:
            for (k,v) in row.items():
                values[k].append(v)
    npvalues = {k: np.array(values[k]) for k in values.keys()}
    for k in npvalues.keys():
        for datatype in [np.int, np.float]:
            try:
                npvalues[k][:1].astype(datatype)
                npvalues[k] = npvalues[k].astype(datatype)
                break
            except:
                pass
    dao = DataAccessObject(npvalues)
    return dao
1304211
def parse_atfile_cat(input):
    
    with open(input[1:]) as f:
        catlist = []
        catdict = {}
        for line in f.readlines():
            if line[0] == '#' or not line.strip():
                continue

            lspl = line.split()
            if len(lspl) > 1:
                catdict[lspl[0]] = lspl[1:]
                catlist.append(lspl[1:])
            else:
                catdict[lspl[0]] = None
                catlist.append(None)

    return catlist, catdict
861782
def split_input(cls, mapper_spec):
    """"""
    filelists = mapper_spec.params[cls.FILES_PARAM]
    max_values_count = mapper_spec.params.get(cls.MAX_VALUES_COUNT_PARAM, -1)
    max_values_size = mapper_spec.params.get(cls.MAX_VALUES_SIZE_PARAM, -1)
    return [cls([0] * len(files), max_values_count, max_values_size)
            for files in filelists]