Return a concretization of the underlying files, as a tuple of (read file, write file).
325682
def concretize(self, **kwargs):
        
        return (self._read_file.concretize(**kwargs), self._write_file.concretize(**kwargs))
965484
def iter_sync(fileobj, max_read):
    

    read = 0
    size = 2
    last_byte = b""
    is_second = lambda b: ord(b) & 0xe0 == 0xe0

    while read < max_read:
        data_offset = fileobj.tell()
        new_data = fileobj.read(min(max_read - read, size))
        if not new_data:
            return
        read += len(new_data)

        if last_byte == b"\xff" and is_second(new_data[0:1]):
            fileobj.seek(data_offset - 1, 0)
            yield

        size *= 2
        last_byte = new_data[-1:]

        find_offset = 0
        while True:
            index = new_data.find(b"\xff", find_offset)
            # if not found or the last byte -> read more
            if index == -1 or index == len(new_data) - 1:
                break

            if is_second(new_data[index + 1:index + 2]):
                fileobj.seek(data_offset + index, 0)
                yield
            find_offset = index + 1

        fileobj.seek(data_offset + len(new_data), 0)
1012023
def tb_c_file_creation(target, source, env):
    """"""

    files = [str(x) for x in source]

    try:
        desc = TBDescriptor(files)
    except pyparsing.ParseException as e:
        raise BuildError("Could not parse tilebus file", parsing_exception=e)

    block = desc.get_block()
    block.render_template(block.CommandFileTemplate, out_path=str(target[0]))
    block.render_template(block.CommandHeaderTemplate, out_path=str(target[1]))
    block.render_template(block.ConfigFileTemplate, out_path=str(target[2]))
    block.render_template(block.ConfigHeaderTemplate, out_path=str(target[3]))
1353157
def create_files(filedef, cleanup=True):
    
    cwd = os.getcwd()
    tmpdir = tempfile.mkdtemp()
    try:
        Filemaker(tmpdir, filedef)
        if not cleanup:  # pragma: nocover
            pass
            # print("TMPDIR =", tmpdir)
        os.chdir(tmpdir)
        yield tmpdir
    finally:
        os.chdir(cwd)
        if cleanup:  # pragma: nocover
            shutil.rmtree(tmpdir, ignore_errors=True)
937611
def combine(in_files, out_file, config):
    
    if not utils.file_exists(out_file):
        with file_transaction(config, out_file) as tx_out_file:
            with open(tx_out_file, "w") as out_handle:
                for in_file in in_files:
                    with open(in_file) as in_handle:
                        shutil.copyfileobj(in_handle, out_handle)
    return out_file
800502
async def transform(self, file):
		""""""
		path = get_path(self.dest, file)
		await self.loop.run_in_executor(self.exe, write_file, path, file.contents)
		return file
1007403
def makefile(self, sock, mode='r', bufsize=DEFAULT_BUFFER_SIZE):
        """"""
        cls = StreamReader if 'r' in mode else StreamWriter
        return cls(sock, mode, bufsize)
1312035
def fromfile(file, dtype, count, *args, **kwargs):
    """"""

    try:
        return np.fromfile(file, dtype=dtype, count=count, *args, **kwargs)
    except (TypeError, IOError, UnsupportedOperation):
        return np.frombuffer(
            file.read(count * np.dtype(dtype).itemsize),
            dtype=dtype, count=count, *args, **kwargs)
1272292
def get_file(f, decoder, read):
    """"""
    with f as f:
        if decoder is None:
            return list(f)
        else:
            d = f.read() if read else f
            out = decoder(d)
            if isinstance(out, (tuple, list)):
                return out
            else:
                return [out]
667177
def read(self, path, ext=None, start=None, stop=None, recursive=False, npartitions=None):
        
        path = uri_to_path(path)
        files = self.list(path, ext=ext, start=start, stop=stop, recursive=recursive)

        nfiles = len(files)
        self.nfiles = nfiles

        if spark and isinstance(self.engine, spark):
            npartitions = min(npartitions, nfiles) if npartitions else nfiles
            rdd = self.engine.parallelize(enumerate(files), npartitions)
            return rdd.map(lambda kv: (kv[0], readlocal(kv[1]), kv[1]))
        else:
            return [(k, readlocal(v), v) for k, v in enumerate(files)]
1317705
def constant_outfile_iterator(outfiles, infiles, arggroups):
    """"""
    assert len(infiles) == 1
    assert len(arggroups) == 1

    return ((outfile, infiles[0], arggroups[0]) for outfile in outfiles)
265983
def atomic_write(path, mode):
  """"""
  tmp_path = "%s%s_%s" % (path, constants.INCOMPLETE_SUFFIX, uuid.uuid4().hex)
  with tf.io.gfile.GFile(tmp_path, mode) as file_:
    yield file_
  tf.io.gfile.rename(tmp_path, path, overwrite=True)
1080574
def generate(self, overwrite=False):
        
        super(Upstart, self).generate(overwrite=overwrite)

        svc_file_template = self.template_prefix + '.conf'
        self.svc_file_path = self.generate_into_prefix + '.conf'

        self.generate_file_from_template(svc_file_template, self.svc_file_path)
        return self.files
932838
def read(*paths):
    """"""
    basedir = os.path.dirname(__file__)
    fullpath = os.path.join(basedir, *paths)
    contents = io.open(fullpath, encoding='utf-8').read().strip()
    return contents
253809
def file_chunks(self, fp):
        
        fsize = utils.file_size(fp)
        offset = 0
        if hasattr(fp, 'readinto'):
            while offset < fsize:
                nb = fp.readinto(self._internal)
                yield self.buf[:nb]
                offset += nb
        else:
            while offset < fsize:
                nb = min(self.chunk_size, fsize - offset)
                yield fp.read(nb)
                offset += nb
405451
def combine_and_copy(f, files, group):
    
    f[group] = np.concatenate([fi[group][:] if group in fi else \
                                   np.array([], dtype=np.uint32) for fi in files])
24003
def read_examples(input_files, batch_size, shuffle, num_epochs=None):
  """"""
  files = []
  for e in input_files:
    for path in e.split(','):
      files.extend(file_io.get_matching_files(path))
  thread_count = multiprocessing.cpu_count()

  # The minimum number of instances in a queue from which examples are drawn
  # randomly. The larger this number, the more randomness at the expense of
  # higher memory requirements.
  min_after_dequeue = 1000

  # When batching data, the queue's capacity will be larger than the batch_size
  # by some factor. The recommended formula is (num_threads + a small safety
  # margin). For now, we use a single thread for reading, so this can be small.
  queue_size_multiplier = thread_count + 3

  # Convert num_epochs == 0 -> num_epochs is None, if necessary
  num_epochs = num_epochs or None

  # Build a queue of the filenames to be read.
  filename_queue = tf.train.string_input_producer(files, num_epochs, shuffle)

  options = tf.python_io.TFRecordOptions(
      compression_type=tf.python_io.TFRecordCompressionType.GZIP)
  example_id, encoded_example = tf.TFRecordReader(options=options).read_up_to(
      filename_queue, batch_size)

  if shuffle:
    capacity = min_after_dequeue + queue_size_multiplier * batch_size
    return tf.train.shuffle_batch(
        [example_id, encoded_example],
        batch_size,
        capacity,
        min_after_dequeue,
        enqueue_many=True,
        num_threads=thread_count)
  else:
    capacity = queue_size_multiplier * batch_size
    return tf.train.batch(
        [example_id, encoded_example],
        batch_size,
        capacity=capacity,
        enqueue_many=True,
        num_threads=thread_count)