Create & concatenate video clips in groups of size BATCH_SIZE and output
    finished video file to output directory.
329688
def create_supercut_in_batches(composition, outputfile, padding):
    
    total_clips = len(composition)
    start_index = 0
    end_index = BATCH_SIZE
    batch_comp = []
    while start_index < total_clips:
        filename = outputfile + '.tmp' + str(start_index) + '.mp4'
        try:
            create_supercut(composition[start_index:end_index], filename, padding)
            batch_comp.append(filename)
            gc.collect()
            start_index += BATCH_SIZE
            end_index += BATCH_SIZE
        except:
            start_index += BATCH_SIZE
            end_index += BATCH_SIZE
            next

    clips = [VideoFileClip(filename) for filename in batch_comp]
    video = concatenate(clips)
    video.to_videofile(outputfile, codec="libx264", temp_audiofile='temp-audio.m4a', remove_temp=True, audio_codec='aac')


    # remove partial video files
    for filename in batch_comp:
        os.remove(filename)

    cleanup_log_files(outputfile)
1302642
def generate_files_distributed(generator,
                               output_name,
                               output_dir,
                               num_shards=1,
                               max_cases=None,
                               task_id=0):
  """"""
  assert task_id < num_shards
  output_filename = sharded_name(output_name, task_id, num_shards)
  output_file = os.path.join(output_dir, output_filename)
  tf.logging.info("Writing to file %s", output_file)
  writer = tf.python_io.TFRecordWriter(output_file)

  counter = 0
  for case in generator:
    if counter % 100000 == 0:
      tf.logging.info("Generating case %d for %s." % (counter, output_name))
    counter += 1
    if max_cases and counter > max_cases:
      break
    example = to_example(case)
    writer.write(example.SerializeToString())

  writer.close()
  return output_file
1018869
private function create_big_files() {
        // Work out how many files and how many blocks to use (up to 64KB).
        $count = self::$parambigfilecount[$this->size];
        $filesize = $this->limit_filesize(self::$parambigfilesize[$this->size]);
        $blocks = ceil($filesize / 65536);
        $blocksize = floor($filesize / $blocks);

        $this->log('createbigfiles', $count, true);

        // Prepare temp area.
        $tempfolder = make_temp_directory('tool_generator');
        $tempfile = $tempfolder . '/' . rand();

        // Create resources and files.
        $fs = get_file_storage();
        $resourcegenerator = $this->generator->get_plugin_generator('mod_resource');
        for ($i = 0; $i < $count; $i++) {
            // Create resource.
            $record = array('course' => $this->course,
                    'name' => get_string('bigfile', 'tool_generator', $i));
            $options = array('section' => $this->get_target_section());
            $resource = $resourcegenerator->create_instance($record, $options);

            // Write file.
            $handle = fopen($tempfile, 'w');
            if (!$handle) {
                throw new coding_exception('Failed to open temporary file');
            }
            for ($j = 0; $j < $blocks; $j++) {
                $data = random_bytes_emulate($blocksize);
                fwrite($handle, $data);
                $this->dot($i * $blocks + $j, $count * $blocks);
            }
            fclose($handle);

            // Add file.
            $context = context_module::instance($resource->cmid);
            $filerecord = array('component' => 'mod_resource', 'filearea' => 'content',
                    'contextid' => $context->id, 'itemid' => 0, 'filepath' => '/',
                    'filename' => 'bigfile' . $i . '.dat');
            $fs->create_file_from_pathname($filerecord, $tempfile);
        }

        unlink($tempfile);
        $this->end_log();
    }
168834
def create_m3u_playlist(section_dir):
    
    path_to_return = os.getcwd()

    for (_path, subdirs, files) in os.walk(section_dir):
        os.chdir(_path)
        globbed_videos = sorted(glob.glob("*.mp4"))
        m3u_name = os.path.split(_path)[1] + ".m3u"

        if len(globbed_videos):
            with open(m3u_name, "w") as m3u:
                for video in globbed_videos:
                    m3u.write(video + "\n")
            os.chdir(path_to_return)
    os.chdir(path_to_return)
1227954
def prt_outfiles_grouped(self, outfiles):
        """"""
        for outfile in outfiles:
            if outfile.endswith(".xlsx"):
                self.wr_xlsx(outfile)
            elif outfile.endswith(".txt"):
                self.wr_txt(outfile)
            else:
                self.wr_tsv(outfile)
917444
def _create_batches(self, X, batch_size, shuffle_data=True):
        
        if shuffle_data:
            X = shuffle(X)

        if batch_size > X.shape[0]:
            batch_size = X.shape[0]

        max_x = int(np.ceil(X.shape[0] / batch_size))
        X = np.resize(X, (max_x, batch_size, X.shape[-1]))

        return X
1020629
def output_files(self):
        """"""
        for dep in self.subgraph.successors(self.address):
            dep_rule = self.subgraph.node[dep]['target_obj']
            for out_file in dep_rule.output_files:
                yield out_file
1164622
def file_writer(self, in_frame):
        """"""
        self.update_config()
        path = self.config['path']
        encoder = self.config['encoder']
        fps = self.config['fps']
        bit16 = self.config['16bit']
        numpy_image = in_frame.as_numpy()
        ylen, xlen, bpc = numpy_image.shape
        if bpc == 3:
            if in_frame.type != 'RGB':
                self.logger.warning('Expected RGB input, got %s', in_frame.type)
            pix_fmt = ('rgb24', 'rgb48le')[bit16]
        elif bpc == 1:
            if in_frame.type != 'Y':
                self.logger.warning('Expected Y input, got %s', in_frame.type)
            pix_fmt = ('gray', 'gray16le')[bit16]
        else:
            self.logger.critical(
                'Cannot write %s frame with %d components', in_frame.type, bpc)
            return
        md = Metadata().copy(in_frame.metadata)
        audit = md.get('audit')
        audit += '%s = data\n' % path
        audit += '    encoder: "%s"\n' % (encoder)
        audit += '    16bit: %s\n' % (self.config['16bit'])
        md.set('audit', audit)
        md.to_file(path)
        with self.subprocess(
                ['ffmpeg', '-v', 'warning', '-y', '-an',
                 '-s', '%dx%d' % (xlen, ylen),
                 '-f', 'rawvideo', '-c:v', 'rawvideo',
                 '-r', '%d' % fps, '-pix_fmt', pix_fmt, '-i', '-',
                 '-r', '%d' % fps] + encoder.split() + [path],
                stdin=subprocess.PIPE) as sp:
            while True:
                in_frame = yield True
                if not in_frame:
                    break
                if bit16:
                    numpy_image = in_frame.as_numpy(dtype=pt_float)
                    numpy_image = numpy_image * pt_float(256.0)
                    numpy_image = numpy_image.clip(
                        pt_float(0), pt_float(2**16 - 1)).astype(numpy.uint16)
                else:
                    numpy_image = in_frame.as_numpy(dtype=numpy.uint8)
                sp.stdin.write(numpy_image.tostring())
                del in_frame
751603
def _split_batches(self, data, batch_size):
        """"""
        for i in range(0, len(data), batch_size):
            yield data[i : i + batch_size]
1061394
def _batchify(self, batch_data, batch_label, start=0):
        """"""
        i = start
        batch_size = self.batch_size
        try:
            while i < batch_size:
                label, s = self.next_sample()
                data = self.imdecode(s)
                try:
                    self.check_valid_image(data)
                except RuntimeError as e:
                    logging.debug('Invalid image, skipping:  %s', str(e))
                    continue
                data = self.augmentation_transform(data)
                assert i < batch_size, 'Batch size must be multiples of augmenter output length'
                batch_data[i] = self.postprocess_data(data)
                batch_label[i] = label
                i += 1
        except StopIteration:
            if not i:
                raise StopIteration
        return i
992162
def samples_to_batches(samples: Iterable, batch_size: int):
        """"""
        it = iter(samples)
        while True:
            with suppress(StopIteration):
                batch_in, batch_out = [], []
                for i in range(batch_size):
                    sample_in, sample_out = next(it)
                    batch_in.append(sample_in)
                    batch_out.append(sample_out)
            if not batch_in:
                raise StopIteration
            yield np.array(batch_in), np.array(batch_out)
998694
def create_batch(cls, size, **kwargs):
        
        return [cls.create(**kwargs) for _ in range(size)]
786922
def _singleframe_to_nifti(grouped_dicoms, output_file):
    

    # Create mosaic block
    logger.info('Creating data block')
    full_block = _singleframe_to_block(grouped_dicoms)

    logger.info('Creating affine')
    # Create the nifti header info
    affine, slice_increment = common.create_affine(grouped_dicoms[0])

    logger.info('Creating nifti')
    # Convert to nifti
    nii_image = nibabel.Nifti1Image(full_block, affine)
    common.set_tr_te(nii_image, float(grouped_dicoms[0][0].RepetitionTime), float(grouped_dicoms[0][0].EchoTime))

    if output_file is not None:
        # Save to disk
        logger.info('Saving nifti to disk %s' % output_file)
        nii_image.to_filename(output_file)

    if _is_singleframe_diffusion_imaging(grouped_dicoms):
        bval_file = None
        bvec_file = None
        # Create the bval en bvec files
        if output_file is not None:
            base_name = os.path.splitext(output_file)[0]
            if base_name.endswith('.nii'):
                base_name = os.path.splitext(base_name)[0]

            logger.info('Creating bval en bvec files')
            bval_file = '%s.bval' % base_name
            bvec_file = '%s.bvec' % base_name
        nii_image, bval, bvec, bval_file, bvec_file = _create_singleframe_bvals_bvecs(grouped_dicoms,
                                                                                      bval_file,
                                                                                      bvec_file,
                                                                                      nii_image,
                                                                                      output_file)

        return {'NII_FILE': output_file,
                'BVAL_FILE': bval_file,
                'BVEC_FILE': bvec_file,
                'NII': nii_image,
                'BVAL': bval,
                'BVEC': bvec,
                'MAX_SLICE_INCREMENT': slice_increment}

    return {'NII_FILE': output_file,
            'NII': nii_image,
            'MAX_SLICE_INCREMENT': slice_increment}
440915
def __init_ffmpeg(self, image_shape):
    """"""
    import itertools  # pylint: disable=g-import-not-at-top
    from subprocess import Popen, PIPE  # pylint: disable=g-import-not-at-top,g-multiple-import,g-importing-member
    ffmpeg = "ffmpeg"
    height, width, channels = image_shape
    self.cmd = [
        ffmpeg, "-y",
        "-f", "rawvideo",
        "-vcodec", "rawvideo",
        "-r", "%.02f" % self.fps,
        "-s", "%dx%d" % (width, height),
        "-pix_fmt", {1: "gray", 3: "rgb24"}[channels],
        "-i", "-",
        "-filter_complex", "[0:v]split[x][z];[x]fifo[w];[z]palettegen,fifo[y];"
                           "[w][y]paletteuse,fifo",
        "-r", "%.02f" % self.fps,
        "-f", self.file_format,
        "-qscale", "0",
        "-"
    ]
    self.proc = Popen(
        self.cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE, bufsize=-1
    )
    (self._out_thread, self._err_thread) = itertools.starmap(
        self._start_reader_thread, [
            (self.proc.stdout, self._out_chunks),
            (self.proc.stderr, self._err_chunks)
        ]
    )
942069
def i_batch(items, size):
    
    for items_batch in iter(lambda: tuple(islice(items, size)),
                            tuple()):
        yield items_batch
454691
def skip(self, video_name, out, start, duration):
        
        cfilter = (r"[0:v]trim=duration={start}[av];"
                   r"[0:a]atrim=duration={start}[aa];"
                   r"[0:v]trim=start={offset},setpts=PTS-STARTPTS[bv];"
                   r"[0:a]atrim=start={offset},asetpts=PTS-STARTPTS[ba];"
                   r"[av][bv]concat[outv];[aa][ba]concat=v=0:a=1[outa]")\
            .format(start=start, offset=start + duration)\
            .replace(' ', '')

        command = ['ffmpeg', '-i', video_name,
            '-filter_complex', cfilter, '-y',
            '-map', '[outv]',
            '-map', '[outa]', '-c:v', 'huffyuv', '-preset', 'veryslow',
            out]

        if self.verbose:
            print 'Skipping {0} into {1}'.format(video_name, out)
            print ' '.join(command)

        call(command)
1045549
def main(args=None):
    """"""

    dlx = Downloader()
    epilog = dlx.epilog()

    options, arguments = parse(args, epilog)

    # create list of video files that don't have accompanying 'srt' subtitles
    targets = [p for p in arguments if os.path.isfile(p) and not
               os.path.exists(os.path.splitext(p)[0] + '.srt')]

    if not targets:
        fatal('No valid targets were specified')

    if options.rename:
        videos = [rename(p) for p in targets]
    else:
        videos = targets

    dlx.download(videos, options.downloader, options.language)

    subtitles = []

    for video_path in videos:
        srt_path = os.path.splitext(video_path)[0] + '.srt'
        if os.path.isfile(srt_path):
            subtitles.append(srt_path)
        elif not options.quiet:
            failure(video_path, options.downloader)

    if options.scan and subtitles:
        scan(subtitles)