Return the starting position (in bytes) of a list of files by
    iteratively summing their lengths
188383
def get_file_starts(file_list):
    '''
    
    '''
    starts = []
    total = 0
    for i in file_list:
        starts.append(total)
        total += i['length']
    print starts
    return starts
695586
def _listFilesPosix(self) -> ['File']:
        

        find = "find %s -type f" % self._path
        output = check_output(args=find.split()).strip().decode().split(
            '\n')
        return output
728481
def _trim_files(files, trim_output):
    '''
    
    '''
    count = 100
    if not isinstance(trim_output, bool):
        count = trim_output

    if not(isinstance(trim_output, bool) and trim_output is False) and len(files) > count:
        files = files[:count]
        files.append("List trimmed after {0} files.".format(count))

    return files
67424
def convert(files, **kwargs):
    

    if len(files) < 1:
        raise ValueError("Files needs to be a non-empty list")

    if len(files) == 1:
        if os.path.isfile(files[0]):
            return files_to_pif(files, **kwargs)
        else:
            return directory_to_pif(files[0], **kwargs)
    else:
        return files_to_pif([x for x in files if os.path.isfile(x)], **kwargs)
182338
def _chunk_list(l, n):
    """"""
    for i in range(0, len(l), n):
        yield l[i:i + n]
457420
def _index_files(path):
        """"""
        with zipfile.ZipFile(path) as zf:
            names = sorted(zf.namelist())
            names = [nn for nn in names if nn.endswith(".tif")]
            names = [nn for nn in names if nn.startswith("SID PHA")]
            phasefiles = []
            for name in names:
                with zf.open(name) as pt:
                    fd = io.BytesIO(pt.read())
                    if SingleTifPhasics.verify(fd):
                        phasefiles.append(name)
            return phasefiles
211031
def tail(filename, number_of_bytes):
    """"""
    with open(filename, "rb") as f:
        if os.stat(filename).st_size > number_of_bytes:
            f.seek(-number_of_bytes, 2)
        return f.read()
1053270
def fast_combine_pairs(files, force_single, full_name, separators):
    
    files = sort_filenames(files)
    chunks = tz.sliding_window(10, files)
    pairs = [combine_pairs(chunk, force_single, full_name, separators) for chunk in chunks]
    pairs = [y for x in pairs for y in x]
    longest = defaultdict(list)
    # for each file, save the longest pair it is in
    for pair in pairs:
        for file in pair:
            if len(longest[file]) < len(pair):
                longest[file] = pair
    # keep only unique pairs
    longest = {tuple(sort_filenames(x)) for x in longest.values()}
    # ensure filenames are R1 followed by R2
    return [sort_filenames(list(x)) for x in longest]
1223640
function getFileList(dir, sort = true) {
    return new Promise((resolve, reject) => {
        readdir(dir, (error, files) => {
            if (error) {
                reject(error);
                return;
            }
            let resultantFilesList;

            if (sort) {
                const stripLength = (dir.substr(0, 2) === './') ? dir.length - 1 : dir.length + 1;
                let pathsUnified = files.map((pth => pth.substr(stripLength).replace(/[\\/]/gm, '-')));
                const temporaryIndex = {};
                files.forEach((file, i) => {
                    temporaryIndex[pathsUnified[i]] = file;
                });
                pathsUnified = pathsUnified.sort();
                const filesSorted = [];
                pathsUnified.forEach((key) => {
                    filesSorted.push(temporaryIndex[key]);
                });

                resultantFilesList = filesSorted;
            } else {
                resultantFilesList = files;
            }
            resolve(resultantFilesList);
        });
    });
}
128445
def get_filetypes(filelist, path=None, size=os.path.getsize):
    
    path = path or (lambda _: _)

    # Get total size for each file extension
    histo = defaultdict(int)
    for entry in filelist:
        ext = os.path.splitext(path(entry))[1].lstrip('.').lower()
        if ext and ext[0] == 'r' and ext[1:].isdigit():
            ext = "rar"
        elif ext == "jpeg":
            ext = "jpg"
        elif ext == "mpeg":
            ext = "mpg"
        histo[ext] += size(entry)

    # Normalize values to integer percent
    total = sum(histo.values())
    if total:
        for ext, val in histo.items():
            histo[ext] = int(val * 100.0 / total + .499)

    return sorted(zip(histo.values(), histo.keys()), reverse=True)
163589
def determine_packet_positions(self):
        """"""
        print("Analysing file...")
        self.rewind_file()
        with ignored(struct.error):
            while True:
                pointer_position = self.blob_file.tell()
                length = struct.unpack('<i', self.blob_file.read(4))[0]
                self.packet_positions.append(pointer_position)
                self.blob_file.seek(length, 1)
        self.rewind_file()
        print("Found {0} CLB UDP packets.".format(len(self.packet_positions)))
779902
def files_in_dir(path, extension):
    """"""
    ends = '.{0}'.format(extension)
    return (f for f in os.listdir(path) if f.endswith(ends))
1108225
def find_all_output_in_range(self, ifo, currSeg, useSplitLists=False):
        
        if not useSplitLists:
            # Slower, but simpler method
            outFiles = [i for i in self if ifo in i.ifo_list]
            outFiles = [i for i in outFiles \
                                      if i.segment_list.intersects_segment(currSeg)]
        else:
            # Faster, but more complicated
            # Basically only check if a subset of files intersects_segment by
            # using a presorted list. Sorting only happens once.
            if not self._check_split_list_validity():
                # FIXME: DO NOT hard code this.
                self._temporal_split_list(100)
            startIdx = int( (currSeg[0] - self._splitListsStart) / \
                                                          self._splitListsStep )
            # Add some small rounding here
            endIdx = (currSeg[1] - self._splitListsStart) / self._splitListsStep
            endIdx = int(endIdx - 0.000001)

            outFiles = []
            for idx in range(startIdx, endIdx + 1):
                if idx < 0 or idx >= self._splitListsNum:
                    continue
                outFilesTemp = [i for i in self._splitLists[idx] \
                                                            if ifo in i.ifo_list]
                outFiles.extend([i for i in outFilesTemp \
                                      if i.segment_list.intersects_segment(currSeg)])
                # Remove duplicates
                outFiles = list(set(outFiles))

        return self.__class__(outFiles)
437478
private FileStatus[] iterativeListing(String src) throws IOException {
    // fetch the first batch of entries in the directory
    DirectoryListing thisListing = namenode.getPartialListing(
        src, HdfsFileStatus.EMPTY_NAME);

    if (thisListing == null) { // the directory does not exist
      return null;
     }
    HdfsFileStatus[] partialListing = thisListing.getPartialListing();
    if (!thisListing.hasMore()) { // got all entries of the directory
      FileStatus[] stats = new FileStatus[partialListing.length];
      for (int i = 0; i < partialListing.length; i++) {
        stats[i] = HdfsFileStatus.toFileStatus(partialListing[i], src);
      }
      return stats;
    }

    // The directory size is too big that it needs to fetch more
    // estimate the total number of entries in the directory
    int totalNumEntries =
      partialListing.length + thisListing.getRemainingEntries();
    ArrayList<FileStatus> listing =
      new ArrayList<FileStatus>(totalNumEntries);
    // add the first batch of entries to the array list
    for (HdfsFileStatus fileStatus : partialListing) {
      listing.add(HdfsFileStatus.toFileStatus(fileStatus, src));
    }

    // now fetch more entries
    do {
      thisListing = namenode.getPartialListing(src, thisListing.getLastName());

      if (thisListing == null) {
        return null; // the directory is deleted
      }

      partialListing = thisListing.getPartialListing();
      for (HdfsFileStatus fileStatus : partialListing) {
        listing.add(HdfsFileStatus.toFileStatus(fileStatus, src));
      }
    } while (thisListing.hasMore());

    return listing.toArray(new FileStatus[listing.size()]);
  }
1055583
def get_next_want_file(self, byte_index, block):
        '''
        
        '''
        while block:
            rightmost = get_rightmost_index(byte_index=byte_index,
                                            file_starts=self.file_starts)
            if rightmost in self.want_file_pos:
                return rightmost, byte_index, block
            else:
                    file_start = (self.file_starts
                                  [rightmost])
                    file_length = self.file_list[rightmost]['length']
                    bytes_rem = file_start + file_length - byte_index
                    if len(block) > bytes_rem:
                        block = block[bytes_rem:]
                        byte_index = byte_index + bytes_rem
                    else:
                        block = ''
        else:
            return None
700049
def get_input_sequence_files(data, default=None):
    
    if "files" not in data or data.get("files") is None:
        file1, file2 = None, None
    elif len(data["files"]) == 2:
        file1, file2 = data["files"]
    else:
        assert len(data["files"]) == 1, data["files"]
        file1, file2 = data["files"][0], None
    return file1, file2
705112
def get_starting_chunk(filename, length=1024):
    
    # Ensure we open the file in binary mode
    try:
        with open(filename, 'rb') as f:
            chunk = f.read(length)
            return chunk
    except IOError as e:
        print(e)