load a partition from disk, then sort and group by key
559621
def _merge_sorted_items(self, index):
        
        def load_partition(j):
            path = self._get_spill_dir(j)
            p = os.path.join(path, str(index))
            with open(p, 'rb', 65536) as f:
                for v in self.serializer.load_stream(f):
                    yield v

        disk_items = [load_partition(j) for j in range(self.spills)]

        if self._sorted:
            # all the partitions are already sorted
            sorted_items = heapq.merge(disk_items, key=operator.itemgetter(0))

        else:
            # Flatten the combined values, so it will not consume huge
            # memory during merging sort.
            ser = self.flattened_serializer()
            sorter = ExternalSorter(self.memory_limit, ser)
            sorted_items = sorter.sorted(itertools.chain(*disk_items),
                                         key=operator.itemgetter(0))
        return ((k, vs) for k, vs in GroupByKey(sorted_items))
1157049
def sort_group(d, return_only_first=False):
    '''  '''
    # First, sort the paths in order (this must be a couple: (parent_dir, filename), so that there's no ambiguity because else a file at root will be considered as being after a folder/file since the ordering is done alphabetically without any notion of tree structure).
    d_sort = sort_dict_of_paths(d)
    # Pop the first item in the ordered list
    base_elt = (-1, None)
    while (base_elt[1] is None and d_sort):
        base_elt = d_sort.pop(0)
    # No element, then we just return
    if base_elt[1] is None:
        return None
    # Else, we will now group equivalent files together (remember we are working on multiple directories, so we can have multiple equivalent relative filepaths, but of course the absolute filepaths are different).
    else:
        # Init by creating the first group and pushing the first ordered filepath into the first group
        lst = []
        lst.append([base_elt])
        if d_sort:
            # For each subsequent filepath
            for elt in d_sort:
                # If the filepath is not empty (generator died)
                if elt[1] is not None:
                    # If the filepath is the same to the latest grouped filepath, we add it to the same group
                    if elt[1] == base_elt[1]:
                        lst[-1].append(elt)
                    # Else the filepath is different: we create a new group, add the filepath to this group, and replace the latest grouped filepath
                    else:
                        if return_only_first: break  # break here if we only need the first group
                        lst.append([elt])
                        base_elt = elt # replace the latest grouped filepath
        return lst
361375
def envdict2listdict(envdict):
    """"""
    sep = os.path.pathsep
    for key in envdict:
        if sep in envdict[key]:
            envdict[key] = [path.strip() for path in envdict[key].split(sep)]
    return envdict
80900
def sort(self, key, *get_patterns,
             by=None, offset=None, count=None,
             asc=None, alpha=False, store=None):
        """"""
        args = []
        if by is not None:
            args += [b'BY', by]
        if offset is not None and count is not None:
            args += [b'LIMIT', offset, count]
        if get_patterns:
            args += sum(([b'GET', pattern] for pattern in get_patterns), [])
        if asc is not None:
            args += [asc is True and b'ASC' or b'DESC']
        if alpha:
            args += [b'ALPHA']
        if store is not None:
            args += [b'STORE', store]
        return self.execute(b'SORT', key, *args)
375697
def load(self, key, noexpire=None):
        ''''''
        with self.load_fd(key, noexpire=noexpire) as fd:
            return fd.read()
772747
def rmpart(self, disk, number):
        
        args = {
            'disk': disk,
            'number': number,
        }

        self._rmpart_chk.check(args)

        response = self._client.raw('disk.rmpart', args)

        result = response.get()

        if result.state != 'SUCCESS':
            raise RuntimeError('failed to remove partition: %s' % result.stderr)
838032
def _split_dict(dic):
    '''
    '''
    keys = sorted(dic.keys())
    return keys, [dic[k] for k in keys]
272499
def group_by(resources, key):
    
    resource_map = {}
    parts = key.split('.')
    for r in resources:
        v = r
        for k in parts:
            v = v.get(k)
            if not isinstance(v, dict):
                break
        resource_map.setdefault(v, []).append(r)
    return resource_map
17833
private static int partitionForKey(byte[] key, int numPartitions) {
        CRC32 crc32 = new CRC32();
        crc32.update(key);
        long rv = (crc32.getValue() >> 16) & 0x7fff;
        return (int) rv &numPartitions - 1;
    }
733960
def keys(self, key=None, reverse=False):
        """"""
        ks = sorted(list(dict.keys(self)), key=key, reverse=reverse)
        return ks
598940
def _load_from_disk(self, dtype_out_time, dtype_out_vert=False,
                        region=False):
        """"""
        ds = xr.open_dataset(self.path_out[dtype_out_time])
        if region:
            arr = ds[region.name]
            # Use region-specific pressure values if available.
            if (self.dtype_in_vert == internal_names.ETA_STR
                    and not dtype_out_vert):
                reg_pfull_str = region.name + '_pressure'
                arr = arr.drop([r for r in arr.coords.iterkeys()
                                if r not in (internal_names.PFULL_STR,
                                             reg_pfull_str)])
                # Rename pfull to pfull_ref always.
                arr = arr.rename({internal_names.PFULL_STR:
                                  internal_names.PFULL_STR + '_ref'})
                # Rename region_pfull to pfull if its there.
                if hasattr(arr, reg_pfull_str):
                    return arr.rename({reg_pfull_str:
                                       internal_names.PFULL_STR})
                return arr
            return arr
        return ds[self.name]
1372374
def rm_keys_from_dict(d, keys):
    
    # Loop for each key given
    for key in keys:
        # Is the key in the dictionary?
        if key in d:
            try:
                d.pop(key, None)
            except KeyError:
                # Not concerned with an error. Keep going.
                pass
    return d
740342
def group(iterable, key):
    
    for _, grouped in groupby(sorted(iterable, key=key), key=key):
        yield list(grouped)
1240912
def sort_dict(self, data, key):
    ''''''
    return sorted(data, key=itemgetter(key)) if data else []
739931
def sort_by(self, *ids):
        
        # Support sorting by file_ids or keys.
        files = {str(f_.file_id): f_.key for f_ in self}
        # self.record['_files'] = [{'key': files.get(id_, id_)} for id_ in ids]
        self.filesmap = OrderedDict([
            (files.get(id_, id_), self[files.get(id_, id_)].dumps())
            for id_ in ids
        ])
        self.flush()
406271
def gather_data(mypath, key="width"):
    
    filenames = [join(mypath, f) for f in listdir(mypath) if isfile(join(mypath, f))]
    filenames = [f for f in filenames if f.endswith(".p")] 
    W = []
    for filename in filenames:
        with open(filename, 'r') as f:
            data = pickle.load(f)
            W.append(np.expand_dims(data[key], axis=-1) )

    W = np.concatenate(W, axis=-1)
    return W
405090
def cli(env, volume_id, sortby, columns):
    """"""
    block_manager = SoftLayer.BlockStorageManager(env.client)
    snapshots = block_manager.get_block_volume_snapshot_list(
        volume_id,
        mask=columns.mask()
    )

    table = formatting.Table(columns.columns)
    table.sortby = sortby

    for snapshot in snapshots:
        table.add_row([value or formatting.blank()
                       for value in columns.row(snapshot)])

    env.fout(table)