generates a report showing nine slices, three per axis, of an
        arbitrary volume of `in_files`, with the resulting segmentation
        overlaid
549313
def _post_run_hook(self, runtime):
        '''  '''
        outputs = self.aggregate_outputs(runtime=runtime)
        self._melodic_dir = outputs.out_dir

        NIWORKFLOWS_LOG.info('Generating report for MELODIC')

        return super(MELODICRPT, self)._post_run_hook(runtime)
742807
def collect_output(mr_out_dir, out_file=None):
    
    if out_file is None:
        output = []
        for fn in iter_mr_out_files(mr_out_dir):
            with hdfs.open(fn, "rt") as f:
                output.append(f.read())
        return "".join(output)
    else:
        block_size = 16777216
        with open(out_file, 'a') as o:
            for fn in iter_mr_out_files(mr_out_dir):
                with hdfs.open(fn) as f:
                    data = f.read(block_size)
                    while len(data) > 0:
                        o.write(data)
                        data = f.read(block_size)
807210
def _make_dataset(cls, coords):
        
        class Slice(cls._SliceType):
            extra_coords = coords

        Slice.__name__ = '%s.slice(%s)' % (
            cls.__name__,
            ', '.join('%s=%r' % item for item in coords.items()),
        )
        return Slice
1095457
def get_X_slice(self, rmind):
        ''''''
        if self.forest.optimize_memory_use:
            return self.forest.X[:, rmind][self.subsample]
        else:
            return self.X[:, rmind]
606355
def _PartitionChunks(chunks):
  """"""
  partitions = [[]]
  partition_size = 0

  for chunk in chunks:
    cursize = len(chunk["blob_chunk"])
    if (cursize + partition_size > BLOB_CHUNK_SIZE or
        len(partitions[-1]) >= CHUNKS_PER_INSERT):
      partitions.append([])
      partition_size = 0
    partitions[-1].append(chunk)
    partition_size += cursize

  return partitions
701468
def calculate_split_output_shapes(operator):
    '''
    
    '''
    check_input_and_output_numbers(operator, input_count_range=1, output_count_range=[1, None])
    check_input_and_output_types(operator, good_input_types=[FloatTensorType])

    output_shape = copy.deepcopy(operator.inputs[0].type.shape)

    divided = output_shape[1] / operator.raw_operator.split.nOutputs
    if divided != int(divided):
        raise RuntimeError('Variable dimension along C-axis must be divisible by partition number')

    output_shape[1] = int(divided)

    for i in range(len(operator.outputs)):
        operator.outputs[i].type.shape = copy.deepcopy(output_shape)
749698
def _slice_data(self, source_area, slices, dataset):
        """"""
        slice_x, slice_y = slices
        dataset = dataset.isel(x=slice_x, y=slice_y)
        assert ('x', source_area.x_size) in dataset.sizes.items()
        assert ('y', source_area.y_size) in dataset.sizes.items()
        dataset.attrs['area'] = source_area

        return dataset
1045725
def parse_volumes_output(out):
    
    if not out:
        return []
    line_iter = islice(out.splitlines(), 1, None)  # Skip header
    return list(map(_volume_info, line_iter))
143115
def write_how_many(self, file):
        
        report = CaseReport(self.case)

        # Map component labels to attribute names
        components = [("Bus", "n_buses"), ("Generator", "n_generators"),
            ("Committed Generator", "n_online_generators"),
            ("Load", "n_loads"), ("Fixed Load", "n_fixed_loads"),
            ("Despatchable Load", "n_online_vloads"), ("Shunt", "n_shunts"),
            ("Branch", "n_branches"), ("Transformer", "n_transformers"),
            ("Inter-tie", "n_interties"), ("Area", "n_areas")
        ]

        # Column 1 width
        longest = max([len(c[0]) for c in components])

        col1_header = "Object"
        col1_width = longest
        col2_header = "Quantity"
        col2_width = len(col2_header)

        # Row separator
        sep = "="*col1_width + " " + "="*col2_width + "\n"

        # Row headers
        file.write(sep)

        file.write(col1_header.center(col1_width))
        file.write(" ")
        file.write("%s\n" % col2_header.center(col2_width))

        file.write(sep)

        # Rows
        for label, attr in components:
            col2_value = str(getattr(report, attr))
            file.write("%s %s\n" %
                (label.ljust(col1_width), col2_value.rjust(col2_width)))
        else:
            file.write(sep)
            file.write("\n")

        del report
1274220
def _generate_splice(self, slice_ind):
        '''
        
        '''
        step_size = slice_ind.step if slice_ind.step else 1
        # Check for each of the four possible scenarios
        if slice_ind.start != None:
            if slice_ind.stop != None:
                newListLen = ((get_non_negative_index(slice_ind.stop, self._length) -
                               get_non_negative_index(slice_ind.start, self._length)) // step_size)
            else:
                newListLen = ((self._length -
                               get_non_negative_index(slice_ind.start, self._length)) // step_size)
        else:
            if slice_ind.stop != None:
                newListLen = ((get_non_negative_index(slice_ind.stop, self._length)) // step_size)
            else:
                newListLen = (self._length // step_size)

        return ZeroList(newListLen)
648249
def build_dataset(filename, max_lines=-1):
    """"""
    encodings = dict(list(map(reversed, enumerate(string.printable))))
    digitize = lambda char: encodings[char] if char in encodings else len(encodings)
    encode_line = lambda line: np.array(list(map(digitize, line)))
    nonblank_line = lambda line: len(line) > 2

    with open(filename) as f:
        lines = f.readlines()

    encoded_lines = list(map(encode_line, list(filter(nonblank_line, lines))[:max_lines]))
    num_outputs = len(encodings) + 1

    return encoded_lines, num_outputs
430934
def slices_to_global_coords(self, slices):
    
    bbox = self.bbox_to_mip(slices, self.mip, 0)
    return bbox.to_slices()
368556
def _fill_slices(self, sls):
        """"""
        # Fill '...' entries with ':'
        nfill = self.ndim - len(sls)
        fsls = list()
        for sl in sls:
            if sl is Ellipsis:
                while nfill:
                    fsls.append(slice(None, None))
                    nfill -= 1
                fsls.append(slice(None, None))
            else:
                fsls.append(sl)

        # Append ':' to the end
        for _ in range(self.ndim - len(fsls)):
            fsls.append(slice(None, None))

        return fsls
125245
def filesampler(files, testsetsize = 0.1, devsetsize = 0, trainsetsize = 0, outputdir = '', encoding='utf-8'):
        """"""

        if not isinstance(files, list):
            files = list(files)

        total = 0
        for filename in files:
            f = io.open(filename,'r', encoding=encoding)
            count = 0
            for line in f:
                count += 1
            f.close()
            if total == 0:
                total = count
            elif total != count:
                raise Exception("Size mismatch, when multiple files are specified they must contain the exact same amount of lines! (" +str(count)  + " vs " + str(total) +")")

        #support for relative values:
        if testsetsize < 1:
            testsetsize = int(total * testsetsize)
        if devsetsize < 1 and devsetsize > 0:
            devsetsize = int(total * devsetsize)


        if testsetsize >= total or devsetsize >= total or testsetsize + devsetsize >= total:
            raise Exception("Test set and/or development set too large! No samples left for training set!")


        trainset = {}
        testset = {}
        devset = {}
        for i in range(1,total+1):
            trainset[i] = True
        for i in random.sample(trainset.keys(), int(testsetsize)):
            testset[i] = True
            del trainset[i]

        if devsetsize > 0:
            for i in random.sample(trainset.keys(), int(devsetsize)):
                devset[i] = True
                del trainset[i]

        if trainsetsize > 0:
            newtrainset = {}
            for i in random.sample(trainset.keys(), int(trainsetsize)):
                newtrainset[i] = True
            trainset = newtrainset

        for filename in files:
            if not outputdir:
                ftrain = io.open(filename + '.train','w',encoding=encoding)
            else:
                ftrain = io.open(outputdir + '/' +  os.path.basename(filename) + '.train','w',encoding=encoding)
            if not outputdir:
                ftest = io.open(filename + '.test','w',encoding=encoding)
            else:
                ftest = io.open(outputdir + '/' + os.path.basename(filename) + '.test','w',encoding=encoding)
            if devsetsize > 0:
                if not outputdir:
                    fdev = io.open(filename + '.dev','w',encoding=encoding)
                else:
                    fdev = io.open(outputdir + '/' +  os.path.basename(filename) + '.dev','w',encoding=encoding)

            f = io.open(filename,'r',encoding=encoding)
            for linenum, line in enumerate(f):
                if linenum+1 in trainset:
                    ftrain.write(line)
                elif linenum+1 in testset:
                    ftest.write(line)
                elif devsetsize > 0 and linenum+1 in devset:
                    fdev.write(line)
            f.close()

            ftrain.close()
            ftest.close()
            if devsetsize > 0: fdev.close()
1158588
def _batch_gvcfs(data, region, vrn_files, ref_file, out_file=None):
    
    if out_file is None:
        out_file = vrn_files[0]
    # group to get below the maximum batch size, using 200 as the baseline
    max_batch = int(dd.get_joint_group_size(data))
    if len(vrn_files) > max_batch:
        out = []
        num_batches = int(math.ceil(float(len(vrn_files)) / max_batch))
        for i, batch_vrn_files in enumerate(tz.partition_all(num_batches, vrn_files)):
            base, ext = utils.splitext_plus(out_file)
            batch_out_file = "%s-b%s%s" % (base, i, ext)
            out.append(run_combine_gvcfs(batch_vrn_files, region, ref_file, batch_out_file, data))
        return _batch_gvcfs(data, region, out, ref_file)
    else:
        return vrn_files
485728
def parse_featurecounts_report (self, f):
        

        file_names = list()
        parsed_data = dict()
        for l in f['f'].splitlines():
            thisrow = list()
            s = l.split("\t")
            if len(s) < 2:
                continue
            if s[0] == 'Status':
                for f_name in s[1:]:
                    file_names.append(f_name)
            else:
                k = s[0]
                if k not in self.featurecounts_keys:
                    self.featurecounts_keys.append(k)
                for val in s[1:]:
                    try:
                        thisrow.append(int(val))
                    except ValueError:
                        pass
            if len(thisrow) > 0:
                parsed_data[k] = thisrow
        # Check that this actually is a featureCounts file, as format and parsing is quite general
        if 'Assigned' not in parsed_data.keys():
            return None
        for idx, f_name in enumerate(file_names):

            # Clean up sample name
            s_name = self.clean_s_name(f_name, f['root'])

            # Reorganised parsed data for this sample
            # Collect total count number
            data = dict()
            data['Total'] = 0
            for k in parsed_data:
                data[k] = parsed_data[k][idx]
                data['Total'] += parsed_data[k][idx]

            # Calculate the percent aligned if we can
            try:
                data['percent_assigned'] = (float(data['Assigned'])/float(data['Total'])) * 100.0
            except (KeyError, ZeroDivisionError):
                pass

            # Add to the main dictionary
            if len(data) > 1:
                if s_name in self.featurecounts_data:
                    log.debug("Duplicate sample name found! Overwriting: {}".format(s_name))
                self.add_data_source(f, s_name)
                self.featurecounts_data[s_name] = data
371089
def timeslice_generator(self):
        """"""
        slice_id = 0
        while slice_id < self.n_timeslices:
            blob = self.get_blob(slice_id)
            yield blob
            slice_id += 1