Takes an optimizer that operates on flat 1D numpy arrays and returns a
    wrapped version that handles trees of nested containers (lists/tuples/dicts)
    with arrays/scalars at the leaves.
328182
def unflatten_optimizer(optimize):
    """"""
    @wraps(optimize)
    def _optimize(grad, x0, callback=None, *args, **kwargs):
        _x0, unflatten = flatten(x0)
        _grad = lambda x, i: flatten(grad(unflatten(x), i))[0]
        if callback:
            _callback = lambda x, i, g: callback(unflatten(x), i, unflatten(g))
        else:
            _callback = None
        return unflatten(optimize(_grad, _x0, _callback, *args, **kwargs))

    return _optimize
523397
def some(arr):
    """"""
    return functools.reduce(lambda x, y: x or (y is not None), arr, False)
837442
def asarray(x, dtype=None):
    ''''''
    iterable = scalarasiter(x)
    if isinstance(iterable, ndarray):
        return iterable
    else:
        if not hasattr(iterable, '__len__'):
            iterable = list(iterable)
        if dtype == object_type:
            a = ndarray((len(iterable),), dtype=dtype)
            for i,v in enumerate(iterable):
                a[i] = v
            return a
        else:
            return array(iterable, dtype=dtype)
46764
def _reshape(self, array, dry=False):
        
        import functools, numpy as np
        if array.ndim == 1:
            shape = (array.shape[0], 1)
        else:
            # hold first dimension, multiply the rest
            shape = (array.shape[0], functools.reduce(lambda x, y: x * y, array.shape[1:]))
        if not dry:
            array = np.reshape(array, shape)
        return array, shape
417545
def flatten(l, types=(list, float)):
    
    l = [item if isinstance(item, types) else [item] for item in l]
    return [item for sublist in l for item in sublist]
219316
def iterator_from_slice(s):
    
    import numpy as np
    start = s.start if s.start is not None else 0
    step = s.step if s.step is not None else 1

    if s.stop is None:
        # Infinite iterator.
        return itertools.count(start=start, step=step)
    else:
        # xrange-like iterator that supports float.
        return iter(np.arange(start, s.stop, step))
560578
def nested_shape(array_or_tuple):
    
    if hasattr(array_or_tuple, 'size'):
        # pytorch tensors use V.size() to get size of tensor
        return list(array_or_tuple.size())
    elif hasattr(array_or_tuple, 'get_shape'):
        # tensorflow uses V.get_shape() to get size of tensor
        return array_or_tuple.get_shape().as_list()
    elif hasattr(array_or_tuple, 'shape'):
        return array_or_tuple.shape

    try:
        # treat object as iterable
        return [nested_shape(item) for item in list(array_or_tuple)]
    except TypeError:
        # object is not actually iterable
        # LB: Maybe we should throw an error?
        return []
673264
def _treeify(values):
        """"""
        if len(values) == 1:  # this case causes problems later
            return values
        tree = np.empty_like(values)
        # Tree indices work as follows:
        # 0 is the root
        # 2n+1 is the left child of n
        # 2n+2 is the right child of n
        # So we now rearrange `values` into that format...

        # The first step is to remove the bottom row of leaves, which might not be exactly full
        last_full_row = int(np.log2(len(values) + 1) - 1)
        len_ragged_row = len(values) - (2 ** (last_full_row + 1) - 1)
        if len_ragged_row > 0:
            bottom_row_ix = np.s_[: 2 * len_ragged_row : 2]
            tree[-len_ragged_row:] = values[bottom_row_ix]
            values = np.delete(values, bottom_row_ix)

        # Now `values` is length 2**n - 1, so can be packed efficiently into a tree
        # Last row of nodes is indices 0, 2, ..., 2**n - 2
        # Second-last row is indices 1, 5, ..., 2**n - 3
        # nth-last row is indices (2**n - 1)::(2**(n+1))
        values_start = 0
        values_space = 2
        values_len = 2 ** last_full_row
        while values_start < len(values):
            tree[values_len - 1 : 2 * values_len - 1] = values[values_start::values_space]
            values_start += int(values_space / 2)
            values_space *= 2
            values_len = int(values_len / 2)
        return tree
681440
def precompute_optimzation_Y(laplacian_matrix, n_samples, relaxation_kwds):
    """"""
    relaxation_kwds.setdefault('presave',False)
    relaxation_kwds.setdefault('presave_name','pre_comp_current.npy')
    relaxation_kwds.setdefault('verbose',False)
    if relaxation_kwds['verbose']:
        print ('Making Lk and nbhds')
    Lk_tensor, nbk, si_map = \
        compute_Lk(laplacian_matrix, n_samples, relaxation_kwds['subset'])
    if relaxation_kwds['presave']:
        raise NotImplementedError('Not yet implemented presave')
    return { 'Lk': Lk_tensor, 'nbk': nbk, 'si_map': si_map }
90036
def optimize_onnx_model(origin_model, nchw_inputs=None):
    
    graph = origin_model.graph
    nodelist = list(graph.node)
    del graph.node[:]

    all_nodes = optimize_onnx(nodelist,
                              inputs=graph.input,
                              outputs=graph.output)
    nodes = [n_ for n_ in all_nodes if not isinstance(n_, tuple)]
    graph.node.extend(nodes)

    alter_tensors = {n_[1]: n_[0] for n_ in all_nodes if isinstance(n_, tuple)}
    update_tensor = lambda x: \
        helper.make_tensor(x.name, x.data_type, (x.dims[0], 1, 1),
                           onnx.numpy_helper.to_array(x).flatten())
    new_initializer = [init_ if init_.name not in alter_tensors else update_tensor(init_)
                       for init_ in graph.initializer]
    del graph.initializer[:]
    graph.initializer.extend(new_initializer)

    update_value_info = lambda x: \
        helper.make_tensor_value_info(x.name, x.type.tensor_type.elem_type,
                                      (x.type.tensor_type.shape.dim[0].dim_value, 1, 1))
    new_input = [in_ if in_.name not in alter_tensors else update_value_info(in_)
                 for in_ in graph.input]
    del graph.input[:]
    graph.input.extend(new_input)
    return origin_model
647825
def optimize_NxN(x, y, fit_offset=False, perc=None):
    
    if perc is not None:
        if not fit_offset and isinstance(perc, (list, tuple)): perc = perc[1]
        weights = get_weight(x, y, perc).astype(bool)
        if issparse(weights): weights = weights.A
    else:
        weights = None

    x, y = x.astype(np.float64), y.astype(np.float64)

    n_vars = x.shape[1]
    offset, gamma = np.zeros(n_vars), np.zeros(n_vars)

    for i in range(n_vars):
        xi = x[:, i] if weights is None else x[:, i][weights[:, i]]
        yi = y[:, i] if weights is None else y[:, i][weights[:, i]]

        if fit_offset:
            offset[i], gamma[i] = minimize(lambda m: np.sum((-yi + xi * m[1] + m[0])**2), method="L-BFGS-B",
                                           x0=(0, 0.1), bounds=[(0, None), (None, None)]).x
        else:
            gamma[i] = minimize(lambda m: np.sum((-yi + xi * m) ** 2), x0=0.1, method="L-BFGS-B").x
    offset[np.isnan(offset)], gamma[np.isnan(gamma)] = 0, 0
    return offset, gamma
1291793
def select_from_array(cls, array, identifier):
        

        base_array = np.zeros(array.shape)
        array_coords = np.where(array == identifier)
        base_array[array_coords] = 1

        return cls(base_array)
612487
def brent_optimise(node1, node2, min_brlen=0.001, max_brlen=10, verbose=False):
    
    from scipy.optimize import minimize_scalar
    wrapper = BranchLengthOptimiser(node1, node2, (min_brlen + max_brlen) / 2.)
    n = minimize_scalar(lambda x: -wrapper(x)[0], method='brent', bracket=(min_brlen, max_brlen))['x']
    if verbose:
        logger.info(wrapper)
    if n < min_brlen:
        n = min_brlen
        wrapper(n)
    return n, -1 / wrapper.get_d2lnl(n)
631396
def create(cls, opt_func:Union[type,Callable], lr:Union[float,Tuple,List], layer_groups:ModuleList, wd:Floats=0., 
               true_wd:bool=False, bn_wd:bool=True)->optim.Optimizer:
        ""
        split_params = split_no_wd_params(layer_groups)
        opt = opt_func([{'params': p, 'lr':0} for p in split_params])
        opt = cls(opt, wd=wd, true_wd=true_wd, bn_wd=bn_wd)
        opt.lr,opt.opt_func = listify(lr, layer_groups),opt_func
        return opt
1071
def nested_get(dictionary, keys):
    
    return reduce(lambda d, k: d[k], keys, dictionary)
86235
def minimise(objective_function, p0, **kwargs):
    

    p0 = np.array(p0)

    op_kwargs = kwargs.copy()

    # Which optimisation algorithm?
    available_methods = ("BFGS", "Nelder-Mead", "Powell", "CG", "TNC")
    method = op_kwargs.pop("method", available_methods[0])
    logger.debug("Optimising with {0} algorithm".format(method))

    op_kwargs.update({ "callback": None, "full_output": True, "retall": False })

    def _drop_forbidden_keywords(op_kwds, allowed):
        for keyword in set(op_kwds).difference(allowed):
            logger.debug("Dropping optimisation keyword {0}: {1}".format(
                keyword, op_kwds.pop(keyword)))

    t_init = time()
    if method == "Nelder-Mead":
        # Need to force some arguments.
        # Optional arguments:
        # xtol, ftol, maxiter, maxfun
        _drop_forbidden_keywords(op_kwargs, ("xtol", "ftol", "maxiter", 
            "maxfun", "full_output", "retall", "callback", "disp"))

        # Set defaults.
        op_kwargs.setdefault("disp", False)

        x_opt, f_opt, num_iter, num_funcalls, warnflag \
            = op.fmin(objective_function, p0, **op_kwargs)

        logger.debug("Number of iterations: {0}, number of function calls: {1}"\
            .format(num_iter, num_funcalls))

        if warnflag == 0:
            logger.debug("Optimisation converged after {0:.1f} seconds.".format(
                time() - t_init))
        else:
            logger.warn("Optimisation failed: {0}".format([
                "Maximum number of function evaluations.",
                "Maximum number of iterations."][warnflag - 1]))

    elif method == "Powell":
        # Optional arguments: xtol, ftol, maxiter, maxfun
        _drop_forbidden_keywords(op_kwargs, ("xtol", "ftol", "maxiter", 
            "maxfun", "retall", "callback", "full_output", "disp"))
        
        # Set defaults.
        op_kwargs.setdefault("disp", False)

        x_opt, f_opt, direc, num_iter, num_funcalls, warnflag \
            = op.fmin_powell(objective_function, p0, **op_kwargs)

        logger.debug("Number of iterations: {0}, number of function calls: {1}"\
            .format(num_iter, num_funcalls))

        if warnflag == 0:
            logger.debug("Optimisation converged after {0:.1f} seconds.".format(
                time() - t_init))
        else:
            logger.warn("Optimisation failed: {0}".format([
                "Maximum number of function evaluations.",
                "Maximum number of iterations."][warnflag - 1]))

    elif method == "CG":
        # Optional arguments: gtol, norm, epsilon, maxiter.
        _drop_forbidden_keywords(op_kwargs, ("gtol", "norm", "epsilon", 
            "maxiter", "retall", "callback", "full_output"))

        x_opt, f_opt, num_funcalls, num_gradcalls, warnflag \
            = op.fmin_cg(objective_function, p0, **op_kwargs)

        logger.debug("Number of function calls: {0}, gradient calls: {1}"\
            .format(num_funcalls, num_gradcalls))

        if warnflag == 0:
            logger.debug("Optimisation converged after {0:.1f} seconds.".format(
                time() - t_init))
        else:
            logger.debug("Optimisation failed: {0}".format([
                "Maximum number of iterations exceeded",
                "Gradient and/or function calls were not changing."
                ][warnflag - 1]))

    elif method == "BFGS":
        # Since we have at least some boundaries, this will actually call
        # the L-BFGS-B algorithm.
        # Optional arguments: m, factr, pgtol, epsilon, maxfun, maxiter

        _drop_forbidden_keywords(op_kwargs, ("m", "factr", "pgtol", 
            "approx_grad", "epsilon", "disp", "iprint", "maxfun", "maxiter"))

        # Default/required:
        op_kwargs.setdefault("factr", 10.0)
        op_kwargs["approx_grad"] = True
        
        # Because the parameters will vary by orders of magnitude, here we
        # scale everything to the initial value so that the epsilon keyword
        # makes some sense.

        scale = p0.copy()
        
        def scaled_objective_function(theta):
            return objective_function(theta.copy() * scale)

        x_opt, f_opt, info_dict = op.fmin_l_bfgs_b(scaled_objective_function,
            np.ones(p0.size, dtype=float), **op_kwargs)

        # Rescale.
        x_opt *= scale

        g_opt = info_dict["grad"]
        num_iter = info_dict["nit"]
        num_funcalls = info_dict["funcalls"]
        warnflag = info_dict["warnflag"]

        logger.debug("Number of iterations: {0}, function calls: {1}".format(
            num_iter, num_funcalls))
        if warnflag == 0:
            logger.debug("Optimisation converged after {0:.1f} seconds.".format(
                time() - t_init))
        else:
            logger.warn("Optimisation failed: {0}".format([
                "Too many function evaluations or iterations!",
                "{0}".format(info_dict.get("task", None))][warnflag - 1]))
    
    elif method == "TNC":
        # Optional arguments:   bounds, epsilon, scale, offset, messages, 
        #                       maxCGit, maxfun, eta, stepmx, accuracy, 
        #                       fmin, ftol, xtol, pgtol, rescale

        _drop_forbidden_keywords(op_kwargs, ("approx_grad", "bounds", "epsilon",
            "scale", "offset", "messages", "maxCGit", "maxfun", "eta", "stepmx",
            "accuracy", "fmin", "ftol", "xtol", "pgtol", "rescale", "disp"))

        # Required:
        op_kwargs["approx_grad"] = True

        x_opt, num_funcalls, rc = op.fmin_tnc(objective_function, p0, 
            **op_kwargs)
        
        rcstring = {
            -1: "Infeasible (lower bound > upper bound)",
            0: "Local minimum reached (|pg| ~= 0)",
            1: "Converged (|f_n-f_(n-1)| ~= 0)",
            2: "Converged (|x_n-x_(n-1)| ~= 0)",
            3: "Max. number of function evaluations reached",
            4: "Linear search failed",
            5: "All lower bounds are equal to the upper bounds",
            6: "Unable to progress",
            7: "User requested end of minimization"
        }[rc]

        logger.debug("Number of function calls: {0}, result: {1}".format(
            num_funcalls, rcstring))
        if rc in (1, 2):
            logger.debug("Optimisation converged after {0:.1f} seconds: {1}"\
                .format(time() - t_init, rcstring))
        else:
            logger.warn("Optimisation failed: {0}".format(rcstring))

    else:
        raise ValueError("optimisation algorithm {0} is not available "\
            "(available methods are {1})".format(
                method, ", ".join(available_methods)))


    return x_opt
1056542
def get_tree(cls, parent=None):
        
        if parent:
            depth = parent.get_depth() + 1
            results = [parent]
        else:
            depth = 1
            results = []
        cls._get_tree_recursively(results, parent, depth)
        return results