Create 1d-array containing all disctinct models from ``tasklist``.
495259
def _flatten_models(tasklist):
        "  "
        ans = gvar.BufferDict()
        for task, mlist in tasklist:
            if task != 'fit':
                continue
            for m in mlist:
                id_m = id(m)
                if id_m not in ans:
                    ans[id_m] = m
        return ans.buf.tolist()
734172
def to_twodim_list(self):
        
        if self._timestampFormat is None:
            return self._timeseriesData

        datalist = []
        append   = datalist.append
        convert  = TimeSeries.convert_epoch_to_timestamp
        for entry in self._timeseriesData:
            append([convert(entry[0], self._timestampFormat), entry[1:]])

        return datalist
36248
def get_models(model_dat_path):
    

    # Check first if we already have a model data file in the data directory


    with open(model_dat_path) as f:

        # model.dat is a text file, no size issues here (will fit in memory)

        model_dat = f.read()

    # Replace \r or \r\n with \n (the former two might be in the file if
    # it originates from Windows)
    if "\n" in model_dat:

        model_dat = model_dat.replace("\r", "")

    else:

        model_dat = model_dat.replace("\r", "\n")

    # Loop through the file and build the model definition
    # dictionary

    lines = model_dat.split("\n")

    model_definitions = collections.OrderedDict()

    for line in lines:

        match = re.match('''(.+(add|mul|con|acn).+)''', line)

        if match is not None:

            # This is a model definition

            tokens = line.split()

            if len(tokens) == 7:

                (model_name, n_parameters,
                 min_energy, max_energy,
                 library_function,
                 model_type,
                 flag) = line.split()

            else:

                (model_name, n_parameters,
                 min_energy, max_energy,
                 library_function,
                 model_type,
                 flag,
                 flag_2) = line.split()

            this_model = collections.OrderedDict()

            this_model['description'] = 'The %s model from XSpec (https://heasarc.gsfc.nasa.gov/xanadu/' \
                                                   'xspec/manual/XspecModels.html)' % model_name

            this_model['parameters'] = collections.OrderedDict()

            model_definitions[(model_name, library_function, model_type)] = this_model

        else:

            # This is a parameter definition

            if len(line.split()) == 0:
                # Empty line
                continue

            # Parameters are free by default, unless they have delta < 0
            # they are switch parameters or scale parameters

            free = True

            if line[0] == '$':

                # Probably a switch parameter

                free = False

                tokens = line.split()

                if len(tokens) == 2:

                    par_name = tokens[0][1:]

                    default_value = tokens[1]

                    par_unit = ""

                    hard_minimum, soft_minimum, soft_maximum, hard_maximum = (0, 0, 1e9, 1e9)

                elif len(tokens) == 3:

                    par_name = tokens[0][1:]

                    default_value = tokens[2]

                    par_unit = ""

                    hard_minimum, soft_minimum, soft_maximum, hard_maximum = (0, 0, 1e9, 1e9)

                else:

                    match = re.match('(\S+)\s+(\".+\"|[a-zA-Z]+)?(.+)*', line[1:])

                    par_name, par_unit, par_spec = match.groups()

                    tokens = par_spec.split()

                    if len(tokens) == 1:

                        default_value = tokens[0]
                        par_unit = ""
                        hard_minimum, soft_minimum, soft_maximum, hard_maximum = (0, 0, 1e9, 1e9)

                    else:

                        par_unit = ""

                        (default_value,
                         hard_minimum, soft_minimum,
                         soft_maximum, hard_maximum,
                         delta) = par_spec.split()


            else:

                # A normal parameter

                match = re.match('(\S+)\s+(\".+\"|\S+)(.+)', line)

                if match is None:

                    raise RuntimeError("Cannot parse parameter %s" % line)

                par_name, par_unit, par_spec = match.groups()

                if par_name[0] == '*':

                    # Scale parameter (always frozen)

                    par_name = par_name[1:]

                    free = False

                    tokens = par_spec.split()

                    if len(tokens) == 1:

                        default_value = tokens[0]
                        (hard_minimum, soft_minimum,
                         soft_maximum, hard_maximum,
                         delta) = (None, None, None, None, 0.1)

                    else:

                        (default_value,
                         hard_minimum, soft_minimum,
                         soft_maximum, hard_maximum,
                         delta) = par_spec.split()

                        delta = abs(float(delta))

                else:

                    (default_value,
                     hard_minimum, soft_minimum,
                     soft_maximum, hard_maximum,
                     delta) = par_spec.split()

                    delta = float(delta)

                    if delta <= 0:
                        free = False

                        delta = abs(delta)

            # Now fix the parameter name removing illegal characters
            # (for example 'log(A)' is not a legal name)

            par_name = re.sub('[\(,\)]', '_', par_name)
            par_name = re.sub('<', '_less_', par_name)
            par_name = re.sub('>', '_more_', par_name)
            par_name = re.sub('/', '_div_', par_name)
            par_name = re.sub('\-', '_minus_', par_name)
            par_name = re.sub('\+', '_plus_', par_name)
            par_name = re.sub('\.', '_dot_', par_name)
            par_name = re.sub('@', '_at_', par_name)

            # Parameter names must be lower case
            par_name = par_name.lower()

            # Some parameters are enclosed between ", like "z"
            par_name = par_name.replace('"', '')

            # "z" is a protected name in astromodels.
            if par_name == "z":

                par_name = "redshift"

            # Check that the parameter name is not an illegal Python name
            if par_name in illegal_variable_names:

                par_name = "xs_%s" % par_name

            # Sometimes the unit is " " which is not recognized by astropy
            if par_unit:

                par_unit = par_unit.replace("\"",'')

            # There are some errors in model.dat , like KeV instead of keV, and ergcm/s instead of erg cm /s
            # Let's correct them
            if par_unit == "KeV":

                par_unit = "keV"

            elif par_unit == "ergcm/s":

                par_unit = "erg cm / s"

            elif par_unit == "days":

                par_unit = "day"

            elif par_unit == "z" and par_name.lower() == "redshift":

                par_unit = ""

            # There are funny units in model.dat, like "Rs" (which means Schwarzschild radius) or other things
            # so let's try to convert the par_unit into an astropy.Unit instance. If that fails, use a unitless unit
            try:

                _ = u.Unit(par_unit)

            except ValueError:

                # Unit not recognized

                #warnings.warn("Unit %s is not recognized by astropy." % par_unit)

                par_unit = ''

            # Make sure that this is a valid python identifier
            # by matching it with the relative regexp
            if re.match('([a-zA-Z_][a-zA-Z0-9_]*)$', par_name) is None:

                raise ValueError("Illegal identifier name %s" % (par_name))

            this_model['parameters'][par_name] = {'initial value': float(default_value),
                                                  'desc': '(see https://heasarc.gsfc.nasa.gov/xanadu/xspec/manual/'
                                                          'XspecModels.html)',
                                                  'min': hard_minimum,
                                                  'max': hard_maximum,
                                                  'delta': float(delta),
                                                  'unit': par_unit,
                                                  'free': free}

    return model_definitions
478303
def getTaskHelp(_Task):
  
  """
  Ret = []

  for k in ['name', 'desc']:
    v = _Task.Config.get(k)

    if v is not None:
      Ret.append('%s: %s' % (k, v))

  Args = _Task.Args

  if Args:
    Ret.append('\nArgs:')

    for argName, Arg in Args.items():
      Ret.append('  %s: %s' % (argName, Arg.get('desc', Arg['type_str'])))

    Ret.append('')

  return '\n'.join(Ret).rstrip()
963571
def mappable_to_model rc_file
      rc_file.map! { |l| line_to_tuple(l) }.compact!
      taskrc = Hash[rc_file]
      hash_to_model(taskrc)
    end
796001
def toy_model(features, mesh):
  """"""
  batch_dim = mtf.Dimension('batch', FLAGS.batch_size)
  io_dim = mtf.Dimension('io', FLAGS.io_size)

  master_dtype = tf.as_dtype(FLAGS.master_dtype)
  slice_dtype = tf.as_dtype(FLAGS.slice_dtype)
  activation_dtype = tf.as_dtype(FLAGS.activation_dtype)

  x = mtf.import_tf_tensor(mesh, features, mtf.Shape([batch_dim, io_dim]))
  x = mtf.cast(x, activation_dtype)
  h = x
  for lnum in xrange(1, FLAGS.num_hidden_layers + 2):
    if lnum + 1 == FLAGS.num_hidden_layers + 2:
      # output layer
      dim = io_dim
    elif lnum % 2 == 0:
      dim = mtf.Dimension('hidden_even', FLAGS.hidden_size)
    else:
      dim = mtf.Dimension('hidden_odd', FLAGS.hidden_size)
    h = mtf.layers.dense(
        h, dim,
        use_bias=False,
        master_dtype=master_dtype,
        slice_dtype=slice_dtype,
        name='layer_%d' % lnum)
  y = h
  g = tf.train.get_global_step()
  if FLAGS.step_with_nan >= 0:
    # Trigger NaN in the forward pass, this is used for testing whether
    # MeshTensorFlow can handle occasional NaN value.
    y += mtf.import_tf_tensor(
        mesh,
        tf.divide(
            0.0,
            tf.cond(tf.equal(g, FLAGS.step_with_nan), lambda: 0., lambda: 1.)),
        mtf.Shape([]))

  loss = mtf.reduce_mean(mtf.square(y - x))
  return y, loss
143438
def _build_model(self, x):
    """"""
    with tf.variable_scope('init'):
      x = self._conv('init_conv', x, 3, x.shape[3], 16,
                     self._stride_arr(1))

    strides = [1, 2, 2]
    activate_before_residual = [True, False, False]
    if self.hps.use_bottleneck:
      res_func = self._bottleneck_residual
      filters = [16, 64, 128, 256]
    else:
      res_func = self._residual
      filters = [16, 16, 32, 64]
      # Uncomment the following codes to use w28-10 wide residual
      # network.
      # It is more memory efficient than very deep residual network and
      # has
      # comparably good performance.
      # https://arxiv.org/pdf/1605.07146v1.pdf
      # filters = [16, 160, 320, 640]
      # Update hps.num_residual_units to 4

    with tf.variable_scope('unit_1_0'):
      x = res_func(x, filters[0], filters[1],
                   self._stride_arr(strides[0]),
                   activate_before_residual[0])
    for i in six.moves.range(1, self.hps.num_residual_units):
      with tf.variable_scope('unit_1_%d' % i):
        x = res_func(x, filters[1], filters[1],
                     self._stride_arr(1), False)

    with tf.variable_scope('unit_2_0'):
      x = res_func(x, filters[1], filters[2],
                   self._stride_arr(strides[1]),
                   activate_before_residual[1])
    for i in six.moves.range(1, self.hps.num_residual_units):
      with tf.variable_scope('unit_2_%d' % i):
        x = res_func(x, filters[2], filters[2],
                     self._stride_arr(1), False)

    with tf.variable_scope('unit_3_0'):
      x = res_func(x, filters[2], filters[3],
                   self._stride_arr(strides[2]),
                   activate_before_residual[2])
    for i in six.moves.range(1, self.hps.num_residual_units):
      with tf.variable_scope('unit_3_%d' % i):
        x = res_func(x, filters[3], filters[3],
                     self._stride_arr(1), False)

    with tf.variable_scope('unit_last'):
      x = self._layer_norm('final_bn', x)
      x = self._relu(x, self.hps.relu_leakiness)
      x = self._global_avg_pool(x)

    with tf.variable_scope('logit'):
      logits = self._fully_connected(x, self.hps.nb_classes)
      predictions = tf.nn.softmax(logits)

    return logits, predictions
413213
def batch(byte_array, funcs):
    
    result = []
    length = bytes_to_int(byte_array[0:4])
    item_size = bytes_to_int(byte_array[4:8])
    for i in range(0, length):
        chunk = byte_array[8+i*item_size:8+(i+1)*item_size]
        for f in funcs:
            f(chunk)
    return result
161281
def foreach_model(self, fn):
        

        results = ray.get([w.foreach_model.remote(fn) for w in self.workers])
        out = []
        for r in results:
            out.extend(r)
        return out
266350
def init_all_objects(self, data, target=None, single_result=True):
        
        if single_result:
            return self.init_target_object(target, data)
        return list(self.expand_models(target, data))
753524
def to_image_list(tensors, size_divisible=0):
    
    if isinstance(tensors, torch.Tensor) and size_divisible > 0:
        tensors = [tensors]

    if isinstance(tensors, ImageList):
        return tensors
    elif isinstance(tensors, torch.Tensor):
        # single tensor shape can be inferred
        assert tensors.dim() == 4
        image_sizes = [tensor.shape[-2:] for tensor in tensors]
        return ImageList(tensors, image_sizes)
    elif isinstance(tensors, (tuple, list)):
        max_size = tuple(max(s) for s in zip(*[img.shape for img in tensors]))

        # TODO Ideally, just remove this and let me model handle arbitrary
        # input sizs
        if size_divisible > 0:
            import math

            stride = size_divisible
            max_size = list(max_size)
            max_size[1] = int(math.ceil(max_size[1] / stride) * stride)
            max_size[2] = int(math.ceil(max_size[2] / stride) * stride)
            max_size = tuple(max_size)

        batch_shape = (len(tensors),) + max_size
        batched_imgs = tensors[0].new(*batch_shape).zero_()
        for img, pad_img in zip(tensors, batched_imgs):
            pad_img[: img.shape[0], : img.shape[1], : img.shape[2]].copy_(img)

        image_sizes = [im.shape[-2:] for im in tensors]

        return ImageList(batched_imgs, image_sizes)
    else:
        raise TypeError("Unsupported type for to_image_list: {}".format(type(tensors)))
419517
def collect(self, name, arr):
        """"""
        name = py_str(name)
        if self.include_layer is not None and not self.include_layer(name):
            return
        handle = ctypes.cast(arr, NDArrayHandle)
        arr = NDArray(handle, writable=False).copyto(cpu())
        if self.logger is not None:
            self.logger.info("Collecting layer %s output of shape %s" % (name, arr.shape))
        if name in self.nd_dict:
            self.nd_dict[name].append(arr)
        else:
            self.nd_dict[name] = [arr]
874865
def build_plane_arrays(x, y, qlist):
    
    if type(qlist) is not list:
        return_list = False
        qlist = [qlist]
    else:
        return_list = True
    xv = x[np.where(y==y[0])[0]]
    yv = y[np.where(x==x[0])[0]]
    qlistp = []
    for n in range(len(qlist)):
        qlistp.append(np.zeros((len(yv), len(xv))))
    for j in range(len(qlist)):
        for n in range(len(yv)):
            i = np.where(y==yv[n])[0]
            qlistp[j][n,:] = qlist[j][i]
    if not return_list:
        qlistp = qlistp[0]
    return xv, yv, qlistp
1028450
def flat_list(input_list):
    

    """
    x = input_list
    if isinstance(x, list):
        return [a for i in x for a in flat_list(i)]
    else:
        return [x]
83591
def gradient_list(start, end, steps):
    """"""
    delta = [end[i] - start[i] for i in range(4)]
    return [bytearray(start[j] + (delta[j] * i) // steps for j in range(4))
            for i in range(steps + 1)]
710600
def all_dims(self):
        """"""
        return [
            _get_dims(arr) if not isinstance(arr, ArrayList) else
            arr.all_dims
            for arr in self]
218028
def as_struct_array(*columns):
    
    columns = [np.asarray(c) for c in columns]
    rows = len(columns[0])

    names = ['f'+str(i) for i in range(len(columns))]
    dtype = [(names[i], c.dtype, c.shape[1:]) for i, c in enumerate(columns)]
    data = np.empty(rows, dtype)
    for i, c in enumerate(columns):
        data[names[i]] = c
    return data