Unpack a flattened Dict or Tuple observation array/tensor.

    Arguments:
        obs: The flattened observation tensor
        space: The original space prior to flattening
        tensorlib: The library used to unflatten (reshape) the array/tensor
961433
def _unpack_obs(obs, space, tensorlib=tf):
    

    if (isinstance(space, gym.spaces.Dict)
            or isinstance(space, gym.spaces.Tuple)):
        prep = get_preprocessor(space)(space)
        if len(obs.shape) != 2 or obs.shape[1] != prep.shape[0]:
            raise ValueError(
                "Expected flattened obs shape of [None, {}], got {}".format(
                    prep.shape[0], obs.shape))
        assert len(prep.preprocessors) == len(space.spaces), \
            (len(prep.preprocessors) == len(space.spaces))
        offset = 0
        if isinstance(space, gym.spaces.Tuple):
            u = []
            for p, v in zip(prep.preprocessors, space.spaces):
                obs_slice = obs[:, offset:offset + p.size]
                offset += p.size
                u.append(
                    _unpack_obs(
                        tensorlib.reshape(obs_slice, [-1] + list(p.shape)),
                        v,
                        tensorlib=tensorlib))
        else:
            u = OrderedDict()
            for p, (k, v) in zip(prep.preprocessors, space.spaces.items()):
                obs_slice = obs[:, offset:offset + p.size]
                offset += p.size
                u[k] = _unpack_obs(
                    tensorlib.reshape(obs_slice, [-1] + list(p.shape)),
                    v,
                    tensorlib=tensorlib)
        return u
    else:
        return obs
63050
def unbound_dimensions(streams, kdims, no_duplicates=True):
    
    params = stream_parameters(streams, no_duplicates)
    return [d for d in kdims if d not in params]
616054
def json_numpy_obj_hook(dct):
    
    if isinstance(dct, dict) and '__ndarray__' in dct:
        data = np.asarray(dct['__ndarray__'], dtype=dct['dtype'])
        return data.reshape(dct['shape'])
    return dct
49143
def _tf_flatten_batch_dims(x, num_nonbatch_dims):
  
  shape = x.shape.as_list()
  assert None not in shape
  new_shape = ([list_product(shape[:-num_nonbatch_dims])]
               + shape[-num_nonbatch_dims:])
  if new_shape != shape:
    x = tf.reshape(x, new_shape)
  return x
1069739
@Override
    Object unpack(Info vi, Buffer opb) {
        InfoMapping0 info = new InfoMapping0();

        if (opb.read(1) != 0) {
            info.submaps = opb.read(4) + 1;
        } else {
            info.submaps = 1;
        }

        if (opb.read(1) != 0) {
            info.coupling_steps = opb.read(8) + 1;

            for (int i = 0; i < info.coupling_steps; i++) {
                int testM = info.coupling_mag[i] = opb.read(Util.ilog2(vi.channels));
                int testA = info.coupling_ang[i] = opb.read(Util.ilog2(vi.channels));

                if (testM < 0 || testA < 0 || testM == testA || testM >= vi.channels
                        || testA >= vi.channels) {
                    //goto err_out;
                    info.free();
                    return (null);
                }
            }
        }

        if (opb.read(2) > 0) { /* 2,3:reserved */

            info.free();
            return (null);
        }

        if (info.submaps > 1) {
            for (int i = 0; i < vi.channels; i++) {
                info.chmuxlist[i] = opb.read(4);
                if (info.chmuxlist[i] >= info.submaps) {
                    info.free();
                    return (null);
                }
            }
        }

        for (int i = 0; i < info.submaps; i++) {
            info.timesubmap[i] = opb.read(8);
            if (info.timesubmap[i] >= vi.times) {
                info.free();
                return (null);
            }
            info.floorsubmap[i] = opb.read(8);
            if (info.floorsubmap[i] >= vi.floors) {
                info.free();
                return (null);
            }
            info.residuesubmap[i] = opb.read(8);
            if (info.residuesubmap[i] >= vi.residues) {
                info.free();
                return (null);
            }
        }
        return info;
    }
1126708
def res_from_obseravtion_data(observation_data):
    
    res_df = observation_data.copy()
    res_df.loc[:, "name"] = res_df.pop("obsnme")
    res_df.loc[:, "measured"] = res_df.pop("obsval")
    res_df.loc[:, "group"] = res_df.pop("obgnme")
    res_df.loc[:, "modelled"] = np.NaN
    res_df.loc[:, "residual"] = np.NaN
    return res_df
576326
def __load_obscov(self):
        
        # if the obscov arg is None, but the pst arg is not None,
        # reset and load from obs weights
        if not self.obscov_arg:
            if self.pst_arg:
                self.obscov_arg = self.pst_arg
            else:
                raise Exception("linear_analysis.__load_obscov(): " +
                                "obscov_arg is None")
        if isinstance(self.obscov_arg, Matrix):
            self.__obscov = self.obscov_arg
            return
        if isinstance(self.obscov_arg,np.ndarray):
            # if the ndarray arg is a vector,
            # assume it is the diagonal of the obscov matrix
            if len(self.obscov_arg.shape) == 1:
                assert self.obscov_arg.shape[0] == self.jco.shape[1]
                isdiagonal = True
            else:
                assert self.obscov_arg.shape[0] == self.jco.shape[0]
                assert self.obscov_arg.shape[1] == self.jco.shape[0]
                isdiagonal = False
            self.logger.warn("linear_analysis.__load_obscov(): " +
                             "instantiating obscov from ndarray,  " +
                             "can't verify observation alignment with jco")
            self.__obscov = Matrix(x=self.obscov_arg,
                                         isdiagonal=isdiagonal,
                                         row_names=self.jco.row_names,
                                         col_names=self.jco.row_names)
        self.log("loading obscov")
        if isinstance(self.obscov_arg, str):
            if self.obscov_arg.lower().endswith(".pst"):
                self.__obscov = Cov.from_obsweights(self.obscov_arg)
            else:
                self.__obscov = self.__fromfile(self.obscov_arg, astype=Cov)
        elif isinstance(self.obscov_arg, Pst):
            self.__obscov = Cov.from_observation_data(self.obscov_arg)
        else:
            raise Exception("linear_analysis.__load_obscov(): " +
                            "obscov_arg must be a " +
                            "matrix object or a file name: " +
                            str(self.obscov_arg))
        self.log("loading obscov")
1056267
def unbroadcast_numpy_to(array, shape):
  
  axis = create_unbroadcast_axis(shape, numpy.shape(array))
  return numpy.reshape(numpy.sum(array, axis=axis), shape)
362441
def uninvert_unique_two_lists(flat_list, reconstruct_tup):
    
    import utool as ut
    (inverse3, cumsum, inverse2, inverse1) = reconstruct_tup
    flat_stacked_ = ut.take(flat_list, inverse3)
    unique_list1_, unique_list2_ = ut.unflatten2(flat_stacked_, cumsum)
    res_list1_ = ut.take(unique_list1_, inverse1)
    res_list2_ = ut.take(unique_list2_, inverse2)
    return res_list1_, res_list2_
738005
def _flatten_ex_dict(self):
        """"""
        odict = {}
        for _, fdict in self._ex_dict.items():
            for (extype, exmsg), value in fdict.items():
                key = value["name"]
                odict[key] = copy.deepcopy(value)
                del odict[key]["name"]
                odict[key]["type"] = extype
                odict[key]["msg"] = exmsg
        return odict
235904
def deconv(attrs, inputs, proto_obj):
    """"""
    new_attrs = translation_utils._fix_attribute_names(attrs, {'kernel_shape' : 'kernel',
                                                               'strides' : 'stride',
                                                               'pads': 'pad',
                                                               'dilations': 'dilate',
                                                               'group': 'num_group'})
    new_attrs = translation_utils._add_extra_attributes(new_attrs, {'num_group' : 1})
    new_attrs = translation_utils._fix_bias('Deconvolution', new_attrs, len(inputs))

    new_attrs = translation_utils._fix_channels('Deconvolution', new_attrs, inputs, proto_obj)
    kernel = new_attrs['kernel']
    stride = new_attrs['stride'] if 'stride' in new_attrs else []
    padding = new_attrs['pad'] if 'pad' in new_attrs else []
    dilations = new_attrs['dilate'] if 'dilate' in new_attrs else []
    num_filter = new_attrs['num_filter']
    num_group = new_attrs['num_group']
    no_bias = new_attrs['no_bias'] if 'no_bias' in new_attrs else False
    bias = None if no_bias is True else inputs[2]

    # Unlike ONNX, MXNet's deconvolution operator does not support asymmetric padding, so we first
    # use 'Pad' operator, which supports asymmetric padding. Then use the deconvolution operator.
    pad_width = (0, 0, 0, 0) + translation_utils._pad_sequence_fix(padding, kernel_dim=len(kernel))
    pad_op = symbol.pad(inputs[0], mode='constant', pad_width=pad_width)

    deconv_op = symbol.Deconvolution(pad_op, inputs[1], bias,
                                     kernel=kernel, stride=stride, dilate=dilations,
                                     num_filter=num_filter, num_group=num_group, no_bias=no_bias)

    return deconv_op, new_attrs, inputs
908939
def merge_obs(self):
        
        for model_type in self.model_types:
            self.matched_forecasts[model_type] = {}
            for model_name in self.model_names[model_type]:
                self.matched_forecasts[model_type][model_name] = pd.merge(self.forecasts[model_type][model_name],
                                                                          self.obs, right_on="Step_ID", how="left",
                                                                          left_index=True)
1174928
def transform_obs(self, obs):
    """"""
    empty = np.array([], dtype=np.int32).reshape((0, 7))
    out = named_array.NamedDict({  # Fill out some that are sometimes empty.
        "single_select": empty,
        "multi_select": empty,
        "build_queue": empty,
        "cargo": empty,
        "cargo_slots_available": np.array([0], dtype=np.int32),
    })

    def or_zeros(layer, size):
      if layer is not None:
        return layer.astype(np.int32, copy=False)
      else:
        return np.zeros((size.y, size.x), dtype=np.int32)

    aif = self._agent_interface_format

    if aif.feature_dimensions:
      out["feature_screen"] = named_array.NamedNumpyArray(
          np.stack(or_zeros(f.unpack(obs.observation),
                            aif.feature_dimensions.screen)
                   for f in SCREEN_FEATURES),
          names=[ScreenFeatures, None, None])
      out["feature_minimap"] = named_array.NamedNumpyArray(
          np.stack(or_zeros(f.unpack(obs.observation),
                            aif.feature_dimensions.minimap)
                   for f in MINIMAP_FEATURES),
          names=[MinimapFeatures, None, None])

    if aif.rgb_dimensions:
      out["rgb_screen"] = Feature.unpack_rgb_image(
          obs.observation.render_data.map).astype(np.int32)
      out["rgb_minimap"] = Feature.unpack_rgb_image(
          obs.observation.render_data.minimap).astype(np.int32)

    out["last_actions"] = np.array(
        [self.reverse_action(a).function for a in obs.actions],
        dtype=np.int32)

    out["action_result"] = np.array([o.result for o in obs.action_errors],
                                    dtype=np.int32)

    out["alerts"] = np.array(obs.observation.alerts, dtype=np.int32)

    out["game_loop"] = np.array([obs.observation.game_loop], dtype=np.int32)

    score_details = obs.observation.score.score_details
    out["score_cumulative"] = named_array.NamedNumpyArray([
        obs.observation.score.score,
        score_details.idle_production_time,
        score_details.idle_worker_time,
        score_details.total_value_units,
        score_details.total_value_structures,
        score_details.killed_value_units,
        score_details.killed_value_structures,
        score_details.collected_minerals,
        score_details.collected_vespene,
        score_details.collection_rate_minerals,
        score_details.collection_rate_vespene,
        score_details.spent_minerals,
        score_details.spent_vespene,
    ], names=ScoreCumulative, dtype=np.int32)

    def get_score_details(key, details, categories):
      row = getattr(details, key.name)
      return [getattr(row, category.name) for category in categories]

    out["score_by_category"] = named_array.NamedNumpyArray([
        get_score_details(key, score_details, ScoreCategories)
        for key in ScoreByCategory
    ], names=[ScoreByCategory, ScoreCategories], dtype=np.int32)

    out["score_by_vital"] = named_array.NamedNumpyArray([
        get_score_details(key, score_details, ScoreVitals)
        for key in ScoreByVital
    ], names=[ScoreByVital, ScoreVitals], dtype=np.int32)

    player = obs.observation.player_common
    out["player"] = named_array.NamedNumpyArray([
        player.player_id,
        player.minerals,
        player.vespene,
        player.food_used,
        player.food_cap,
        player.food_army,
        player.food_workers,
        player.idle_worker_count,
        player.army_count,
        player.warp_gate_count,
        player.larva_count,
    ], names=Player, dtype=np.int32)

    def unit_vec(u):
      return np.array((
          u.unit_type,
          u.player_relative,
          u.health,
          u.shields,
          u.energy,
          u.transport_slots_taken,
          int(u.build_progress * 100),  # discretize
      ), dtype=np.int32)

    ui = obs.observation.ui_data

    with sw("ui"):
      groups = np.zeros((10, 2), dtype=np.int32)
      for g in ui.groups:
        groups[g.control_group_index, :] = (g.leader_unit_type, g.count)
      out["control_groups"] = groups

      if ui.single:
        out["single_select"] = named_array.NamedNumpyArray(
            [unit_vec(ui.single.unit)], [None, UnitLayer])

      if ui.multi and ui.multi.units:
        out["multi_select"] = named_array.NamedNumpyArray(
            [unit_vec(u) for u in ui.multi.units], [None, UnitLayer])

      if ui.cargo and ui.cargo.passengers:
        out["single_select"] = named_array.NamedNumpyArray(
            [unit_vec(ui.single.unit)], [None, UnitLayer])
        out["cargo"] = named_array.NamedNumpyArray(
            [unit_vec(u) for u in ui.cargo.passengers], [None, UnitLayer])
        out["cargo_slots_available"] = np.array([ui.cargo.slots_available],
                                                dtype=np.int32)

      if ui.production and ui.production.build_queue:
        out["single_select"] = named_array.NamedNumpyArray(
            [unit_vec(ui.production.unit)], [None, UnitLayer])
        out["build_queue"] = named_array.NamedNumpyArray(
            [unit_vec(u) for u in ui.production.build_queue],
            [None, UnitLayer])

    def full_unit_vec(u, pos_transform, is_raw=False):
      screen_pos = pos_transform.fwd_pt(
          point.Point.build(u.pos))
      screen_radius = pos_transform.fwd_dist(u.radius)
      return np.array((
          # Match unit_vec order
          u.unit_type,
          u.alliance,  # Self = 1, Ally = 2, Neutral = 3, Enemy = 4
          u.health,
          u.shield,
          u.energy,
          u.cargo_space_taken,
          int(u.build_progress * 100),  # discretize

          # Resume API order
          int(u.health / u.health_max * 255) if u.health_max > 0 else 0,
          int(u.shield / u.shield_max * 255) if u.shield_max > 0 else 0,
          int(u.energy / u.energy_max * 255) if u.energy_max > 0 else 0,
          u.display_type,  # Visible = 1, Snapshot = 2, Hidden = 3
          u.owner,  # 1-15, 16 = neutral
          screen_pos.x,
          screen_pos.y,
          u.facing,
          screen_radius,
          u.cloak,  # Cloaked = 1, CloakedDetected = 2, NotCloaked = 3
          u.is_selected,
          u.is_blip,
          u.is_powered,
          u.mineral_contents,
          u.vespene_contents,

          # Not populated for enemies or neutral
          u.cargo_space_max,
          u.assigned_harvesters,
          u.ideal_harvesters,
          u.weapon_cooldown,
          len(u.orders),
          u.tag if is_raw else 0
      ), dtype=np.int32)

    raw = obs.observation.raw_data

    if aif.use_feature_units:
      with sw("feature_units"):
        # Update the camera location so we can calculate world to screen pos
        self._update_camera(point.Point.build(raw.player.camera))
        feature_units = []
        for u in raw.units:
          if u.is_on_screen and u.display_type != sc_raw.Hidden:
            feature_units.append(
                full_unit_vec(u, self._world_to_feature_screen_px))
        out["feature_units"] = named_array.NamedNumpyArray(
            feature_units, [None, FeatureUnit], dtype=np.int32)

    if aif.use_raw_units:
      with sw("raw_units"):
        raw_units = [full_unit_vec(u, self._world_to_world_tl, is_raw=True)
                     for u in raw.units]
        out["raw_units"] = named_array.NamedNumpyArray(
            raw_units, [None, FeatureUnit], dtype=np.int32)

    if aif.use_unit_counts:
      with sw("unit_counts"):
        unit_counts = collections.defaultdict(int)
        for u in raw.units:
          if u.alliance == sc_raw.Self:
            unit_counts[u.unit_type] += 1
        out["unit_counts"] = named_array.NamedNumpyArray(
            sorted(unit_counts.items()), [None, UnitCounts], dtype=np.int32)

    if aif.use_camera_position:
      camera_position = self._world_to_world_tl.fwd_pt(
          point.Point.build(raw.player.camera))
      out["camera_position"] = np.array((camera_position.x, camera_position.y),
                                        dtype=np.int32)

    out["available_actions"] = np.array(self.available_actions(obs.observation),
                                        dtype=np.int32)

    return out
912511
def reshape(self, shape: tf.TensorShape) -> 'TensorFluent':
        '''
        '''
        t = tf.reshape(self.tensor, shape)
        scope = self.scope.as_list()
        batch = self.batch
        return TensorFluent(t, scope, batch=batch)
584962
def unflatten(flat_dict, separator='_'):
    
    _unflatten_asserts(flat_dict, separator)

    # This global dictionary is mutated and returned
    unflattened_dict = dict()

    def _unflatten(dic, keys, value):
        for key in keys[:-1]:
            dic = dic.setdefault(key, {})

        dic[keys[-1]] = value

    for item in flat_dict:
        _unflatten(unflattened_dict, item.split(separator), flat_dict[item])

    return unflattened_dict
500835
def unpack(self, buff, offset=0):
        """"""
        begin = offset
        for name, value in list(self.get_class_attributes())[:-1]:
            size = self._unpack_attribute(name, value, buff, begin)
            begin += size
        self._unpack_attribute('oxm_match_fields', type(self).oxm_match_fields,
                               buff[:offset+self.length], begin)
1200825
def _inplace_subset_obs(self, index):
        
        adata_subset = self[index].copy()
        self._init_as_actual(adata_subset, dtype=self._X.dtype)