Format metadata attributes for and indexed array

    This function is used to provide the necessary metadata to meet
    the (proposed) LIGO Common Data Format specification for series data
    in HDF5.
484490
def format_index_array_attrs(series):
    
    attrs = {}
    # loop through named axes
    for i, axis in zip(range(series.ndim), ('x', 'y')):
        # find property names
        unit = '{}unit'.format(axis)
        origin = '{}0'.format(axis)
        delta = 'd{}'.format(axis)

        # store attributes
        aunit = getattr(series, unit)
        attrs.update({
            unit: str(aunit),
            origin: getattr(series, origin).to(aunit).value,
            delta: getattr(series, delta).to(aunit).value,
        })
    return attrs
114442
def parse_meta(self, selected_meta_data):
        
        # reads all meta data files
        files = self._get_files("meta", self.path)
        df = pd.DataFrame()
        print("Parsing the metadata files...")
        for f in tqdm(files):
            data = self.parse_single_meta(f, selected_meta_data)
            if data is not None:
                df = pd.concat([df, data], axis=0)
        df.index = df['sample']
        #
        # df = df.drop('sample', 1)  # 1 for the columns
        return df
648133
def shaped_metadata(self):
        """"""
        if not self.is_shaped:
            return None
        return tuple(json_description_metadata(s.pages[0].is_shaped)
                     for s in self.series if s.kind.lower() == 'shaped')
172064
def imagej_metadata(data, bytecounts, byteorder):
    
    def _string(data, byteorder):
        return data.decode('utf-16' + {'>': 'be', '<': 'le'}[byteorder])

    def _doubles(data, byteorder):
        return struct.unpack(byteorder+('d' * (len(data) // 8)), data)

    def _lut(data, byteorder):
        return numpy.frombuffer(data, 'uint8').reshape(-1, 256)

    def _bytes(data, byteorder):
        return data

    metadata_types = {  # big-endian
        b'info': ('Info', _string),
        b'labl': ('Labels', _string),
        b'rang': ('Ranges', _doubles),
        b'luts': ('LUTs', _lut),
        b'plot': ('Plots', _bytes),
        b'roi ': ('ROI', _bytes),
        b'over': ('Overlays', _bytes)}
    metadata_types.update(  # little-endian
        dict((k[::-1], v) for k, v in metadata_types.items()))

    if not bytecounts:
        raise ValueError('no ImageJ metadata')

    if not data[:4] in (b'IJIJ', b'JIJI'):
        raise ValueError('invalid ImageJ metadata')

    header_size = bytecounts[0]
    if header_size < 12 or header_size > 804:
        raise ValueError('invalid ImageJ metadata header size')

    ntypes = (header_size - 4) // 8
    header = struct.unpack(byteorder+'4sI'*ntypes, data[4:4+ntypes*8])
    pos = 4 + ntypes * 8
    counter = 0
    result = {}
    for mtype, count in zip(header[::2], header[1::2]):
        values = []
        name, func = metadata_types.get(mtype, (bytes2str(mtype), read_bytes))
        for _ in range(count):
            counter += 1
            pos1 = pos + bytecounts[counter]
            values.append(func(data[pos:pos1], byteorder))
            pos = pos1
        result[name.strip()] = values[0] if count == 1 else values
    return result
358706
def read_format_from_metadata(text, ext):
    """"""
    metadata = read_metadata(text, ext)
    rearrange_jupytext_metadata(metadata)
    return format_name_for_ext(metadata, ext, explicit_default=False)
122048
private void buildMetadataColumnMap(JsonArray array) {
    if (array != null) {
      for (JsonElement columnElement : array) {
        Schema schemaObj = gson.fromJson(columnElement, Schema.class);
        String columnName = schemaObj.getColumnName();
        this.metadataColumnMap.put(columnName.toLowerCase(), schemaObj);
        this.metadataColumnList.add(columnName.toLowerCase());
      }
    }
  }
552007
def define_data_format(self):
        
        data = javabridge.call(self.jobject, "defineDataFormat", "()Lweka/core/Instances;")
        if data is None:
            return None
        else:
            return Instances(data)
637142
def _to_meta_data(pif_obj, dataset_hit, mdf_acl):
    """"""
    pif = pif_obj.as_dictionary()
    dataset = dataset_hit.as_dictionary()
    mdf = {}
    try:
        if pif.get("names"):
            mdf["title"] = pif["names"][0]
        else:
            mdf["title"] = "Citrine PIF " + str(pif["uid"])

        if pif.get("chemicalFormula"):
            mdf["composition"] = pif["chemicalFormula"]
        elif pif.get("composition"):
            mdf["composition"] = ''.join([comp["element"] for comp in pif["composition"] if comp["element"]])
        if not mdf["composition"]:
            mdf.pop("composition")

        mdf["acl"] = mdf_acl
        mdf["source_name"] = _construct_new_key(dataset["name"])

        if pif.get("contacts"):
            mdf["data_contact"] = []
            for contact in pif["contacts"]:
                data_c = {
                    "given_name": contact["name"]["given"],  #REQ
                    "family_name": contact["name"]["family"]  #REQ
                    }
                if contact.get("email"):
                    data_c["email"] = contact.get("email", "")
                if contact.get("orcid"):
                    data_c["orcid"] = contact.get("orcid", "")
                mdf["data_contact"].append(data_c)
            if not mdf["data_contact"]:
                mdf.pop("data_contact")
        
        mdf["data_contributor"] = [{}] 
        if "owner" in dataset:
            name = dataset["owner"].split()
            contributor = {
                "given_name": name[0],
                "family_name": name[1],
                "email": dataset["email"]
            }
            mdf["data_contributor"] = [contributor]

        mdf["links"] = {
            "landing_page": "https://citrination.com/datasets/{}".format(dataset["id"]),
            "publication": []
            }
        if pif.get("references"):
            mdf["author"] = []
            mdf["citation"] = []
            for ref in pif["references"]:
                if ref.get("doi"):
                    mdf["citation"].append(ref["doi"]) #TODO: Make actual citation
                    mdf["links"]["publication"].append(ref["doi"])
                if ref.get("authors"):
                    for author in ref["authors"]:
                        if author.get("given") and author.get("family"):
                            mdf["author"].append({
                                "given_name": author["given"],
                                "family_name": author["family"]
                                })
            # Remove fields if blank
            if not mdf["author"]:
                mdf.pop("author")
            if not mdf["citation"]:
                mdf.pop("citation")
        if not mdf["links"]["publication"]:
            mdf["links"].pop("publication")

        if pif.get("licenses", [{}])[0].get("url"):
            mdf["license"] = pif["licenses"][0]["url"]
        if pif.get("tags"):
            mdf["tags"] = pif["tags"]

    # If required MDF metadata is missing from PIF, abort
    except KeyError as e:
        print("Error: Required MDF metadata", str(e), "not found in PIF", pif["uid"])
        return None

    return mdf
793606
def _format_data(self, name=None):
        

        # do we want to justify (only do so for non-objects)
        is_justify = not (self.inferred_type in ('string', 'unicode') or
                          (self.inferred_type == 'categorical' and
                           is_object_dtype(self.categories)))

        return format_object_summary(self, self._formatter_func,
                                     is_justify=is_justify, name=name)
1202143
def create_data_and_metadata_from_data(self, data: numpy.ndarray, intensity_calibration: CalibrationModule.Calibration=None, dimensional_calibrations: typing.List[CalibrationModule.Calibration]=None, metadata: dict=None, timestamp: str=None) -> DataAndMetadata.DataAndMetadata:
        
        return self.create_data_and_metadata(numpy.copy(data), intensity_calibration, dimensional_calibrations, metadata, timestamp)
1167347
def get_movielens_item_metadata(use_item_ids):
    

    features = {}
    genre_set = set()

    for line in _get_movie_raw_metadata():

        if not line:
            continue

        splt = line.split('|')
        item_id = int(splt[0])

        genres = [idx for idx, val in
                  zip(range(len(splt[5:])), splt[5:])
                  if int(val) > 0]

        if use_item_ids:
            # Add item-specific features too
            genres.append(item_id)

        for genre_id in genres:
            genre_set.add(genre_id)

        features[item_id] = genres

    mat = sp.lil_matrix((len(features) + 1,
                         len(genre_set)),
                        dtype=np.int32)

    for item_id, genre_ids in features.items():
        for genre_id in genre_ids:
            mat[item_id, genre_id] = 1

    return mat
791538
def register_hdf5_array_io(array_type, format='hdf5', identify=True):
    
    def from_hdf5(*args, **kwargs):
        """Read an array from HDF5
        """
        kwargs.setdefault('array_type', array_type)
        return read_hdf5_array(*args, **kwargs)

    io_registry.register_reader(format, array_type, from_hdf5)
    if issubclass(array_type, Series):
        io_registry.register_writer(format, array_type, write_hdf5_series)
    else:
        io_registry.register_writer(format, array_type, write_hdf5_array)

    if identify:
        io_registry.register_identifier(format, array_type,
                                        io_hdf5.identify_hdf5)
1361981
def load_metadata(self, data_dir, feature_name=None):
    """"""
    # Restore names if defined
    names_filepath = _get_names_filepath(data_dir, feature_name)
    if tf.io.gfile.exists(names_filepath):
      self.names = _load_names_from_file(names_filepath)
622706
@Override
    public void writeMetadata(final byte[] bytes, final IHLLMetadata metadata) {
        final HLLType type = metadata.HLLType();
        final int typeOrdinal = getOrdinal(type);

        final int explicitCutoffValue;
        if(metadata.explicitOff()) {
            explicitCutoffValue = EXPLICIT_OFF;
        } else if(metadata.explicitAuto()) {
            explicitCutoffValue = EXPLICIT_AUTO;
        } else {
            explicitCutoffValue = metadata.log2ExplicitCutoff() + 1/*per spec*/;
        }

        bytes[0] = SerializationUtil.packVersionByte(SCHEMA_VERSION, typeOrdinal);
        bytes[1] = SerializationUtil.packParametersByte(metadata.registerWidth(), metadata.registerCountLog2());
        bytes[2] = SerializationUtil.packCutoffByte(explicitCutoffValue, metadata.sparseEnabled());
    }
480605
def apply_meta(self, arr, funcs):
        
        if arr is None:
            logging.error("unable to convert data: emtpy section")
            return None

        tmp = np.copy(arr)
        for col in arr.dtype.names:
            for row in self._get_row_hdrs():
                meta = self._get_meta(row, col)
                for mk, mv in sorted(meta.iteritems()):
                    if mk in funcs.iterkeys():
                        tmp[col][self.irt[row]] = \
                            funcs[mk](tmp[col][self.irt[row]], mv)
        return tmp
104148
def generate_metadata(self):
        # type: (Descriptor) -> dict
        
        genmeta = {}
        encmeta = {}
        # page align md5
        if (self.must_compute_md5 and
                self._ase.mode == blobxfer.models.azure.StorageModes.Page):
            aligned = blobxfer.util.page_align_content_length(self._offset)
            diff = aligned - self._offset
            if diff > 0:
                with self._hasher_lock:
                    self.md5.update(b'\0' * diff)
        # generate encryption metadata
        if self._ase.is_encrypted:
            if self.must_compute_md5:
                md5digest = blobxfer.util.base64_encode_as_string(
                    self.md5.digest())
            else:
                md5digest = None
            if self.hmac is not None:
                hmacdigest = blobxfer.util.base64_encode_as_string(
                    self.hmac.digest())
            else:
                hmacdigest = None
            encmeta = self._ase.encryption_metadata.convert_to_json_with_mac(
                md5digest, hmacdigest)
        # generate file attribute metadata
        if self._store_file_attr and not self.local_path.use_stdin:
            merged = blobxfer.models.metadata.generate_fileattr_metadata(
                self.local_path, genmeta)
            if merged is not None:
                genmeta = merged
        # generate vectored io metadata
        if self.local_path.view.mode == VectoredIoDistributionMode.Stripe:
            merged = blobxfer.models.metadata.\
                generate_vectored_io_stripe_metadata(self.local_path, genmeta)
            if merged is not None:
                genmeta = merged
        if len(encmeta) > 0:
            metadata = encmeta
        else:
            metadata = {}
        if len(genmeta) > 0:
            metadata[blobxfer.models.metadata.JSON_KEY_BLOBXFER_METADATA] = \
                json.dumps(genmeta, ensure_ascii=False, sort_keys=True)
        if len(metadata) == 0:
            return None
        return metadata
717935
def metadata(self):
        

        sheet = self.result.add_sheet("metadata")
        self.header(sheet, "metadata")

        n_row = 1  # row number

        for k in self.po.metadata:
            row = sheet.row(n_row)
            row.write(0, k)
            row.write(1, self.po.metadata[k])
            n_row += 1
            sheet.flush_row_data()