Parse content and metadata of an rdf file
483655
def read(self, source_path):
        """"""
        logger.debug("Loading graph described in "+source_path)
        graph = rdflib.Graph()
        graph.load(source_path)
        meta = {}
        queries = [
            f for f in listdir(self.settings["VOC_QUERIES_PATH"])
            if (isfile(join(self.settings["VOC_QUERIES_PATH"], f)) 
                and f.endswith(".sparql"))]
        for query_path in queries:
            query_file_path = self.settings["VOC_QUERIES_PATH"]+"/"+query_path
            with open(query_file_path, "r") as query_file:
                query = query_file.read()

                # The name of the query identifies the elements in the context
                query_key=query_path.split(".")[0]
                result_set = graph.query(query)
                # Each query result will be stored as a dictionnary in the
                # vocabulary context, referenced by the query name as its key.
                # Multiple results are stored in a list.
                for result in result_set:
                    if not query_key in meta.keys():
                        meta[query_key]=result.asdict()
                    elif type(meta[query_key]) == list:
                        meta[query_key].append(result.asdict())
                    else:
                        meta[query_key]=[meta[query_key], result.asdict()]
        meta["iri"] = meta["lov_metadata"]["iri"]
        meta["description"] = meta["lov_metadata"]["description"]
        meta["version"] = meta["lov_metadata"]["version"]
        meta["title"] = meta["lov_metadata"]["title"]
        return "", meta
248407
def extract_meta(self, text):
        
        first_line = True
        metadata = []
        content = []
        metadata_parsed = False

        for line in text.split('\n'):
            if first_line:
                first_line = False
                if line.strip() != '---':
                    raise MetaParseException('Invalid metadata')
                else:
                    continue
            if line.strip() == '' and not metadata_parsed:
                continue
            if line.strip() == '---' and not metadata_parsed:
                # reached the last line
                metadata_parsed = True
            elif not metadata_parsed:
                metadata.append(line)
            else:
                content.append(line)

        content = '\n'.join(content)
        try:
            metadata = yaml.load('\n'.join(metadata))
        except:
            raise
            content = text
            metadata = yaml.load('')

        return content, metadata
451702
def url_from_rdf(rdf, prop):
    '''
    
    '''
    value = rdf.value(prop)
    if isinstance(value, (URIRef, Literal)):
        return value.toPython()
    elif isinstance(value, RdfResource):
        return value.identifier.toPython()
977192
def process_file(self,filename=None, format=None):
        
        rdfgraph = rdflib.Graph()
        if format is None:
            if filename.endswith(".ttl"):
                format='turtle'
            elif filename.endswith(".rdf"):
                format='xml'
        rdfgraph.parse(filename, format=format)
        return self.process_rdfgraph(rdfgraph)
1280882
def get_meta(self):
        
        rdf = self.get_meta_rdf(fmt='n3')
        return ThingMeta(self, rdf, self._client.default_lang, fmt='n3')
870322
def parse_meta(content, meta_name, source_descr, content_attr_name="content"):
    
    dom = dhtmlparser.parseString(content)

    meta_tags = dom.find(
        "meta",
        fn=lambda x: x.params.get("name", "").lower() == meta_name.lower()
    )

    return [
        SourceString(tag.params[content_attr_name], source_descr)
        for tag in meta_tags
        if content_attr_name in tag.params
    ]
748887
def load_rdf(self,
                 uri_or_path=None,
                 data=None,
                 file_obj=None,
                 rdf_format="",
                 verbose=False,
                 hide_base_schemas=True,
                 hide_implicit_types=True,
                 hide_implicit_preds=True):
        """"""
        loader = RDFLoader(verbose=verbose)
        loader.load(uri_or_path, data, file_obj, rdf_format)
        self.rdflib_graph = loader.rdflib_graph
        self.sources = loader.sources_valid
        self.sparqlHelper = SparqlHelper(self.rdflib_graph)
        self.namespaces = sorted(self.rdflib_graph.namespaces())
1162739
def rdf_source(self, format="turtle"):
        
        s = self.rdflib_graph.serialize(format=format)
        if isinstance(s, bytes):
            s = s.decode('utf-8')
        return s
998018
def parse_meta_data(self, line):
        """"""
        line = line.rstrip()
        logger.debug("Parsing metadata line:{0}".format(line))
        line_info = line[2:].split('=')
        match = False

        if line_info[0] == 'fileformat':
            logger.debug("Parsing fileformat")
            try:
                self.fileformat = line_info[1]
                logger.debug("Found fileformat {0}".format(self.fileformat))
            except IndexError:
                raise SyntaxError("fileformat must have a value")

        elif line_info[0] == 'INFO':
            match = self.info_pattern.match(line)
            if not match:
                raise SyntaxError("One of the INFO lines is malformed:{0}".format(line))

            matches = [
                match.group('id'), match.group('number'),
                match.group('type'), match.group('desc')
            ]

            # extra_info is a dictionary to check the metadata about the INFO values:
            self.extra_info[matches[0]] = dict(
                zip(self.header_keys['info'][1:], matches[1:])
            )

            info_line = dict(list(zip(self.header_keys['info'],matches)))

            if len(info_line['Description'].split('Format:')) > 1:
                info_line['Format'] = [
                    info.strip() for info in info_line['Description'].split('Format:')
                ][-1]
            self.info_lines.append(info_line)

            # Store the vep columns:
            if info_line['ID'] == 'CSQ':
                self.vep_columns = info_line.get('Format', '').split('|')

            if info_line['ID'] == 'ANN':
                self.snpeff_columns = [
                    annotation.strip("' ") for annotation in
                    info_line.get('Description', '').split(':')[-1].split('|')]

            self.info_dict[match.group('id')] = line

        elif line_info[0] == 'FILTER':
            match = self.filter_pattern.match(line)
            if not match:
                raise SyntaxError("One of the FILTER lines is malformed: {0}".format(line))
            matches = [match.group('id'), match.group('desc')]
            self.filter_lines.append(dict(
                list(zip(self.header_keys['filt'],matches)))
            )
            self.filter_dict[match.group('id')] = line

        elif line_info[0] == 'contig':
            match = self.contig_pattern.match(line)
            if not match:
                print()
                raise SyntaxError("One of the contig lines is malformed: {0}".format(line))

            matches = [match.group('id'), match.group('length')]
            self.contig_lines.append(dict(
                list(zip(self.header_keys['contig'],matches)))
            )
            self.contig_dict[match.group('id')] = line

        elif line_info[0] == 'FORMAT':
            match = self.format_pattern.match(line)
            if not match:
                raise SyntaxError("One of the FORMAT lines is malformed: {0}".format(line))

            matches = [
                match.group('id'), match.group('number'),
                match.group('type'), match.group('desc')
            ]
            self.format_lines.append(dict(
                list(zip(self.header_keys['form'],matches)))
            )
            self.format_dict[match.group('id')] = line

        elif line_info[0] == 'ALT':
            match = self.alt_pattern.match(line)
            if not match:
                raise SyntaxError("One of the ALT lines is malformed: {0}".format(line))

            matches = [match.group('id'), match.group('desc')]
            self.alt_lines.append(dict(
                list(zip(self.header_keys['alt'],matches)))
            )
            self.alt_dict[match.group('id')] = line

        else:
            match = self.meta_pattern.match(line)
            if not match:
                raise SyntaxError("One of the meta data lines is malformed: {0}".format(line))

            self.other_lines.append({match.group('key'): match.group('val')})
            self.other_dict[match.group('key')] = line
1197310
def load_from_rdf_string(self, rdf_str):
        
        self.graph = rdflib.Graph()
        self.graph.parse(data=rdf_str, format='nt')
        self.initialize()
1301139
def parse_object_like_triples(self):

		'''
		
		'''

		# parse triples as object-like attributes in self.rdf.triples
		self.rdf.triples = SimpleNamespace() # prepare triples
		for s,p,o in self.rdf.graph:

			# get ns info
			ns_prefix, ns_uri, predicate = self.rdf.graph.compute_qname(p)

			# if prefix as list not yet added, add
			if not hasattr(self.rdf.triples, ns_prefix):
				setattr(self.rdf.triples, ns_prefix, SimpleNamespace())

			# same for predicate
			if not hasattr(getattr(self.rdf.triples, ns_prefix), predicate):
				setattr(getattr(self.rdf.triples, ns_prefix), predicate, [])

			# append object for this prefix
			getattr(getattr(self.rdf.triples, ns_prefix), predicate).append(o)
880108
def parsemetadata(self, node):
        """"""

        if 'type' in node.attrib:
            self.metadatatype = node.attrib['type']
        else:
            #no type specified, default to native
            self.metadatatype = "native"

        if 'src' in node.attrib:
            self.metadata = ExternalMetaData(node.attrib['src'])
        elif self.metadatatype == "native":
            self.metadata = NativeMetaData()
        else:
            self.metadata = None #may be set below to ForeignData

        for subnode in node:
            if subnode.tag == '{' + NSFOLIA + '}annotations':
                self.parsexmldeclarations(subnode)
            elif subnode.tag == '{' + NSFOLIA + '}meta':
                if self.metadatatype == "native":
                    if subnode.text:
                        self.metadata[subnode.attrib['id']] = subnode.text
                else:
                    raise MetaDataError("Encountered a meta element but metadata type is not native!")
            elif subnode.tag == '{' + NSFOLIA + '}provenance':
                #forward compatibility with FoLiA 2.0; ignore provenance
                print("WARNING: Ignoring provenance data. Use foliapy instead of pynlpl.formats.folia for FoLiA v2.0 compatibility!",file=sys.stderr)
                pass
            elif subnode.tag == '{' + NSFOLIA + '}foreign-data':
                if self.metadatatype == "native":
                    raise MetaDataError("Encountered a foreign-data element but metadata type is native!")
                elif self.metadata is not None:
                    #multiple foreign-data elements, chain:
                    e = self.metadata
                    while e.next is not None:
                        e = e.next
                    e.next = ForeignData(self, node=subnode)
                else:
                    self.metadata = ForeignData(self, node=subnode)
            elif subnode.tag == '{' + NSFOLIA + '}submetadata':
                self.parsesubmetadata(subnode)
            elif subnode.tag == '{http://www.mpi.nl/IMDI/Schema/IMDI}METATRANSCRIPT': #backward-compatibility for old IMDI without foreign-key
                E = ElementMaker(namespace=NSFOLIA,nsmap={None: NSFOLIA, 'xml' : "http://www.w3.org/XML/1998/namespace"})
                self.metadatatype = "imdi"
                self.metadata = ForeignData(self, node=subnode)
231751
def _parse(self):
        
        return MetadataFile(
            self.filename,
            self.raw_data,
            decoders.parse_meta(self.filename, self.raw_data)
        )
973433
def _parse_graph(self):

		'''
		
		'''

		# if resource exists, parse self.rdf.data
		if self.exists:
			self.rdf.graph = self.repo.api.parse_rdf_payload(self.rdf.data, self.headers)

		# else, create empty graph
		else:
			self.rdf.graph = rdflib.Graph()

		# bind any additional namespaces from repo instance, but do not override
		self.rdf.namespace_manager = rdflib.namespace.NamespaceManager(self.rdf.graph)
		for ns_prefix, ns_uri in self.rdf.prefixes.__dict__.items():
			self.rdf.namespace_manager.bind(ns_prefix, ns_uri, override=False)

		# conversely, add namespaces from parsed graph to self.rdf.prefixes
		for ns_prefix, ns_uri in self.rdf.graph.namespaces():
			setattr(self.rdf.prefixes, ns_prefix, rdflib.Namespace(ns_uri))
			setattr(self.rdf.uris, rdflib.Namespace(ns_uri), ns_prefix)

		# pin old graph to resource, create copy graph for modifications
		self.rdf._orig_graph = copy.deepcopy(self.rdf.graph)

		# parse triples for object-like access
		self.parse_object_like_triples()
1125521
def _parse_attribute_details_file(self, prop=ATTRIBUTES):
        

        # Parse content from remote file URL, which may be stored in one of two places:
        #    Starting at: contentInfo/MD_FeatureCatalogueDescription/featureCatalogueCitation
        #    ATTRIBUTE: href
        #    ELEMENT TEXT: CI_Citation/.../CI_Contact/onlineResource/CI_OnlineResource/linkage

        self._attr_details_file_url = parse_property(
            self._xml_tree, None, self._data_map, '_attributes_file'
        )
        if not self._attr_details_file_url:
            return None

        try:
            tree_to_parse = get_remote_element(self._attr_details_file_url)
        except Exception:
            self._attr_details_file_url = None
            return None

        xpath_map = self._data_structures[ATTRIBUTES]
        xpath_root = self._get_xroot_for(prop)

        return parse_complex_list(tree_to_parse, xpath_root, xpath_map, prop)
736505
def _iter_metadata_triples(cls, metadata_archive_path):
        
        pg_rdf_regex = re.compile(r'pg\d+.rdf$')
        with closing(tarfile.open(metadata_archive_path)) as metadata_archive:
            for item in metadata_archive:
                if pg_rdf_regex.search(item.name):
                    with disable_logging():
                        extracted = metadata_archive.extractfile(item)
                        graph = Graph().parse(extracted)
                    for fact in graph:
                        if cls._metadata_is_invalid(fact):
                            logging.info('skipping invalid triple %s', fact)
                        else:
                            yield fact
468143
def load_uri(self, uri, verbose):
        

        if verbose: printDebug("----------")
        if verbose: printDebug("Reading: <%s>" % uri)
        success = False
        for f in self.rdf_format_opts:
            if verbose: printDebug(".. trying rdf serialization: <%s>" % f)
            try:
                self.rdfgraph.parse(uri, format=f)
                if verbose: printDebug("..... success!", bold=True)
                success = True
                self.sources_valid += [uri]
                break
            except:
                if verbose: printDebug("..... failed")

        if not success == True:
            self.loading_failed(self.rdf_format_opts)
            self.sources_invalid += [uri]