Parse an internal wikilink at the head of the wikicode string.
496666
def _parse_wikilink(self):
        """"""
        reset = self._head + 1
        self._head += 2
        try:
            # If the wikilink looks like an external link, parse it as such:
            link, extra, delta = self._really_parse_external_link(True)
        except BadRoute:
            self._head = reset + 1
            try:
                # Otherwise, actually parse it as a wikilink:
                wikilink = self._parse(contexts.WIKILINK_TITLE)
            except BadRoute:
                self._head = reset
                self._emit_text("[[")
            else:
                self._emit(tokens.WikilinkOpen())
                self._emit_all(wikilink)
                self._emit(tokens.WikilinkClose())
        else:
            if self._context & contexts.EXT_LINK_TITLE:
                # In this exceptional case, an external link that looks like a
                # wikilink inside of an external link is parsed as text:
                self._head = reset
                self._emit_text("[[")
                return
            self._emit_text("[")
            self._emit(tokens.ExternalLinkOpen(brackets=True))
            self._emit_all(link)
            self._emit(tokens.ExternalLinkClose())
1176894
def parseExternalSubset(self, ExternalID, SystemID):
        
        libxml2mod.xmlParseExternalSubset(self._o, ExternalID, SystemID)
867932
private String parseLink(final String link) {
		String[] tokens = link.substring(7, link.length() - 1).split("\\s");
		if (tokens.length == 1) {
			return tokens[0];
		}
		StringBuilder result = new StringBuilder();

		boolean parametersSeen = false;
		boolean inParameters = false;

		for (int index = 0; index < tokens.length; index++) {
			result.append(" ").append(tokens[index]);
			if (tokens[index].indexOf('(') != -1 && !parametersSeen) {
				inParameters = true;
			}
			if (index == 1 && !inParameters) {
				result = new StringBuilder(tokens[index]);
			}
			if (tokens[index].indexOf(')') != -1 && !parametersSeen) {
				parametersSeen = true;
				if (index != tokens.length - 1) {
					result = new StringBuilder();
				}

			}

		}

		return result.toString().trim();
	}
817296
private String processLink(JsonParser jsonParser) throws IOException {
        final String link = jsonParser.getText();
        if (link.contains(SVC_EXTENSION)) {
            return link.substring(link.indexOf(SVC_EXTENSION) + SVC_EXTENSION.length());
        }
        return link;
    }
1120432
def import_wikilink(self, title, href, **attrs):
        
        return WikiLinks(self.requester).import_(self.id, title, href, **attrs)
394583
def parse_id_literal(ast, _variables=None):
    """"""
    if isinstance(ast, (StringValueNode, IntValueNode)):
        return ast.value
    return INVALID
1058975
public void processDocument(WikiDoc doc) {

        String rawArticleName = doc.name;        
        String articleName = StringUtils.unescapeHTML(rawArticleName);
        articleName = articleName.trim().toLowerCase();

        // skip articles that are not text-based or are
        // wikipedia-specific
        if (!isArticleLink(articleName)) {
            LOGGER.fine("skipping non-article document: " + articleName);
            return;
        } 
        // Skip processing articles that exist solely to direct the user to a
        // different article
        else if (articleName.indexOf("#REDIRECT") >= 0 ||
                   doc.text.indexOf("#REDIRECT") >= 0) {
            LOGGER.fine("skipping redirect: " + articleName);
            return;
        }
        LOGGER.log(Level.FINE, "Procesing article {0} with {1} characters",
                   new Object[] { articleName, doc.text.length() });
    
        // Gets the raw tokens contained in the article XML
        StringBuilder rawArticleText = doc.text;

        // Remove all the header and footer information from the article
        LOGGER.finer("extracting raw article text");        
        extractArticle(rawArticleText);

        // Remove any tables
        LOGGER.finer("removing tables");
        removeTables(rawArticleText);       

        // Remove the {{ text }} content from the document first as it sometimes
        // contains [[ text ]], which would be needlessly processed prior to
        // removal if this were called later.
        LOGGER.finer("removing {{text}} from article");
        removeDoubleBraceMarkup(rawArticleText);

        // Next remove the [[ link ]] markup
        LOGGER.finer("removing [[wiki-link]] from article");
        removeWikiLinkMarkup(rawArticleText, articleName);

        // Once the wiki-links are removed, go after external links, which are
        // only a single [link].  Occassinally, these will have text.
        LOGGER.finer("removing [external-link] from article");
        removeExternalLinkMarkup(rawArticleText);
                
        // Once all of the wiki markup has been removed, replace all of the
        // encoded HTML with its equivalents
        LOGGER.finer("unescaping HTML");
        StringUtils.unescapeHTML(rawArticleText);

        // Remove auto-generated Wikipedia warnings which appear in HTML comment
        // text
        LOGGER.finer("removing HTML comments");
        removeHtmlComments(rawArticleText);
        
        String article = rawArticleText.toString();

        // Being removing any tokens according to the options
        if (options.contains(CleanerOption.USE_PREPROCESSOR)) {
            LOGGER.finer("applying preprocessor");
            article = new DocumentPreprocessor().process(article);
        }
        if (options.contains(CleanerOption.FILTER_TOKENS)) {
            LOGGER.finer("filtering tokens");
            article = filterTokens(article);
        }
            
        // Count how many tokens remain in the document after all of the
        // processing stages.  If too few remain, do not write the document
        int finalTokenCount = getTokenCount(article);
        if (finalTokenCount < minTokensPerArticle) {
            LOGGER.log(Level.FINE, "Document {0} contained only {1} tokens"
                       + " and was not printed", new Object[] {
                           articleName, finalTokenCount });
            return;
        }

        if (options.contains(CleanerOption.INCLUDE_TITLES)) {
            processedArticleWriter.print(articleName);
            processedArticleWriter.print(" ");
        }
        
        // Write the remaining part of the article
        processedArticleWriter.println(article);
        processedArticleWriter.flush();
    }
381225
private int findEndOfNowiki (int startBlock) {
        // NOTE: this method could step back one char from startBlock position
        int endBlock=startBlock-3;
        do {
            endBlock=wikiText.indexOf("}}}", endBlock+3);
            if (endBlock<0) return wikiLength; // no matching '}}}' found
            while (endBlock+3<wikiLength && wikiChars[endBlock+3]=='}')
                endBlock++; // shift to end of sequence of more than 3x'}' (eg. '}}}}}')
        } while (wikiChars[endBlock-1]=='~');
        return endBlock;
    }
484350
def parse_lheading(self, m):
        """"""
        level = 1 if m.group(2) == '=' else 2
        self.renderer.heading(m.group(1), level=level)
1129668
def decode_link(self, link):
        

        if link.HasField("bucket"):
            bucket = bytes_to_str(link.bucket)
        else:
            bucket = None
        if link.HasField("key"):
            key = bytes_to_str(link.key)
        else:
            key = None
        if link.HasField("tag"):
            tag = bytes_to_str(link.tag)
        else:
            tag = None

        return (bucket, key, tag)
485415
def parse_link(value):
    

    links = []

    replace_chars = ' \'"'

    value = value.strip(replace_chars)
    if not value:
        return links

    for val in re.split(', *<', value):
        try:
            url, params = val.split(';', 1)
        except ValueError:
            url, params = val, ''

        link = {'url': url.strip('<> \'"')}

        for param in params.split(';'):
            try:
                key, value = param.split('=')
            except ValueError:
                break

            link[key.strip(replace_chars)] = value.strip(replace_chars)

        links.append(link)

    return links
815218
def _parse(self, context=0, push=True):
        """"""
        if push:
            self._push(context)
        while True:
            this = self._read()
            if self._context & contexts.UNSAFE:
                if not self._verify_safe(this):
                    if self._context & contexts.DOUBLE:
                        self._pop()
                    self._fail_route()
            if this not in self.MARKERS:
                self._emit_text(this)
                self._head += 1
                continue
            if this is self.END:
                return self._handle_end()
            next = self._read(1)
            if this == next == "{":
                if self._can_recurse():
                    self._parse_template_or_argument()
                else:
                    self._emit_text("{")
            elif this == "|" and self._context & contexts.TEMPLATE:
                self._handle_template_param()
            elif this == "=" and self._context & contexts.TEMPLATE_PARAM_KEY:
                self._handle_template_param_value()
            elif this == next == "}" and self._context & contexts.TEMPLATE:
                return self._handle_template_end()
            elif this == "|" and self._context & contexts.ARGUMENT_NAME:
                self._handle_argument_separator()
            elif this == next == "}" and self._context & contexts.ARGUMENT:
                if self._read(2) == "}":
                    return self._handle_argument_end()
                else:
                    self._emit_text("}")
            elif this == next == "[" and self._can_recurse():
                if not self._context & contexts.NO_WIKILINKS:
                    self._parse_wikilink()
                else:
                    self._emit_text("[")
            elif this == "|" and self._context & contexts.WIKILINK_TITLE:
                self._handle_wikilink_separator()
            elif this == next == "]" and self._context & contexts.WIKILINK:
                return self._handle_wikilink_end()
            elif this == "[":
                self._parse_external_link(True)
            elif this == ":" and self._read(-1) not in self.MARKERS:
                self._parse_external_link(False)
            elif this == "]" and self._context & contexts.EXT_LINK_TITLE:
                return self._pop()
            elif this == "=" and not self._global & contexts.GL_HEADING:
                if self._read(-1) in ("\n", self.START):
                    self._parse_heading()
                else:
                    self._emit_text("=")
            elif this == "=" and self._context & contexts.HEADING:
                return self._handle_heading_end()
            elif this == "\n" and self._context & contexts.HEADING:
                self._fail_route()
            elif this == "&":
                self._parse_entity()
            elif this == "<" and next == "!":
                if self._read(2) == self._read(3) == "-":
                    self._parse_comment()
                else:
                    self._emit_text(this)
            elif this == "<" and next == "/" and self._read(2) is not self.END:
                if self._context & contexts.TAG_BODY:
                    self._handle_tag_open_close()
                else:
                    self._handle_invalid_tag_start()
            elif this == "<" and not self._context & contexts.TAG_CLOSE:
                if self._can_recurse():
                    self._parse_tag()
                else:
                    self._emit_text("<")
            elif this == ">" and self._context & contexts.TAG_CLOSE:
                return self._handle_tag_close_close()
            elif this == next == "'" and not self._skip_style_tags:
                result = self._parse_style()
                if result is not None:
                    return result
            elif self._read(-1) in ("\n", self.START) and this in ("#", "*", ";", ":"):
                self._handle_list()
            elif self._read(-1) in ("\n", self.START) and (
                    this == next == self._read(2) == self._read(3) == "-"):
                self._handle_hr()
            elif this in ("\n", ":") and self._context & contexts.DL_TERM:
                self._handle_dl_term()
                if this == "\n":
                    # Kill potential table contexts
                    self._context &= ~contexts.TABLE_CELL_LINE_CONTEXTS
            # Start of table parsing
            elif this == "{" and next == "|" and (
                    self._read(-1) in ("\n", self.START) or
                    (self._read(-2) in ("\n", self.START) and self._read(-1).isspace())):
                if self._can_recurse():
                    self._parse_table()
                else:
                    self._emit_text("{")
            elif self._context & contexts.TABLE_OPEN:
                if this == next == "|" and self._context & contexts.TABLE_TD_LINE:
                    if self._context & contexts.TABLE_CELL_OPEN:
                        return self._handle_table_cell_end()
                    self._handle_table_cell("||", "td", contexts.TABLE_TD_LINE)
                elif this == next == "|" and self._context & contexts.TABLE_TH_LINE:
                    if self._context & contexts.TABLE_CELL_OPEN:
                        return self._handle_table_cell_end()
                    self._handle_table_cell("||", "th", contexts.TABLE_TH_LINE)
                elif this == next == "!" and self._context & contexts.TABLE_TH_LINE:
                    if self._context & contexts.TABLE_CELL_OPEN:
                        return self._handle_table_cell_end()
                    self._handle_table_cell("!!", "th", contexts.TABLE_TH_LINE)
                elif this == "|" and self._context & contexts.TABLE_CELL_STYLE:
                    return self._handle_table_cell_end(reset_for_style=True)
                # on newline, clear out cell line contexts
                elif this == "\n" and self._context & contexts.TABLE_CELL_LINE_CONTEXTS:
                    self._context &= ~contexts.TABLE_CELL_LINE_CONTEXTS
                    self._emit_text(this)
                elif (self._read(-1) in ("\n", self.START) or
                      (self._read(-2) in ("\n", self.START) and self._read(-1).isspace())):
                    if this == "|" and next == "}":
                        if self._context & contexts.TABLE_CELL_OPEN:
                            return self._handle_table_cell_end()
                        if self._context & contexts.TABLE_ROW_OPEN:
                            return self._handle_table_row_end()
                        return self._handle_table_end()
                    elif this == "|" and next == "-":
                        if self._context & contexts.TABLE_CELL_OPEN:
                            return self._handle_table_cell_end()
                        if self._context & contexts.TABLE_ROW_OPEN:
                            return self._handle_table_row_end()
                        self._handle_table_row()
                    elif this == "|":
                        if self._context & contexts.TABLE_CELL_OPEN:
                            return self._handle_table_cell_end()
                        self._handle_table_cell("|", "td", contexts.TABLE_TD_LINE)
                    elif this == "!":
                        if self._context & contexts.TABLE_CELL_OPEN:
                            return self._handle_table_cell_end()
                        self._handle_table_cell("!", "th", contexts.TABLE_TH_LINE)
                    else:
                        self._emit_text(this)
                else:
                    self._emit_text(this)

            else:
                self._emit_text(this)
            self._head += 1
194648
def wiki(searchterm):
    """"""
    searchterm = quote(searchterm)

    url = "https://en.wikipedia.org/w/api.php?action=query&list=search&srsearch={0}&format=json"
    url = url.format(searchterm)

    result = requests.get(url).json()

    pages = result["query"]["search"]

    # try to reject disambiguation pages
    pages = [p for p in pages if 'may refer to' not in p["snippet"]]

    if not pages:
        return ""

    page = quote(pages[0]["title"].encode("utf8"))
    link = "http://en.wikipedia.org/wiki/{0}".format(page)

    r = requests.get(
        "http://en.wikipedia.org/w/api.php?format=json&action=parse&page={0}".
        format(page)).json()
    soup = BeautifulSoup(r["parse"]["text"]["*"], "html5lib")
    p = soup.find('p').get_text()
    p = p[:8000]

    return u"{0}\n{1}".format(p, link)
919321
def parse_header(cls, value='*/*'):
        """"""
        yield from sorted((
            cls.from_string(token.strip())
            for token in value.split(',')
            if token.strip()
        ), reverse=True)
480782
def from_string(cls, link):
        
        ma = cls._pattern.search(link)
        if ma is None:
            raise ValueError(link)
        id = ma.group('id')
        return cls(id)
1037139
public void removeWikiLinkMarkup(StringBuilder article, String title) {
        int bracketStart = article.indexOf("[[");
        boolean includeLinkText = 
            options.contains(CleanerOption.INCLUDE_LINK_TEXT);
        while (bracketStart >= 0) {
            
            // grab the linked article name which is all text to the next ]], or
            // to
            int bracketEnd = article.indexOf("]]", bracketStart);
            // If there wasn't a matching closing bracket (i.e. malformatted
            // wiki), then abort the replacement
            if (bracketEnd < 0)
                break;
            
            // If the link text is supposed to be included in the document, then
            // strip out the pertinent text.  However, ensure that the link
            // points to an article, which filters out non-content links in the
            // article headers and footers
            if (includeLinkText && 
                isArticleLink(article.substring(bracketStart+2, 
                                                bracketEnd), title)) {

                // the link may also have a text description that replaces the
                // link text, e.g.  [[article title|link text]].
                int optionalLinkDescriptionStart = 
                    article.indexOf("|", bracketStart);
                // When selecting the optional text, ensure that the | delimeter
                // falls within the link structure itself
                int linkTextStart = 
                    (optionalLinkDescriptionStart >= 0 && 
                     optionalLinkDescriptionStart < bracketEnd)
                    ? optionalLinkDescriptionStart + 1
                    : bracketStart + 2;
                // Parse out the link text
                String linkText = article.substring(linkTextStart, bracketEnd);
                // Then replace the entire link with the desired text
                article.replace(bracketStart, bracketEnd+2, linkText);
            }
            // If the link text isn't to be used in the document, remove it
            // completely
            else {
                article.delete(bracketStart, bracketEnd + 2);
            }
            bracketStart = article.indexOf("[[", bracketStart);
        }   
    }
468469
public static Link parse(String origin, String link, boolean allowModuleName) {
        String originMod = origin;
        int index = link.indexOf('#');
        if (index == -1) {
            if (allowModuleName) {
                index = link.lastIndexOf('/');
                if (index == -1) {
                    return new Link(null, null, link);
                }

                return new Link(null, link.substring(0, index), link.substring(index + 1));
            }

            return new Link(null, null, link);
        }

        String name = link.substring(index + 1);
        String uri = link.substring(0, index);
        if (originMod == null) {
            originMod = "";
        } else {
            index = originMod.lastIndexOf('/');
            originMod = index == -1 ? "" : originMod.substring(0, index);
        }

        if (uri.startsWith("../")) {
            uri = uri.substring(3);
        }

        while (uri.startsWith("../")) {
            index = originMod.lastIndexOf('/');
            if (index == -1) {
                originMod = "";
            } else {
                originMod = originMod.substring(0, index);
            }
            uri = uri.substring(3);
        }

        return new Link(originMod.isEmpty() ? uri : originMod + '/' + uri, null, name);
    }