parse the text (tokenize, tag..etc)
972748
function (obj, contents, weight, categoryWeights) {
        obj._gloss.parse(contents);
        var current = obj._gloss.root;
        var prev = null;
        do {
            var s = current.toJSON();
            if (!s.children && !s.orig && !s.isFiltered) {
                // increment total and add/incr bidirectional edges for current node
                prev = obj._incrEdges(id, obj._fmtNode(s), prev, weight, categoryWeights);
            } else if (!s.isFiltered) {
                if (s.orig && s.children) {
                    // increment total and add bidirectional edge weight for original node
                    if (!s.orig.isFiltered) {
                        prev = obj._incrEdges(id, obj._fmtNode(s.orig), prev, weight, categoryWeights);
                    }

                    for (var i = 0; i < s.children.length; i++) {
                        // increment total and add/incr bidirectional edges on child
                        if (!s.children[i].isFiltered) {
                            prev = obj._incrEdges(id, obj._fmtNode(s.children[i]), prev, weight, categoryWeights);
                        }
                    }
                }
            }

            current = current.next;
        } while (current);
    }
855004
def tokenize(self, text, context=0, skip_style_tags=False):
        """"""
        split = self.regex.split(text)
        self._text = [segment for segment in split if segment]
        self._head = self._global = self._depth = 0
        self._bad_routes = set()
        self._skip_style_tags = skip_style_tags

        try:
            tokens = self._parse(context)
        except BadRoute:  # pragma: no cover (untestable/exceptional case)
            raise ParserError("Python tokenizer exited with BadRoute")
        if self._stacks:  # pragma: no cover (untestable/exceptional case)
            err = "Python tokenizer exited with non-empty token stack"
            raise ParserError(err)
        return tokens
557202
protected function parseSingleQuotes()
	{
		if (!$this->hasSingleQuote)
		{
			return;
		}

		$regexp = "/(?<=\\pL)'|(?<!\\S)'(?=\\pL|[0-9]{2})/uS";
		preg_match_all($regexp, $this->text, $matches, PREG_OFFSET_CAPTURE);
		foreach ($matches[0] as $m)
		{
			// Give this tag a worse priority than default so that quote pairs take precedence
			$this->addTag($m[1], 1, "\xE2\x80\x99", 10);
		}
	}
1319461
def tag(s, tokenize=True, encoding="utf-8", **kwargs):
    
    tags = []
    for sentence in parse(s, tokenize, True, False, False, False, encoding, **kwargs).split():
        for token in sentence:
            tags.append((token[0], token[1]))
    return tags
1100280
private ArrayList<ArrayList<String>> parseString(String text) {

        ArrayList<ArrayList<String>> result = new ArrayList<ArrayList<String>>();
        StringTokenizer linetoken = new StringTokenizer(text, "\n");
        StringTokenizer token;
        String current;
        while (linetoken.hasMoreTokens()) {
            current = linetoken.nextToken();
            if (current.contains(",")) {
                token = new StringTokenizer(current, ",");
            } else {
                token = new StringTokenizer(current);
            }
            ArrayList<String> line = new ArrayList<String>();
            while (token.hasMoreTokens()) {
                line.add(token.nextToken());
            }
            result.add(line);
        }
        return result;
    }
1245764
def parse(self, text):
        """"""
        text = self.preprocess(text)
        token_stack = []
        last_pos = 0

        # Iterate through all matched tokens
        for match in self.regex.finditer(text):
            # Find which token has been matched by regex
            token, match_type, group = self.get_matched_token(match)

            # Get params from stack of tokens
            params = self.get_params(token_stack)

            # Should we skip interpreting tokens?
            skip = token_stack[-1].skip if token_stack else False

            # Check for end token first
            if match_type == MatchType.end:
                if not skip or token_stack[-1] == token:
                    removed = self.remove_token(token_stack, token)
                    if removed:
                        skip = False
                    else:
                        skip = True

            if not skip:
                # Append text preceding matched token
                start_pos = match.start(group)
                if start_pos > last_pos:
                    yield Segment(self.postprocess(text[last_pos:start_pos]), **params)

                # Actions specific for start token or single token
                if match_type == MatchType.start:
                    token_stack.append(token)
                elif match_type == MatchType.single:
                    single_params = params.copy()
                    single_params.update(token.params)
                    single_text = token.text if token.text is not None else match.group(group)
                    yield Segment(single_text, token=token, match=match, **single_params)

                # Move last position pointer to the end of matched token
                last_pos = match.end(group)

        # Append anything that's left
        if last_pos < len(text):
            params = self.get_params(token_stack)
            yield Segment(self.postprocess(text[last_pos:]), **params)
1180628
def parse(self, text, element, context='eqn'):
        
        
        # Remove the inline comments from `text` before parsing the grammar
        # http://docs.oasis-open.org/xmile/xmile/v1.0/csprd01/xmile-v1.0-csprd01.html#_Toc398039973
        text = re.sub(r"\{[^}]*\}", "", text)
        
        self.ast = self.grammar.parse(text)
        self.context = context
        self.element = element
        self.new_structure = []
        
        py_expr = self.visit(self.ast)
        
        return ({
            'py_expr': py_expr
        }, self.new_structure)
1065304
def tokenize(text, format=None):
    
    text = Text(text)
    text = text.replace("\t", " ")
    tokens = re.findall(patterns, text)
    tokens = [token[0] for token in tokens]
    if format == "text":
        return " ".join(tokens)
    else:
        return tokens
905105
def parse(self, text):
        
        tokens = self.lex(text)
        parser = Parser(tokens)
        return parser.parse()
701972
def _parse(self, text):
        '''
        
        '''
        # ensure unicode first
        text = str(text).strip()

        if not text.startswith(DELIMITER):
            return {}, text

        try:
            _, fm, content = BOUNDARY.split(text, 2)
        except ValueError:
            # if we can't split, bail
            return {}, text
        # loader_class = self.loader_factory(self)
        metadata = yaml.load(fm, Loader=self.loader_class)
        metadata = metadata if (isinstance(metadata, dict)) else {}
        return metadata, content
747469
def tag(self, sentence, tokenize=True):
        
        #: Do not process empty strings (Issue #3)
        if sentence.strip() == "":
            return []
        #: Do not process strings consisting of a single punctuation mark (Issue #4)
        elif sentence.strip() in PUNCTUATION:
            if self.include_punc:
                _sym = sentence.strip()
                if _sym in tuple('.?!'):
                    _tag = "."
                else:
                    _tag = _sym
                return [(_sym, _tag)]
            else:
                return []
        if tokenize:
            _tokenized = " ".join(self.tokenizer.tokenize(sentence))
            sentence = _tokenized
        # Sentence is tokenized before it is passed on to pattern.de.tag
        # (i.e. it is either submitted tokenized or if )
        _tagged = pattern_tag(sentence, tokenize=False,
                              encoding=self.encoding,
                              tagset=self.tagset)
        if self.include_punc:
            return _tagged
        else:
            _tagged = [
                (word, t) for word, t in _tagged if not PUNCTUATION_REGEX.match(
                    unicode(t))]
            return _tagged
394342
def tokenize(cls, text, mode='c'):
        
        if mode == 'c':
            return [ch for ch in text]
        else:
            return [w for w in text.split()]
287591
def _handle_tag_text(self, text):
        """"""
        next = self._read(1)
        if not self._can_recurse() or text not in self.MARKERS:
            self._emit_text(text)
        elif text == next == "{":
            self._parse_template_or_argument()
        elif text == next == "[":
            self._parse_wikilink()
        elif text == "<":
            self._parse_tag()
        else:
            self._emit_text(text)
700610
private void parse()
    {
        this.elements = new ArrayList<>();

        try {
            for (Tokenizer tokenizer = new Tokenizer(this.rawVersion); tokenizer.next();) {
                Element element = new Element(tokenizer);
                this.elements.add(element);
                if (element.getVersionType() != Type.STABLE) {
                    this.type = element.getVersionType();
                }
            }

            trimPadding(this.elements);
        } catch (Exception e) {
            // Make sure to never fail no matter what
            LOGGER.error("Failed to parse version [" + this.rawVersion + "]", e);
            this.elements.add(new Element(this.rawVersion));
        }
    }
1145973
function parse($text) {
		$text=parent::parse($text);
		// Build tree structure
		for ($ptr=0,$w=5,$len=strlen($text),$tree=[],$tmp='';$ptr<$len;)
			if (preg_match('/^(.{0,'.$w.'}?)<(\/?)(?:F3:)?'.
				'('.$this->tags.')\b((?:\s+[\w.:@!-]+'.
				'(?:\h*=\h*(?:"(?:.*?)"|\'(?:.*?)\'))?|'.
				'\h*\{\{.+?\}\})*)\h*(\/?)>/is',
				substr($text,$ptr),$match)) {
				if (strlen($tmp) || $match[1])
					$tree[]=$tmp.$match[1];
				// Element node
				if ($match[2]) {
					// Find matching start tag
					$stack=[];
					for($i=count($tree)-1;$i>=0;$i--) {
						$item=$tree[$i];
						if (is_array($item) &&
							array_key_exists($match[3],$item) &&
							!isset($item[$match[3]][0])) {
							// Start tag found
							$tree[$i][$match[3]]+=array_reverse($stack);
							$tree=array_slice($tree,0,$i+1);
							break;
						}
						else $stack[]=$item;
					}
				}
				else {
					// Start tag
					$node=&$tree[][$match[3]];
					$node=[];
					if ($match[4]) {
						// Process attributes
						preg_match_all(
							'/(?:(\{\{.+?\}\})|([^\s\/"\'=]+))'.
							'\h*(?:=\h*(?:"(.*?)"|\'(.*?)\'))?/s',
							$match[4],$attr,PREG_SET_ORDER);
						foreach ($attr as $kv)
							if (!empty($kv[1]) && !isset($kv[3]) && !isset($kv[4]))
								$node['@attrib'][]=$kv[1];
							else
								$node['@attrib'][$kv[1]?:$kv[2]]=
									(isset($kv[3]) && $kv[3]!==''?
										$kv[3]:
										(isset($kv[4]) && $kv[4]!==''?
											$kv[4]:NULL));
					}
				}
				$tmp='';
				$ptr+=strlen($match[0]);
				$w=5;
			}
			else {
				// Text node
				$tmp.=substr($text,$ptr,$w);
				$ptr+=$w;
				if ($w<50)
					$w++;
			}
		if (strlen($tmp))
			// Append trailing text
			$tree[]=$tmp;
		// Break references
		unset($node);
		return $tree;
	}
178349
def parse(self, text):
        
        gen = self.lexer.lex(text)
        next_val_fn = partial(next, *(gen,))
        token = next_val_fn()
        return self.rule_svg_path(next_val_fn, token)
1366488
def process_text(self, kind, context):
		""""""
		
		result = None
		
		while True:
			chunk = yield None
			
			if chunk is None:
				if result:
					yield result.clone(line=repr(result.line))
				
				return
			
			if not result:
				result = chunk
				continue
			
			result.line += chunk.line