Return a Condition instance with PEG grammar from text

        :param text: PEG parsable string
        :return:
978181
def condition_from_text(text) -> Condition:
        
        try:
            condition = pypeg2.parse(text, output.Condition)
        except SyntaxError:
            # Invalid conditions are possible, see https://github.com/duniter/duniter/issues/1156
            # In such a case, they are store as empty PEG grammar object and considered unlockable
            condition = Condition(text)
        return condition
744896
def tcc(text: str) -> str:
    
    if not text or not isinstance(text, str):
        return ""

    p = 0
    while p < len(text):
        m = PAT_TCC.match(text[p:])
        if m:
            n = m.span()[1]
        else:
            n = 1
        yield text[p : p + n]
        p += n
554242
def lemmatize(text_string):
    '''
        
    '''
    if text_string is None or text_string == "":
        return ""
    elif isinstance(text_string, str):
        return LEMMATIZER.lemmatize(text_string)
    else:
        raise InputError("string not passed as primary argument")
181610
def eval(self, text):
        
        assert is_text_string(text)
        try:
            return eval(text, self.locals), True
        except:
            return None, False
762574
def load_grammar(self, grammar_text, grammar_name='<?>'):
        ""

        try:
            tree = self.canonize_tree.transform( self.parser.parse(grammar_text+'\n') )
        except UnexpectedCharacters as e:
            context = e.get_context(grammar_text)
            raise GrammarError("Unexpected input at line %d column %d in %s: \n\n%s" %
                               (e.line, e.column, grammar_name, context))
        except UnexpectedToken as e:
            context = e.get_context(grammar_text)
            error = e.match_examples(self.parser.parse, {
                'Unclosed parenthesis': ['a: (\n'],
                'Umatched closing parenthesis': ['a: )\n', 'a: [)\n', 'a: (]\n'],
                'Expecting rule or terminal definition (missing colon)': ['a\n', 'a->\n', 'A->\n', 'a A\n'],
                'Alias expects lowercase name': ['a: -> "a"\n'],
                'Unexpected colon': ['a::\n', 'a: b:\n', 'a: B:\n', 'a: "a":\n'],
                'Misplaced operator': ['a: b??', 'a: b(?)', 'a:+\n', 'a:?\n', 'a:*\n', 'a:|*\n'],
                'Expecting option ("|") or a new rule or terminal definition': ['a:a\n()\n'],
                '%import expects a name': ['%import "a"\n'],
                '%ignore expects a value': ['%ignore %import\n'],
            })
            if error:
                raise GrammarError("%s at line %s column %s\n\n%s" % (error, e.line, e.column, context))
            elif 'STRING' in e.expected:
                raise GrammarError("Expecting a value at line %s column %s\n\n%s" % (e.line, e.column, context))
            raise

        tree = PrepareGrammar().transform(tree)

        # Extract grammar items
        defs = classify(tree.children, lambda c: c.data, lambda c: c.children)
        term_defs = defs.pop('term', [])
        rule_defs = defs.pop('rule', [])
        statements = defs.pop('statement', [])
        assert not defs

        term_defs = [td if len(td)==3 else (td[0], 1, td[1]) for td in term_defs]
        term_defs = [(name.value, (t, int(p))) for name, p, t in term_defs]
        rule_defs = [options_from_rule(*x) for x in rule_defs]

        # Execute statements
        ignore, imports = [], {}
        for (stmt,) in statements:
            if stmt.data == 'ignore':
                t ,= stmt.children
                ignore.append(t)
            elif stmt.data == 'import':
                if len(stmt.children) > 1:
                    path_node, arg1 = stmt.children
                else:
                    path_node, = stmt.children
                    arg1 = None

                if isinstance(arg1, Tree):  # Multi import
                    dotted_path = tuple(path_node.children)
                    names = arg1.children
                    aliases = dict(zip(names, names))  # Can't have aliased multi import, so all aliases will be the same as names
                else:  # Single import
                    dotted_path = tuple(path_node.children[:-1])
                    name = path_node.children[-1]  # Get name from dotted path
                    aliases = {name: arg1 or name}  # Aliases if exist

                if path_node.data == 'import_lib':  # Import from library
                    base_paths = []
                else:  # Relative import
                    if grammar_name == '<string>':  # Import relative to script file path if grammar is coded in script
                        try:
                            base_file = os.path.abspath(sys.modules['__main__'].__file__)
                        except AttributeError:
                            base_file = None
                    else:
                        base_file = grammar_name  # Import relative to grammar file path if external grammar file
                    if base_file:
                        base_paths = [os.path.split(base_file)[0]]
                    else:
                        base_paths = [os.path.abspath(os.path.curdir)]

                try:
                    import_base_paths, import_aliases = imports[dotted_path]
                    assert base_paths == import_base_paths, 'Inconsistent base_paths for %s.' % '.'.join(dotted_path)
                    import_aliases.update(aliases)
                except KeyError:
                    imports[dotted_path] = base_paths, aliases

            elif stmt.data == 'declare':
                for t in stmt.children:
                    term_defs.append([t.value, (None, None)])
            else:
                assert False, stmt

        # import grammars
        for dotted_path, (base_paths, aliases) in imports.items():
            grammar_path = os.path.join(*dotted_path) + EXT
            g = import_grammar(grammar_path, base_paths=base_paths)
            new_td, new_rd = import_from_grammar_into_namespace(g, '__'.join(dotted_path), aliases)

            term_defs += new_td
            rule_defs += new_rd

        # Verify correctness 1
        for name, _ in term_defs:
            if name.startswith('__'):
                raise GrammarError('Names starting with double-underscore are reserved (Error at %s)' % name)

        # Handle ignore tokens
        # XXX A slightly hacky solution. Recognition of %ignore TERMINAL as separate comes from the lexer's
        #     inability to handle duplicate terminals (two names, one value)
        ignore_names = []
        for t in ignore:
            if t.data=='expansions' and len(t.children) == 1:
                t2 ,= t.children
                if t2.data=='expansion' and len(t2.children) == 1:
                    item ,= t2.children
                    if item.data == 'value':
                        item ,= item.children
                        if isinstance(item, Token) and item.type == 'TERMINAL':
                            ignore_names.append(item.value)
                            continue

            name = '__IGNORE_%d'% len(ignore_names)
            ignore_names.append(name)
            term_defs.append((name, (t, 1)))

        # Verify correctness 2
        terminal_names = set()
        for name, _ in term_defs:
            if name in terminal_names:
                raise GrammarError("Terminal '%s' defined more than once" % name)
            terminal_names.add(name)

        if set(ignore_names) > terminal_names:
            raise GrammarError("Terminals %s were marked to ignore but were not defined!" % (set(ignore_names) - terminal_names))

        resolve_term_references(term_defs)

        rules = rule_defs

        rule_names = set()
        for name, _x, _o in rules:
            if name.startswith('__'):
                raise GrammarError('Names starting with double-underscore are reserved (Error at %s)' % name)
            if name in rule_names:
                raise GrammarError("Rule '%s' defined more than once" % name)
            rule_names.add(name)

        for name, expansions, _o in rules:
            used_symbols = {t for x in expansions.find_data('expansion')
                              for t in x.scan_values(lambda t: t.type in ('RULE', 'TERMINAL'))}
            for sym in used_symbols:
                if sym.type == 'TERMINAL':
                    if sym not in terminal_names:
                        raise GrammarError("Token '%s' used but not defined (in rule %s)" % (sym, name))
                else:
                    if sym not in rule_names:
                        raise GrammarError("Rule '%s' used but not defined (in rule %s)" % (sym, name))

        return Grammar(rules, term_defs, ignore_names)
1138844
function (text) {
        text = text.replace(new RegExp('</([^d])','g'), '</disabled $1');
        text = text.replace(new RegExp('</(\S{2,})','g'), '</disabled $1');
        text = text.replace(new RegExp('<([^d/]\W*?)','g'), '<disabled $1');
        text = text.replace(new RegExp('<([^/ ]{2,}\W*?)','g'), '<disabled $1');
        return text;
    }
174305
def __normalize(self,text):
        
        return self._normalizer_regex.sub(lambda mo:
                self._normalizer[mo.string[mo.start():mo.end()]],
                    text.lower())
255956
def run(self, text):
        
        for regex in self.regexes:
            text = regex.sub(self.repl, text)
        return text
399299
def tokenize(self, text: str, model: object = None):
        
        sentences = re.split(self.pattern, text)
        return sentences
896482
def translate(self, text):
        

        # Reset substitution counter
        self.count = 0

        # Process text
        return self._make_regex().sub(self, text)
1260464
def parse_input(self, text):
        

        parts = util.split(text)
        command = parts[0] if text and parts else None
        command = command.lower() if command else None
        args = parts[1:] if len(parts) > 1 else []

        return (command, args)
751742
def text_to_data(self, text, elt, ps):
        '''
        '''
        if text is None:
            return None
        
        m = self.lex_pattern.match(text)
        if not m:
            raise EvaluateException('Bad Gregorian: %s' %text, ps.Backtrace(elt))
        try:
            retval = _dict_to_tuple(m.groupdict())
        except ValueError, e:
            #raise EvaluateException(str(e))
            raise
        
        if self.pyclass is not None:
            return self.pyclass(retval)
        return retval
915996
def parse(self, conditions):
        
        conditions_text = conditions
        try:
            conditions = shlex.split(fmt.to_utf8(conditions))
        except AttributeError:
            # Not a string, assume parsed tree
            conditions_text = self._tree2str(conditions)

        # Empty list?
        if not conditions:
            raise FilterError("No conditions given at all!")

        # NOT *must* appear at the start of a group
        negate = conditions[:1] == ["NOT"]
        if negate:
            conditions = conditions[1:]
            if not conditions:
                raise FilterError("NOT must be followed by some conditions!")

        # Handle grouping
        if '[' in conditions:
            tree = [[]]
            for term in conditions:
                if term == '[':
                    tree.append([]) # new grouping
                elif term == ']':
                    subtree = tree.pop()
                    if not tree:
                        raise FilterError("Unbalanced brackets, too many closing ']' in condition %r" % (conditions_text,))
                    tree[-1].append(subtree) # append finished group to containing level
                else:
                    tree[-1].append(term) # append to current level

            if len(tree) > 1:
                raise FilterError("Unbalanced brackets, too many open '[' in condition %r" % (conditions_text,))
            conditions = tree[0]

        # Prepare root matcher
        conditions = list(conditions)
        matcher = CompoundFilterAll()
        if "OR" in conditions:
            root = CompoundFilterAny()
            root.append(matcher)
        else:
            root = matcher

        # Go through conditions and parse them
        for condition in conditions:
            if condition == "OR":
                # Leading OR, or OR OR in sequence?
                if not matcher:
                    raise FilterError("Left-hand side of OR missing in %r!" % (conditions_text,))

                # Start next run of AND conditions
                matcher = CompoundFilterAll()
                root.append(matcher)
            elif isinstance(condition, list):
                matcher.append(self.parse(condition))
            else:
                matcher.append(self._create_filter(condition))

        # Trailing OR?
        if not matcher:
            raise FilterError("Right-hand side of OR missing in %r!" % (conditions_text,))

        return NegateFilter(root) if negate else root
433684
def parse(cls, parser, text, pos):
        """"""
        try:
            parser._parsing_parenthesized_simple_values_expression = True
            remaining_text, recognized_tokens = parser.parse(text, cls.grammar)
            return remaining_text, recognized_tokens
        except SyntaxError as e:
            return text, e
        finally:
            parser._parsing_parenthesized_simple_values_expression = False
34604
def generate_corpus(self, text):
        
        if isinstance(text, str):
            sentences = self.sentence_split(text)
        else:
            sentences = []
            for line in text:
                sentences += self.sentence_split(line)
        passing = filter(self.test_sentence_input, sentences)
        runs = map(self.word_split, passing)
        return runs
322688
def tokenize(self, text: str, customize=True, disable=[]) -> List[Token]:
        

        """Tokenize text"""
        if not self.keep_multi_space:
            text = re.sub(' +', ' ', text)
        # disable spacy parsing, tagging etc as it takes a long time if the text is short
        tokens = self.nlp(text, disable=disable)
        if customize:
            tokens = [self.custom_token(a_token) for a_token in tokens]

        return tokens
291500
def parse(text):
    
    ret = Docstring()
    if not text:
        return ret

    text = inspect.cleandoc(text)
    match = re.search('^:', text, flags=re.M)
    if match:
        desc_chunk = text[:match.start()]
        meta_chunk = text[match.start():]
    else:
        desc_chunk = text
        meta_chunk = ''

    parts = desc_chunk.split('\n', 1)
    ret.short_description = parts[0] or None
    if len(parts) > 1:
        long_desc_chunk = parts[1] or ''
        ret.blank_after_short_description = long_desc_chunk.startswith('\n')
        ret.blank_after_long_description = long_desc_chunk.endswith('\n\n')
        ret.long_description = long_desc_chunk.strip() or None

    for match in re.finditer(
            r'(^:.*?)(?=^:|\Z)', meta_chunk, flags=re.S | re.M
    ):
        chunk = match.group(0)
        if not chunk:
            continue
        try:
            args_chunk, desc_chunk = chunk.lstrip(':').split(':', 1)
        except ValueError:
            raise ParseError(
                'Error parsing meta information near "{}".'.format(chunk)
            )
        args = args_chunk.split()
        desc = desc_chunk.strip()
        if '\n' in desc:
            first_line, rest = desc.split('\n', 1)
            desc = first_line + '\n' + inspect.cleandoc(rest)
        ret.meta.append(DocstringMeta(args, description=desc, type=None))

    return ret