Generate the rules for the punctuation symbols.

@param punctuation the punctuation symbols.
@return the rules.
484319
protected List<Map<String, ?>> generatePunctuation(Set<String> punctuation) {
		final List<Map<String, ?>> list = new ArrayList<>();
		if (!punctuation.isEmpty()) {
			list.add(pattern(it -> {
				it.matches(orRegex(punctuation));
				it.style(PUNCTUATION_STYLE);
				it.comment("Operators and Punctuations"); //$NON-NLS-1$
			}));
		}
		return list;
	}
766981
private String getPatternCode(final List<Token> tokens) {
        final StringBuilder sb = new StringBuilder();
        sb.append(tokens.size());
        for (final Token token : tokens) {
            sb.append(token.getType().ordinal());
        }
        return sb.toString();
    }
677506
def is_cjk_punctuation(char):
    """"""
    lower = int('0x3000', 16)
    higher = int('0x300F', 16)
    return ord(char) >= lower and ord(char) <= higher
746104
public static String getRuleName(AbstractRule rule) {
		return org.eclipse.xtext.xtext.generator.parser.antlr.AntlrGrammarGenUtil.getRuleName(rule);
	}
184311
public List<RuleMatch> check(AnnotatedText annotatedText, boolean tokenizeText, ParagraphHandling paraMode, RuleMatchListener listener, Mode mode) throws IOException {
    List<String> sentences;
    if (tokenizeText) { 
      sentences = sentenceTokenize(annotatedText.getPlainText());
    } else {
      sentences = new ArrayList<>();
      sentences.add(annotatedText.getPlainText());
    }
    List<Rule> allRules = getAllRules();
    if (printStream != null) {
      printIfVerbose(allRules.size() + " rules activated for language " + language);
    }

    unknownWords = new HashSet<>();
    List<AnalyzedSentence> analyzedSentences = analyzeSentences(sentences);
    
    List<RuleMatch> ruleMatches = performCheck(analyzedSentences, sentences, allRules, paraMode, annotatedText, listener, mode);
    ruleMatches = new SameRuleGroupFilter().filter(ruleMatches);
    // no sorting: SameRuleGroupFilter sorts rule matches already
    if (cleanOverlappingMatches) {
      ruleMatches = new CleanOverlappingFilter(language).filter(ruleMatches);
    }
    return ruleMatches;
  }
1055456
public java.util.List<String> getRuleNames() {
        if (ruleNames == null) {
            ruleNames = new com.amazonaws.internal.SdkInternalList<String>();
        }
        return ruleNames;
    }
826964
public static SymbolToken[] newSymbolTokens(SymbolTable symtab,
                                                String... text)
    {
        if (text != null)
        {
            int count = text.length;
            if (count != 0)
            {
                SymbolToken[] result = new SymbolToken[count];
                for (int i = 0; i < count; i++)
                {
                    String s = text[i];
                    result[i] = newSymbolToken(symtab, s);
                }
                return result;
            }
        }
        return SymbolToken.EMPTY_ARRAY;
    }
1276065
def legacy_all_punctuation():  # pragma: no cover b/c tested but Coveralls: ¯\_(ツ)_/¯
    
    punc = symbols.ALL_PUNC
    return RegexBuilder(
        pattern_args=punc,
        pattern_func=lambda x: u"{}".format(x)).regex
1202204
private static String generateSignByRule(TreeMap<String, String> map) {
    String sign = "";
    Iterator<Map.Entry<String, String>> it = map.entrySet().iterator();
    while (it.hasNext()) {
      Map.Entry<String, String> entry = it.next();
      if (it.hasNext()) {
        if (entry.getValue() != null && !"".equals(entry.getValue().toString().trim())) {
          sign += entry.getKey() + "=" + entry.getValue() + "&";
        }
      } else {
        if (entry.getValue() != null && !"".equals(entry.getValue().toString().trim())) {
          sign += entry.getKey() + "=" + entry.getValue();
        } else {
          sign = sign.substring(0, sign.length() - 1);
        }
      }
    }
    return getMD5(sign);
  }
1178009
@SuppressWarnings("checkstyle:all")
	protected void generateIDocumentationFormatter() {
		final TypeReference formatter = getIDocumentationFormatter();
		StringConcatenationClient content = new StringConcatenationClient() {
			@Override
			protected void appendTo(TargetStringConcatenation it) {
				it.append("/** Format a documentation string."); //$NON-NLS-1$
				it.newLine();
				it.append(" */"); //$NON-NLS-1$
				it.newLine();
				it.append("public interface "); //$NON-NLS-1$
				it.append(formatter.getSimpleName());
				it.append(" {"); //$NON-NLS-1$
				it.newLineIfNotEmpty();
				it.newLine();
				it.append("\t/** Replies the characters that must be used as prefix of each line in a multiline comment."); //$NON-NLS-1$
				it.newLine();
				it.append("\t * @return the prefix."); //$NON-NLS-1$
				it.newLine();
				it.append("\t */"); //$NON-NLS-1$
				it.newLine();
				it.append("\t@"); //$NON-NLS-1$
				it.append(Pure.class);
				it.newLine();
				it.append("\tString getMultilineCommentLinePrefix();"); //$NON-NLS-1$
				it.newLineIfNotEmpty();
				it.newLine();
				it.append("\t/** Replies the characters that must be used to start a comment."); //$NON-NLS-1$
				it.newLine();
				it.append("\t * @return the symbols."); //$NON-NLS-1$
				it.newLine();
				it.append("\t */"); //$NON-NLS-1$
				it.newLine();
				it.append("\t@"); //$NON-NLS-1$
				it.append(Pure.class);
				it.newLine();
				it.append("\tString getMultilineCommentStartSymbols();"); //$NON-NLS-1$
				it.newLineIfNotEmpty();
				it.newLine();
				it.append("\t/** Replies the characters that must be used to end a comment."); //$NON-NLS-1$
				it.newLine();
				it.append("\t * @return the symbols."); //$NON-NLS-1$
				it.newLine();
				it.append("\t */"); //$NON-NLS-1$
				it.newLine();
				it.append("\t@"); //$NON-NLS-1$
				it.append(Pure.class);
				it.newLine();
				it.append("\tString getMultilineCommentEndSymbols();"); //$NON-NLS-1$
				it.newLineIfNotEmpty();
				it.newLine();
				it.append("\t/** Replies the string that must be used as prefix of a singleline comment."); //$NON-NLS-1$
				it.newLine();
				it.append("\t * @return the prefix."); //$NON-NLS-1$
				it.newLine();
				it.append("\t */"); //$NON-NLS-1$
				it.newLine();
				it.append("\t@"); //$NON-NLS-1$
				it.append(Pure.class);
				it.newLine();
				it.append("\tString getSinglelineCommentPrefix();"); //$NON-NLS-1$
				it.newLineIfNotEmpty();
				it.newLine();
				it.append("\t/** Replies the formatted string that corresponds to the given documentation."); //$NON-NLS-1$
				it.newLine();
				it.append("\t * @param doc the documentation text. It may be on multiple lines."); //$NON-NLS-1$
				it.newLine();
				it.append("\t * @return the formatted comment."); //$NON-NLS-1$
				it.newLine();
				it.append("\t */"); //$NON-NLS-1$
				it.newLine();
				it.append("\t@"); //$NON-NLS-1$
				it.append(Pure.class);
				it.newLine();
				it.append("\tString formatMultilineComment(String doc);"); //$NON-NLS-1$
				it.newLineIfNotEmpty();
				it.newLine();
				it.append("\t/** Replies the formatted string that corresponds to the given documentation."); //$NON-NLS-1$
				it.newLine();
				it.append("\t * @param doc the documentation text. It may be on multiple lines."); //$NON-NLS-1$
				it.newLine();
				it.append("\t * @param indentation the string to put for indenting the comment."); //$NON-NLS-1$
				it.newLine();
				it.append("\t * @return the formatted comment."); //$NON-NLS-1$
				it.newLine();
				it.append("\t */"); //$NON-NLS-1$
				it.newLine();
				it.append("\t@"); //$NON-NLS-1$
				it.append(Pure.class);
				it.newLine();
				it.append("\tString formatMultilineComment(String doc, String indentation);"); //$NON-NLS-1$
				it.newLineIfNotEmpty();
				it.newLine();
				it.append("\t/** Replies the formatted string that corresponds to the given documentation."); //$NON-NLS-1$
				it.newLine();
				it.append("\t * @param doc the documentation text. It may be on multiple lines."); //$NON-NLS-1$
				it.newLine();
				it.append("\t * @param appendable the receiver of the formatted string."); //$NON-NLS-1$
				it.newLine();
				it.append("\t */"); //$NON-NLS-1$
				it.newLine();
				it.append("\t@"); //$NON-NLS-1$
				it.append(Pure.class);
				it.newLine();
				it.append("\tvoid formatMultilineComment(String doc, "); //$NON-NLS-1$
				it.append(IAppendable.class);
				it.append(" appendable);"); //$NON-NLS-1$
				it.newLineIfNotEmpty();
				it.newLine();
				it.append("\t/** Replies the formatted string that corresponds to the given documentation."); //$NON-NLS-1$
				it.newLine();
				it.append("\t * @param doc the documentation text. It may be on multiple lines."); //$NON-NLS-1$
				it.newLine();
				it.append("\t * @param indentation the string to put for indenting the comment."); //$NON-NLS-1$
				it.newLine();
				it.append("\t * @param appendable the receiver of the formatted string."); //$NON-NLS-1$
				it.newLine();
				it.append("\t */"); //$NON-NLS-1$
				it.newLine();
				it.append("\t@"); //$NON-NLS-1$
				it.append(Pure.class);
				it.newLine();
				it.append("\tvoid formatMultilineComment(String doc, String indentation, "); //$NON-NLS-1$
				it.append(IAppendable.class);
				it.append(" appendable);"); //$NON-NLS-1$
				it.newLineIfNotEmpty();
				it.newLine();
				it.append("\t/** Format the given multiline documentation."); //$NON-NLS-1$
				it.newLine();
				it.append("\t *"); //$NON-NLS-1$
				it.newLine();
				it.append("\t * @param context the formatting context."); //$NON-NLS-1$
				it.newLine();
				it.append("\t * @param comment the comment to format out."); //$NON-NLS-1$
				it.newLine();
				it.append("\t */"); //$NON-NLS-1$
				it.newLine();
				it.append("\t@"); //$NON-NLS-1$
				it.append(Pure.class);
				it.newLine();
				it.append("\tpublic void formatMultilineComment("); //$NON-NLS-1$
				it.append(ITextReplacerContext.class);
				it.append(" context, "); //$NON-NLS-1$
				it.append(IComment.class);
				it.append(" comment);"); //$NON-NLS-1$
				it.newLineIfNotEmpty();
				it.newLine();
				it.append("\t/** Replies the formatted string that corresponds to the given documentation."); //$NON-NLS-1$
				it.newLine();
				it.append("\t * @param doc the documentation text. It may be on multiple lines."); //$NON-NLS-1$
				it.newLine();
				it.append("\t * @return the formatted comment."); //$NON-NLS-1$
				it.newLine();
				it.append("\t */"); //$NON-NLS-1$
				it.newLine();
				it.append("\t@"); //$NON-NLS-1$
				it.append(Pure.class);
				it.newLine();
				it.append("\tString formatSinglelineComment(String doc);"); //$NON-NLS-1$
				it.newLineIfNotEmpty();
				it.newLine();
				it.append("\t/** Replies the formatted string that corresponds to the given documentation."); //$NON-NLS-1$
				it.newLine();
				it.append("\t * @param doc the documentation text. It may be on multiple lines."); //$NON-NLS-1$
				it.newLine();
				it.append("\t * @param indentation the string to put for indenting the comment."); //$NON-NLS-1$
				it.newLine();
				it.append("\t * @return the formatted comment."); //$NON-NLS-1$
				it.newLine();
				it.append("\t */"); //$NON-NLS-1$
				it.newLine();
				it.append("\t@"); //$NON-NLS-1$
				it.append(Pure.class);
				it.newLine();
				it.append("\tString formatSinglelineComment(String doc, String indentation);"); //$NON-NLS-1$
				it.newLineIfNotEmpty();
				it.newLine();
				it.append("\t/** Replies the formatted string that corresponds to the given documentation."); //$NON-NLS-1$
				it.newLine();
				it.append("\t * @param doc the documentation text. It may be on multiple lines."); //$NON-NLS-1$
				it.newLine();
				it.append("\t * @param appendable the receiver of the formatted string."); //$NON-NLS-1$
				it.newLine();
				it.append("\t */"); //$NON-NLS-1$
				it.newLine();
				it.append("\t@"); //$NON-NLS-1$
				it.append(Pure.class);
				it.newLine();
				it.append("\tvoid formatSinglelineComment(String doc, "); //$NON-NLS-1$
				it.append(IAppendable.class);
				it.append(" appendable);"); //$NON-NLS-1$
				it.newLineIfNotEmpty();
				it.newLine();
				it.append("\t/** Replies the formatted string that corresponds to the given documentation."); //$NON-NLS-1$
				it.newLine();
				it.append("\t * @param doc the documentation text. It may be on multiple lines."); //$NON-NLS-1$
				it.newLine();
				it.append("\t * @param indentation the string to put for indenting the comment."); //$NON-NLS-1$
				it.newLine();
				it.append("\t * @param appendable the receiver of the formatted string."); //$NON-NLS-1$
				it.newLine();
				it.append("\t */"); //$NON-NLS-1$
				it.newLine();
				it.append("\t@"); //$NON-NLS-1$
				it.append(Pure.class);
				it.newLine();
				it.append("\tvoid formatSinglelineComment(String doc, String indentation, "); //$NON-NLS-1$
				it.append(IAppendable.class);
				it.append(" appendable);"); //$NON-NLS-1$
				it.newLineIfNotEmpty();
				it.newLine();
				it.append("\t/** Format the given singleline documentation."); //$NON-NLS-1$
				it.newLine();
				it.append("\t *"); //$NON-NLS-1$
				it.newLine();
				it.append("\t * @param context the formatting context."); //$NON-NLS-1$
				it.newLine();
				it.append("\t * @param comment the comment to format out."); //$NON-NLS-1$
				it.newLine();
				it.append("\t */"); //$NON-NLS-1$
				it.newLine();
				it.append("\t@"); //$NON-NLS-1$
				it.append(Pure.class);
				it.newLine();
				it.append("\tvoid formatSinglelineComment("); //$NON-NLS-1$
				it.append(ITextReplacerContext.class);
				it.append(" context, "); //$NON-NLS-1$
				it.append(IComment.class);
				it.append(" comment);"); //$NON-NLS-1$
				it.newLineIfNotEmpty();
				it.newLine();
				it.append("}"); //$NON-NLS-1$
				it.newLineIfNotEmpty();
				it.newLine();
			}

		};
		JavaFileAccess createJavaFile = getFileAccessFactory().createJavaFile(formatter, content);
		createJavaFile.writeTo(getSrcGen());
	}
947762
String toRules(boolean escapeUnprintable) {
        int i;
        int count = ruleVector.size();
        StringBuilder ruleSource = new StringBuilder();
        for (i=0; i<count; ++i) {
            if (i != 0) {
                ruleSource.append('\n');
            }
            TransliterationRule r = ruleVector.get(i);
            ruleSource.append(r.toRule(escapeUnprintable));
        }
        return ruleSource.toString();
    }
76059
public static LineSymbolizer lineSymbolizerFromRule( Rule rule ) {
        List<Symbolizer> symbolizers = rule.symbolizers();
        LineSymbolizer lineSymbolizer = null;
        for( Symbolizer symbolizer : symbolizers ) {
            if (symbolizer instanceof LineSymbolizer) {
                lineSymbolizer = (LineSymbolizer) symbolizer;
                break;
            }
        }
        if (lineSymbolizer == null) {
            throw new IllegalArgumentException();
        }
        return lineSymbolizer;
    }
746608
@SuppressSubnodes
    @MemoMismatches
    Rule Identifier() {
        return Sequence(TestNot(Keyword()), Letter(), ZeroOrMore(LetterOrDigit()), Spacing());
    }
544105
protected void configureCurseWords (String curseWords)
    {
        StringTokenizer st = new StringTokenizer(curseWords);
        int numWords = st.countTokens();

        _matchers = new Matcher[numWords];
        _replacements = new String[numWords];
        _vernacular = new String[numWords];
        _comicLength = new int[numWords];

        for (int ii=0; ii < numWords; ii++) {
            String mapping = st.nextToken();
            StringTokenizer st2 = new StringTokenizer(mapping, "=");
            if (st2.countTokens() != 2) {
                log.warning("Something looks wrong in the x.cursewords properties (" +
                    mapping + "), skipping.");
                continue;
            }
            String curse = st2.nextToken();

            String s = "";
            String p = "";
            if (curse.startsWith("*")) {
                curse = curse.substring(1);
                p += "([\\p{L}\\p{Digit}]*)";
                s += "$1";
            } else {
                p += "()";
            }
            s += " ";
            p += " ";
            if (curse.endsWith("*")) {
                curse = curse.substring(0, curse.length() - 1);
                p += "([\\p{L}\\p{Digit}]*)";
                s += "$3";
            }

            String pattern = "\\b" + p.replace(" ", "(" + curse + ")") + "\\b";
            Pattern pat = Pattern.compile(pattern, Pattern.CASE_INSENSITIVE | Pattern.UNICODE_CASE);
            _matchers[ii] = pat.matcher("");
            _replacements[ii] = s;
            _vernacular[ii] = st2.nextToken().replace('_', ' ');
            _comicLength[ii] = curse.codePointCount(0, curse.length());
        }
    }
190916
public static Grammar forProductions(String ... prods) {
        StringArrayList symbolTable;
        Grammar grammar;
        String s;
        String[] stringProd;
        int[] prod;
        int idx;

        grammar = new Grammar();
        symbolTable = grammar.symbolTable;
        idx = 0; // definite assignment
        for (int p = 0; p < prods.length; p++) {
            stringProd = Strings.toArray(Separator.SPACE.split(prods[p]));
            if (stringProd.length == 0) {
                throw new IllegalArgumentException();
            }
            prod = new int[stringProd.length];
            // count backwards to have the left-hand-side in idx when the loop has finished
            for (int ofs = prod.length - 1; ofs >= 0; ofs--) {
                s = stringProd[ofs];
                idx = symbolTable.indexOf(s);
                if (idx == -1) {
                    idx = symbolTable.size();
                    symbolTable.add(s);
                }
                prod[ofs] = idx;
            }
            if (p == 0) {
                grammar.start = idx;
            }
            grammar.addProduction(prod);
        }
        return grammar;
    }
85382
public static String printComments(String indent, List<JavaComment> comments) {
        final StringBuilder builder = new StringBuilder();
        for (JavaComment comment : comments) {
            builder.append(print(indent, comment));
            builder.append("\n");
        }
        return builder.toString();
    }
1104502
def _expressions_from_rules(self, rule_syntax, custom_rules):
        
        # Hard-code enough of the rules to parse the grammar that describes the
        # grammar description language, to bootstrap:
        comment = Regex(r'#[^\r\n]*', name='comment')
        meaninglessness = OneOf(Regex(r'\s+'), comment, name='meaninglessness')
        _ = ZeroOrMore(meaninglessness, name='_')
        equals = Sequence(Literal('='), _, name='equals')
        label = Sequence(Regex(r'[a-zA-Z_][a-zA-Z_0-9]*'), _, name='label')
        reference = Sequence(label, Not(equals), name='reference')
        quantifier = Sequence(Regex(r'[*+?]'), _, name='quantifier')
        # This pattern supports empty literals. TODO: A problem?
        spaceless_literal = Regex(r'u?r?"[^"\\]*(?:\\.[^"\\]*)*"',
                                  ignore_case=True,
                                  dot_all=True,
                                  name='spaceless_literal')
        literal = Sequence(spaceless_literal, _, name='literal')
        regex = Sequence(Literal('~'),
                         literal,
                         Regex('[ilmsuxa]*', ignore_case=True),
                         _,
                         name='regex')
        atom = OneOf(reference, literal, regex, name='atom')
        quantified = Sequence(atom, quantifier, name='quantified')

        term = OneOf(quantified, atom, name='term')
        not_term = Sequence(Literal('!'), term, _, name='not_term')
        term.members = (not_term,) + term.members

        sequence = Sequence(term, OneOrMore(term), name='sequence')
        or_term = Sequence(Literal('/'), _, term, name='or_term')
        ored = Sequence(term, OneOrMore(or_term), name='ored')
        expression = OneOf(ored, sequence, term, name='expression')
        rule = Sequence(label, equals, expression, name='rule')
        rules = Sequence(_, OneOrMore(rule), name='rules')

        # Use those hard-coded rules to parse the (more extensive) rule syntax.
        # (For example, unless I start using parentheses in the rule language
        # definition itself, I should never have to hard-code expressions for
        # those above.)

        rule_tree = rules.parse(rule_syntax)

        # Turn the parse tree into a map of expressions:
        return RuleVisitor().visit(rule_tree)