Main access to the ShiftReduceParser train functionalities.

@throws IOException
input output exception if problems with corpora
183273
private void parserTrain() throws IOException {

    // load training parameters file
    final String paramFile = this.parsedArguments.getString("params");
    final String taggerParamsFile = this.parsedArguments
        .getString("taggerParams");
    final String chunkerParamsFile = this.parsedArguments
        .getString("chunkerParams");
    final TrainingParameters params = IOUtils.loadTrainingParameters(paramFile);
    final TrainingParameters chunkerParams = IOUtils
        .loadTrainingParameters(chunkerParamsFile);
    ParserModel trainedModel;
    String outModel;
    if (params.getSettings().get("OutputModel") == null
        || params.getSettings().get("OutputModel").length() == 0) {
      outModel = Files.getNameWithoutExtension(paramFile) + ".bin";
      params.put("OutputModel", outModel);
    } else {
      outModel = Flags.getModel(params);
    }
    if (taggerParamsFile.endsWith(".bin")) {
      final InputStream posModel = new FileInputStream(taggerParamsFile);
      final ShiftReduceParserTrainer parserTrainer = new ShiftReduceParserTrainer(
          params, chunkerParams);
      trainedModel = parserTrainer.train(params, posModel, chunkerParams);
    } else {
      final TrainingParameters taggerParams = IOUtils
          .loadTrainingParameters(taggerParamsFile);
      final ShiftReduceParserTrainer parserTrainer = new ShiftReduceParserTrainer(
          params, taggerParams, chunkerParams);
      trainedModel = parserTrainer.train(params, taggerParams, chunkerParams);
    }
    CmdLineUtil.writeModel("ixa-pipe-ml", new File(outModel), trainedModel);
  }
1073164
public static void main(String[] args) throws IOException, ClassNotFoundException {
    CoNLLDocumentReaderAndWriter f = new CoNLLDocumentReaderAndWriter();
    f.init(new SeqClassifierFlags());
    int numDocs = 0;
    int numTokens = 0;
    int numEntities = 0;
    String lastAnsBase = "";
    for (Iterator<List<CoreLabel>> it = f.getIterator(new FileReader(args[0])); it.hasNext(); ) {
      List<CoreLabel> doc = it.next();
      numDocs++;
      for (CoreLabel fl : doc) {
        // System.out.println("FL " + (++i) + " was " + fl);
        if (fl.word().equals(BOUNDARY)) {
          continue;
        }
        String ans = fl.get(AnswerAnnotation.class);
        String ansBase;
        String ansPrefix;
        String[] bits = ans.split("-");
        if (bits.length == 1) {
          ansBase = bits[0];
          ansPrefix = "";
        } else {
          ansBase = bits[1];
          ansPrefix = bits[0];
        }
        numTokens++;
        if (ansBase.equals("O")) {
        } else if (ansBase.equals(lastAnsBase)) {
          if (ansPrefix.equals("B")) {
            numEntities++;
          }
        } else {
          numEntities++;
        }
      }
    }
    System.out.println("File " + args[0] + " has " + numDocs + " documents, " +
            numTokens + " (non-blank line) tokens and " +
            numEntities + " entities.");
  }
282675
public static void main(String[] args) throws IOException {
    if (args.length == 0) {
      System.err.println("Usage: java opennlp.tools.english.TreebankChunker model < tokenized_sentences");
      System.exit(1);
    }
    TreebankChunker chunker = new TreebankChunker(args[0]);
    java.io.BufferedReader inReader = new java.io.BufferedReader(new java.io.InputStreamReader(System.in));
    for (String line = inReader.readLine(); line != null; line = inReader.readLine()) {
      if (line.equals("")) {
        System.out.println();
      }
      else {
        String[] tts = line.split(" ");
        String[] tokens = new String[tts.length];
        String[] tags = new String[tts.length];
        for (int ti=0,tn=tts.length;ti<tn;ti++) {
          String[] tt = tts[ti].split("/");
          tokens[ti]=tt[0];
          tags[ti]=tt[1]; 
        }
        String[] chunks = chunker.chunk(tokens,tags);
        //System.err.println(java.util.Arrays.asList(chunks));
        for (int ci=0,cn=chunks.length;ci<cn;ci++) {
          if (ci > 0 && !chunks[ci].startsWith("I-") && !chunks[ci-1].equals("O")) {
            System.out.print(" ]");
          }            
          if (chunks[ci].startsWith("B-")) {
            System.out.print(" ["+chunks[ci].substring(2));
          }
          /*
           else {
           System.out.print(" ");
           }
           */
          System.out.print(" "+tokens[ci]+"/"+tags[ci]);
        }
        if (!chunks[chunks.length-1].equals("O")) {
          System.out.print(" ]");
        }
        System.out.println();
      }
    }
  }
1135584
public final static void main(String[] args) throws IOException {

      Writer writer = new OutputStreamWriter(System.out, "utf-8");

      XMLOutputter outputter = new XMLOutputter(writer, "iso-8859-1");
      writeXML(outputter);

      System.out.println();
      System.out.println();

      outputter.reset(writer, "utf-8");
      writeXML(outputter);
   }
871236
private JsonToken startParsing() throws IOException {
    JsonToken currentToken = getCurrentToken();
    // token is null at start, so get next token
    if (currentToken == null) {
      currentToken = nextToken();
    }
    Preconditions.checkArgument(currentToken != null, "no JSON input found");
    return currentToken;
  }
553569
public void readDump() throws IOException {
		try {
			SAXParserFactory factory = SAXParserFactory.newInstance();
			factory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, false);
			SAXParser parser = factory.newSAXParser();

			parser.parse(input, this);
		} catch (ParserConfigurationException | SAXException e) {
			throw new IOException(e);
		}
		writer.close();
	}
1097030
public static void main(String[] args) throws IOException {

    String encoding = args[1];
    Reader in = new BufferedReader(new InputStreamReader(new FileInputStream(args[0]), encoding));

    Tokenizer<String> st = new CHTBTokenizer(in);

    while (st.hasNext()) {
      String s = st.next();
      EncodingPrintWriter.out.println(s, encoding);
      // EncodingPrintWriter.out.println("|" + s + "| (" + s.length() + ")",
      //				encoding);
    }
  }
274387
private ExampleSet readDataSet(File in) throws IOException {
        logger.info("read the example set");

        //
        ExampleSet inputSet1 = new SentenceSetCopy();
        inputSet1.read(new BufferedReader(new FileReader(in)));

        return inputSet1;
    }
1245536
public static void main(String[] args) {
    if (args.length < 2) {
      System.err.println("Input and output files not provided");
      System.err
          .println("You need to provide both the input serialized java fst model");
      System.err.println("and the output binary openfst model.");
      System.exit(1);
    }

    MutableFst fst = FstInputOutput.readFstFromBinaryResource(args[0]);

    // Serialize the java fst model to disk
    System.out.println("Saving as openfst text model...");
    Convert.export(fst, args[1]);
  }
62615
public static void main(String[] args) throws java.io.IOException {
    if (args.length == 0) {
      System.err.println("Usage: Parse -fun head_rules < train_parses");
      System.err.println("Reads training parses (one-sentence-per-line) and displays parse structure.");
      System.exit(1);
    }
    int ai=0;
    if (args[0].equals("-fun")) {
      Parse.useFunctionTags(true);
      ai++;
    }
    HeadRules rules = new ch.epfl.bbp.shaded.opennlp.tools.lang.english.HeadRules(args[ai]);
    java.io.BufferedReader in = new java.io.BufferedReader(new java.io.InputStreamReader(System.in));
    for (String line = in.readLine(); line != null; line = in.readLine()) {
      Parse p = Parse.parseParse(line);
      p.updateHeads(rules);
      p.show();
      //p.showCodeTree();
    }
  }
918500
@Override
	public void open(FileInputSplit split) throws IOException {
		super.open(split);
		
		// instantiate the parsers
		@SuppressWarnings("unchecked")
		FieldParser<Object>[] parsers = new FieldParser[fieldTypes.length];
		
		for (int i = 0; i < fieldTypes.length; i++) {
			if (fieldTypes[i] != null) {
				Class<? extends FieldParser<?>> parserType = FieldParser.getParserForType(fieldTypes[i]);
				if (parserType == null) {
					throw new RuntimeException("No parser available for type '" + fieldTypes[i].getName() + "'.");
				}
				

				@SuppressWarnings("unchecked")
				FieldParser<Object> p = (FieldParser<Object>) InstantiationUtil.instantiate(parserType, FieldParser.class);
				parsers[i] = p;
			}
		}
		this.fieldParsers = parsers;
		
		// skip the first line, if we are at the beginning of a file and have the option set
		if (this.skipFirstLineAsHeader && this.splitStart == 0) {
			readLine(); // read and ignore
		}
	}
307247
def parse(input_: Union[str, FileStream], default_base: Optional[str]=None) -> Optional[Schema]:
    

    # Step 1: Tokenize the input stream
    error_listener = ParseErrorListener()
    if not isinstance(input_, FileStream):
        input_ = InputStream(input_)
    lexer = ShExDocLexer(input_)
    lexer.addErrorListener(error_listener)
    tokens = CommonTokenStream(lexer)
    tokens.fill()
    if error_listener.n_errors:         # Lexer prints errors directly
        return None

    # Step 2: Generate the parse tree
    parser = ShExDocParser(tokens)
    parser.addErrorListener(error_listener)
    parse_tree = parser.shExDoc()
    if error_listener.n_errors:
        print('\n'.join(error_listener.errors), file=sys.stderr)
        return None

    # Step 3: Transform the results the results
    parser = ShexDocParser(default_base=default_base)
    parser.visit(parse_tree)

    return parser.context.schema
260684
private static void processText(NameFinder[] finders, String[] tags, BufferedReader input) throws IOException {
    String[][] finderTags = new String[finders.length][];
    Map[] prevTokenMaps = createPrevTokenMaps(finders); 
    for (String line = input.readLine(); null != line; line = input.readLine()) {
      if (line.equals("")) {
        clearPrevTokenMaps(prevTokenMaps);
        System.out.println();
        continue;
      }
      Span[] spans = tokenizeToSpans(line);
      String[] tokens = spansToStrings(spans,line);
      for (int fi = 0, fl = finders.length; fi < fl; fi++) {
        finderTags[fi] = finders[fi].find(tokens, prevTokenMaps[fi]);
        //System.err.println("EnglighNameFinder.processText: "+tags[fi] + " " + java.util.Arrays.asList(finderTags[fi]));
      }
      updatePrevTokenMaps(prevTokenMaps,tokens,finderTags);
      for (int ti = 0, tl = tokens.length; ti < tl; ti++) {
        for (int fi = 0, fl = finders.length; fi < fl; fi++) {
          //check for end tags
          if (ti != 0) {
            if ((finderTags[fi][ti].equals(NameFinderME.START) || finderTags[fi][ti].equals(NameFinderME.OTHER)) && (finderTags[fi][ti - 1].equals(NameFinderME.START) || finderTags[fi][ti - 1].equals(NameFinderME.CONTINUE))) {
              System.out.print("</" + tags[fi] + ">");
            }
          }
        }
        if (ti > 0 && spans[ti - 1].getEnd() < spans[ti].getStart()) {
          System.out.print(line.substring(spans[ti - 1].getEnd(), spans[ti].getStart()));
        }
        //check for start tags
        for (int fi = 0, fl = finders.length; fi < fl; fi++) {
          if (finderTags[fi][ti].equals(NameFinderME.START)) {
            System.out.print("<" + tags[fi] + ">");
          }
        }
        System.out.print(tokens[ti]);
      }
      //final end tags
      if (tokens.length != 0) {
        for (int fi = 0, fl = finders.length; fi < fl; fi++) {
          if (finderTags[fi][tokens.length - 1].equals(NameFinderME.START) || finderTags[fi][tokens.length - 1].equals(NameFinderME.CONTINUE)) {
            System.out.print("</" + tags[fi] + ">");
          }
        }
      }
      if (tokens.length != 0) {
        if (spans[tokens.length - 1].getEnd() < line.length()) {
          System.out.print(line.substring(spans[tokens.length - 1].getEnd()));
        }
      }
      System.out.println();
    }
  }
319342
static void compileRules(String rules, OutputStream os) throws IOException
    {
        //
        // Read the input rules, generate a parse tree, symbol table,
        // and list of all Unicode Sets referenced by the rules.
        //
        RBBIRuleBuilder builder = new RBBIRuleBuilder(rules);
        builder.fScanner.parse();

        //
        // UnicodeSet processing.
        //    Munge the Unicode Sets to create a set of character categories.
        //    Generate the mapping tables (TRIE) from input 32-bit characters to
        //    the character categories.
        //
        builder.fSetBuilder.build();

        //
        //   Generate the DFA state transition table.
        //
        builder.fForwardTables = new RBBITableBuilder(builder, fForwardTree);
        builder.fReverseTables = new RBBITableBuilder(builder, fReverseTree);
        builder.fSafeFwdTables = new RBBITableBuilder(builder, fSafeFwdTree);
        builder.fSafeRevTables = new RBBITableBuilder(builder, fSafeRevTree);
        builder.fForwardTables.build();
        builder.fReverseTables.build();
        builder.fSafeFwdTables.build();
        builder.fSafeRevTables.build();
        if (builder.fDebugEnv != null
                && builder.fDebugEnv.indexOf("states") >= 0) {
            builder.fForwardTables.printRuleStatusTable();
        }

        //
        //   Package up the compiled data, writing it to an output stream
        //      in the serialization format.  This is the same as the ICU4C runtime format.
        //
        builder.flattenData(os);
    }
271808
private void seqTrain() throws IOException {

    // load training parameters file
    final String paramFile = this.parsedArguments.getString("params");
    final TrainingParameters params = IOUtils.loadTrainingParameters(paramFile);
    String outModel = null;
    if (params.getSettings().get("OutputModel") == null
        || params.getSettings().get("OutputModel").length() == 0) {
      outModel = Files.getNameWithoutExtension(paramFile) + ".bin";
      params.put("OutputModel", outModel);
    } else {
      outModel = Flags.getModel(params);
    }
    final SequenceLabelerTrainer nercTrainer = new SequenceLabelerTrainer(
        params);
    final SequenceLabelerModel trainedModel = nercTrainer.train(params);
    CmdLineUtil.writeModel("ixa-pipe-ml", new File(outModel), trainedModel);
  }
501873
public static void main(String[] args) throws IOException {
    	String infile = "../example-data/data-lda.txt";
    	String stopwordfile = "../models/stopwords/stopwords.txt";
		BufferedReader in = new BufferedReader(new InputStreamReader(
				new FileInputStream(infile ), "utf8"));

		//		BufferedWriter out = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(
//				outfile), enc2));
		StopWords sw = new StopWords(stopwordfile);
		
		LabelAlphabet dict = new LabelAlphabet();
		// words in documents
		ArrayList<TIntArrayList> documentsList= new ArrayList<TIntArrayList>();
		
		
		String line = null;
		while ((line = in.readLine()) != null) {
			line = line.trim();	
			if(line.length()==0)
				continue;
			String[] toks = line.split("\\s+");
			TIntArrayList wordlist = new TIntArrayList(); 
			for(int j=0;j<toks.length;j++){
				String tok = toks[j];
				if(sw.isStopWord(tok))
					continue;
				int idx = dict.lookupIndex(tok);
				wordlist.add(idx);
			}
			documentsList.add(wordlist);
		}
		in.close();
		int[][] documents;
		documents = new int[documentsList.size()][];
		for(int i=0;i<documents.length;i++){
			documents[i] = documentsList.get(i).toArray();
		}
        // vocabulary
        int V = dict.size();
        int M = documents.length;
        // # topics
        int K = 4;
        // good values alpha = 2, beta = .5
        float alpha = 2f;
        float beta = .5f;

        System.out.println("Latent Dirichlet Allocation using Gibbs Sampling.");

        LdaGibbsSampler lda = new LdaGibbsSampler(documents, V);
        lda.configure(10000, 2000, 100, 10);
        lda.gibbs(K, alpha, beta);

        float[][] theta = lda.getTheta();
        float[][] phi = lda.getPhi();

        System.out.println();
        System.out.println();
        System.out.println("Document--Topic Associations, Theta[d][k] (alpha="
            + alpha + ")");
        System.out.print("d\\k\t");
        for (int m = 0; m < theta[0].length; m++) {
            System.out.print("   " + m % 10 + "    ");
        }
        System.out.println();
        for (int m = 0; m < theta.length; m++) {
            System.out.print(m + "\t");
            for (int k = 0; k < theta[m].length; k++) {
                // System.out.print(theta[m][k] + " ");
                System.out.print(shadefloat(theta[m][k], 1) + " ");
            }
            System.out.println();
        }
        System.out.println();
        System.out.println("Topic--Term Associations, Phi[k][w] (beta=" + beta
            + ")");

        System.out.print("k\\w\t");
        for (int w = 0; w < phi[0].length; w++) {
            System.out.print("   " + dict.lookupString(w) + "    ");
        }
        System.out.println();
        for (int k = 0; k < phi.length; k++) {
            System.out.print(k + "\t");
            for (int w = 0; w < phi[k].length; w++) {
            	System.out.print(lnf.format(phi[k][w]) + " ");
//            	System.out.print(phi[k][w] + " ");
//                System.out.print(shadefloat(phi[k][w], 1) + " ");
            }
            System.out.println();
        }
        for (int k = 0; k < phi.length; k++) {
        	int[] top = MyArrays.sort(phi[k]);
        	
            for (int w = 0; w < 10; w++) {
            	System.out.print(dict.lookupString(top[w]) + " ");
            }
            System.out.println();
        }
    }
951846
private static void process(String input, String output) throws Exception
   {
      FileInputStream is = new FileInputStream(input);
      PrintWriter pw = new PrintWriter(new FileWriter(output));

      POIFSFileSystem fs = new POIFSFileSystem(is);
      dumpTree(pw, fs.getRoot(), "", true, true, null);

      is.close();
      pw.flush();
      pw.close();
      fs.close();
   }