Invoke a new SplittingBAMIndexer object, operating on the supplied {@link
org.apache.hadoop.conf.Configuration} object instead of a supplied
argument list

@throws java.lang.IllegalArgumentException if the "input" property is not
in the Configuration
193334
public static void run(final Configuration conf) throws IOException {
		final String inputString = conf.get("input");
		if (inputString == null)
			throw new IllegalArgumentException(
				"String property \"input\" path not found in given Configuration");

		final FileSystem fs = FileSystem.get(conf);

		final Path input = new Path(inputString);

		SplittingBAMIndexer.index(
			fs.open(input),
			fs.create(input.suffix(OUTPUT_FILE_EXTENSION)),
			fs.getFileStatus(input).getLen(),
			conf.getInt("granularity", DEFAULT_GRANULARITY));
	}
177919
private void startInteractiveConfWizard() throws IOException {
        BufferedReader br = null;
        try {
            br = new BufferedReader(new InputStreamReader(System.in));
            System.out.println("");
            chooseBootstrapPack(br);
            chooseSampleSqlAndConf(br);
            enterAppName(br);
            enterPackageName(br);
        } finally {
            br.close();
        }
    }
1229169
public static <K, V> HadoopInputFormat<K, V> createHadoopInput(org.apache.hadoop.mapred.InputFormat<K, V> mapredInputFormat, Class<K> key, Class<V> value, JobConf job) {
		return new HadoopInputFormat<>(mapredInputFormat, key, value, job);
	}
48232
public static void main(String[] args) throws Exception {

		// Checking input parameters
		final ParameterTool params = ParameterTool.fromArgs(args);
		System.out.println("Usage: TwitterExample [--output <path>] " +
				"[--twitter-source.consumerKey <key> --twitter-source.consumerSecret <secret> --twitter-source.token <token> --twitter-source.tokenSecret <tokenSecret>]");

		// set up the execution environment
		StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

		// make parameters available in the web interface
		env.getConfig().setGlobalJobParameters(params);

		env.setParallelism(params.getInt("parallelism", 1));

		// get input data
		DataStream<String> streamSource;
		if (params.has(TwitterSource.CONSUMER_KEY) &&
				params.has(TwitterSource.CONSUMER_SECRET) &&
				params.has(TwitterSource.TOKEN) &&
				params.has(TwitterSource.TOKEN_SECRET)
				) {
			streamSource = env.addSource(new TwitterSource(params.getProperties()));
		} else {
			System.out.println("Executing TwitterStream example with default props.");
			System.out.println("Use --twitter-source.consumerKey <key> --twitter-source.consumerSecret <secret> " +
					"--twitter-source.token <token> --twitter-source.tokenSecret <tokenSecret> specify the authentication info.");
			// get default test text data
			streamSource = env.fromElements(TwitterExampleData.TEXTS);
		}

		DataStream<Tuple2<String, Integer>> tweets = streamSource
				// selecting English tweets and splitting to (word, 1)
				.flatMap(new SelectEnglishAndTokenizeFlatMap())
				// group by words and sum their occurrences
				.keyBy(0).sum(1);

		// emit result
		if (params.has("output")) {
			tweets.writeAsText(params.get("output"));
		} else {
			System.out.println("Printing result to stdout. Use --output to specify output path.");
			tweets.print();
		}

		// execute program
		env.execute("Twitter Streaming Example");
	}
920963
public final void mINSERT_HADOOP() throws RecognitionException {
		try {
			int _type = INSERT_HADOOP;
			int _channel = DEFAULT_TOKEN_CHANNEL;
			// druidG.g:582:15: ( ( 'INSERT_HADOOP' | 'insert_hadoop' ) )
			// druidG.g:582:16: ( 'INSERT_HADOOP' | 'insert_hadoop' )
			{
			// druidG.g:582:16: ( 'INSERT_HADOOP' | 'insert_hadoop' )
			int alt2=2;
			int LA2_0 = input.LA(1);
			if ( (LA2_0=='I') ) {
				alt2=1;
			}
			else if ( (LA2_0=='i') ) {
				alt2=2;
			}

			else {
				NoViableAltException nvae =
					new NoViableAltException("", 2, 0, input);
				throw nvae;
			}

			switch (alt2) {
				case 1 :
					// druidG.g:582:17: 'INSERT_HADOOP'
					{
					match("INSERT_HADOOP"); 

					}
					break;
				case 2 :
					// druidG.g:582:33: 'insert_hadoop'
					{
					match("insert_hadoop"); 

					}
					break;

			}

			}

			state.type = _type;
			state.channel = _channel;
		}
		finally {
			// do for sure before leaving
		}
	}
72687
@Override
	public OUTPUT newInstance(INPUT input, INPUT... inputs) throws FAILURE {
		
		StringBuilder builder = new StringBuilder()
		.append(getClass().getName())
		.append(".newInstance(Input, Input...) is unsupported.");
		
		throw new UnsupportedOperationException(builder.toString());
	}
1081623
@Override
	public void invoke() throws Exception {
		// --------------------------------------------------------------------
		// Initialize
		// --------------------------------------------------------------------
		if (LOG.isDebugEnabled()) {
			LOG.debug(formatLogString("Start registering input and output."));
		}

		// obtain task configuration (including stub parameters)
		Configuration taskConf = getTaskConfiguration();
		this.config = new TaskConfig(taskConf);

		// now get the operator class which drives the operation
		final Class<? extends Driver<S, OT>> driverClass = this.config.getDriver();
		this.driver = InstantiationUtil.instantiate(driverClass, Driver.class);

		String headName =  getEnvironment().getTaskInfo().getTaskName().split("->")[0].trim();
		this.metrics = getEnvironment().getMetricGroup()
			.getOrAddOperator(headName.startsWith("CHAIN") ? headName.substring(6) : headName);
		this.metrics.getIOMetricGroup().reuseInputMetricsForTask();
		if (config.getNumberOfChainedStubs() == 0) {
			this.metrics.getIOMetricGroup().reuseOutputMetricsForTask();
		}

		// initialize the readers.
		// this does not yet trigger any stream consuming or processing.
		initInputReaders();
		initBroadcastInputReaders();

		// initialize the writers.
		initOutputs();

		if (LOG.isDebugEnabled()) {
			LOG.debug(formatLogString("Finished registering input and output."));
		}

		// --------------------------------------------------------------------
		// Invoke
		// --------------------------------------------------------------------
		if (LOG.isDebugEnabled()) {
			LOG.debug(formatLogString("Start task code."));
		}

		this.runtimeUdfContext = createRuntimeContext(metrics);

		// whatever happens in this scope, make sure that the local strategies are cleaned up!
		// note that the initialization of the local strategies is in the try-finally block as well,
		// so that the thread that creates them catches its own errors that may happen in that process.
		// this is especially important, since there may be asynchronous closes (such as through canceling).
		try {
			// initialize the remaining data structures on the input and trigger the local processing
			// the local processing includes building the dams / caches
			try {
				int numInputs = driver.getNumberOfInputs();
				int numComparators = driver.getNumberOfDriverComparators();
				int numBroadcastInputs = this.config.getNumBroadcastInputs();
				
				initInputsSerializersAndComparators(numInputs, numComparators);
				initBroadcastInputsSerializers(numBroadcastInputs);
				
				// set the iterative status for inputs and broadcast inputs
				{
					List<Integer> iterativeInputs = new ArrayList<Integer>();
					
					for (int i = 0; i < numInputs; i++) {
						final int numberOfEventsUntilInterrupt = getTaskConfig().getNumberOfEventsUntilInterruptInIterativeGate(i);
			
						if (numberOfEventsUntilInterrupt < 0) {
							throw new IllegalArgumentException();
						}
						else if (numberOfEventsUntilInterrupt > 0) {
							this.inputReaders[i].setIterativeReader();
							iterativeInputs.add(i);
				
							if (LOG.isDebugEnabled()) {
								LOG.debug(formatLogString("Input [" + i + "] reads in supersteps with [" +
										+ numberOfEventsUntilInterrupt + "] event(s) till next superstep."));
							}
						}
					}
					this.iterativeInputs = asArray(iterativeInputs);
				}
				
				{
					List<Integer> iterativeBcInputs = new ArrayList<Integer>();
					
					for (int i = 0; i < numBroadcastInputs; i++) {
						final int numberOfEventsUntilInterrupt = getTaskConfig().getNumberOfEventsUntilInterruptInIterativeBroadcastGate(i);
						
						if (numberOfEventsUntilInterrupt < 0) {
							throw new IllegalArgumentException();
						}
						else if (numberOfEventsUntilInterrupt > 0) {
							this.broadcastInputReaders[i].setIterativeReader();
							iterativeBcInputs.add(i);
				
							if (LOG.isDebugEnabled()) {
								LOG.debug(formatLogString("Broadcast input [" + i + "] reads in supersteps with [" +
										+ numberOfEventsUntilInterrupt + "] event(s) till next superstep."));
							}
						}
					}
					this.iterativeBroadcastInputs = asArray(iterativeBcInputs);
				}
				
				initLocalStrategies(numInputs);
			}
			catch (Exception e) {
				throw new RuntimeException("Initializing the input processing failed" +
						(e.getMessage() == null ? "." : ": " + e.getMessage()), e);
			}

			if (!this.running) {
				if (LOG.isDebugEnabled()) {
					LOG.debug(formatLogString("Task cancelled before task code was started."));
				}
				return;
			}

			// pre main-function initialization
			initialize();

			// read the broadcast variables. they will be released in the finally clause 
			for (int i = 0; i < this.config.getNumBroadcastInputs(); i++) {
				final String name = this.config.getBroadcastInputName(i);
				readAndSetBroadcastInput(i, name, this.runtimeUdfContext, 1 /* superstep one for the start */);
			}

			// the work goes here
			run();
		}
		finally {
			// clean up in any case!
			closeLocalStrategiesAndCaches();

			clearReaders(inputReaders);
			clearWriters(eventualOutputs);

		}

		if (this.running) {
			if (LOG.isDebugEnabled()) {
				LOG.debug(formatLogString("Finished task code."));
			}
		} else {
			if (LOG.isDebugEnabled()) {
				LOG.debug(formatLogString("Task code cancelled."));
			}
		}
	}
644354
@SuppressWarnings("unchecked")  // Generic array assignment
  public void readFields(DataInput in) throws IOException {
    int card = WritableUtils.readVInt(in);
    if (splits == null || splits.length != card) {
      splits = new InputSplit[card];
    }
    Class<? extends InputSplit>[] cls = new Class[card];
    try {
      for (int i = 0; i < card; ++i) {
        cls[i] =
          Class.forName(Text.readString(in)).asSubclass(InputSplit.class);
      }
      for (int i = 0; i < card; ++i) {
        splits[i] = ReflectionUtils.newInstance(cls[i], null);
        splits[i].readFields(in);
      }
    } catch (ClassNotFoundException e) {
      throw (IOException)new IOException("Failed split init").initCause(e);
    }
  }
496514
public static void main(String[] args) throws Exception {
    Builder builder = Builder.newBuilder();

    /**
     * The processing graph consists of a supplier streamlet that emits
     * random integers between 1 and 100. From there, a series of transformers
     * is applied. At the end of the graph, the original value is ultimately
     * unchanged.
     */
    builder.newSource(() -> ThreadLocalRandom.current().nextInt(100))
        .transform(new DoNothingTransformer<>())
        .transform(new IncrementTransformer(10))
        .transform(new IncrementTransformer(-7))
        .transform(new DoNothingTransformer<>())
        .transform(new IncrementTransformer(-3))
        .log();

    Config config = Config.defaultConfig();

    // Fetches the topology name from the first command-line argument
    String topologyName = StreamletUtils.getTopologyName(args);

    // Finally, the processing graph and configuration are passed to the Runner, which converts
    // the graph into a Heron topology that can be run in a Heron cluster.
    new Runner().run(topologyName, config, builder);
  }
866567
private void init(final Map<DistributedDataSetPartition, InputSplit[]> splitsPerPartition) {
    final Pair<InputSplit[], DistributedDataSetPartition[]>
                                      splitsAndPartitions = getSplitsAndPartitions(splitsPerPartition);
    final InputSplit[] splits = splitsAndPartitions.getFirst();
    final DistributedDataSetPartition[] partitions = splitsAndPartitions.getSecond();
    Validate.isTrue(splits.length == partitions.length);
    for (int splitNum = 0; splitNum < splits.length; splitNum++) {
      LOG.log(Level.FINE, "Processing split: " + splitNum);
      final InputSplit split = splits[splitNum];
      final NumberedSplit<InputSplit> numberedSplit = new NumberedSplit<>(split, splitNum,
          partitions[splitNum]);
      unallocatedSplits.add(numberedSplit);
      updateLocations(numberedSplit);
    }
    if (LOG.isLoggable(Level.FINE)) {
      for (final Map.Entry<String, BlockingQueue<NumberedSplit<InputSplit>>> locSplit : locationToSplits.entrySet()) {
        LOG.log(Level.FINE, locSplit.getKey() + ": " + locSplit.getValue().toString());
      }
    }
  }
1039937
@Override
  public int run(String[] args) throws Exception {

    myConf = getConf();

    // Presume this is all HDFS paths, even when access as file://
    hdfs = FileSystem.get(myConf);

    // Grab input args and allow for -Dxyz style arguments
    String[] otherArgs = new GenericOptionsParser(myConf, args)
        .getRemainingArgs();

    // Grab the arguments we're looking for.
    CommandLine commandLine = parseArgs(otherArgs);

    // Grab the input path argument
    input = commandLine.getOptionValue("i");
    LOG.info("input=" + input);

    // Grab the input path argument
    String output = commandLine.getOptionValue("o");
    LOG.info("output=" + output);

    skipExisting = commandLine.hasOption("s");
    LOG.info("skipExisting=" + skipExisting);

    moveFiles = commandLine.hasOption("m");
    LOG.info("moveFiles=" + moveFiles);

    if (skipExisting && moveFiles) {
      throw new IllegalArgumentException(
          "Cannot use both options skipExisting and move simultaneously.");
    }

    if (commandLine.hasOption("x")) {
      try {
        maXretention = Integer.parseInt(commandLine.getOptionValue("x"));
      } catch (NumberFormatException nfe) {
        throw new IllegalArgumentException(
            "maXretention option -x is is not a valid number: "
                + commandLine.getOptionValue("x"), nfe);
      }
      // Additional check
      if (maXretention < 0) {
        throw new IllegalArgumentException(
            "Cannot retain less than 0 files. Specified maXretention option -x is: "
                + commandLine.getOptionValue("x"));
      }
      LOG.info("maXretention=" + maXretention);
      if (moveFiles) {
        throw new IllegalArgumentException(
            "Cannot use both options maXretention and move simultaneously.");
      }
    } else {
      maXretention = Integer.MAX_VALUE;
    }

    outputPath = new Path(output);
    FileStatus outputFileStatus = hdfs.getFileStatus(outputPath);

    if (!outputFileStatus.isDir()) {
      throw new IOException("Output is not a directory"
          + outputFileStatus.getPath().getName());
    }

    Path inputPath = new Path(input);
    URI inputURI = inputPath.toUri();
    String inputScheme = inputURI.getScheme();

    LOG.info("input scheme is: " + inputScheme);

    // If input directory is HDFS, then process as such. Assume not scheme is
    // HDFS
    if ((inputScheme == null)
        || (hdfs.getUri().getScheme().equals(inputScheme))) {
      processHDFSSources(inputPath);
    } else if (inputScheme.equals("file")) {
      if (moveFiles) {
        throw new IllegalArgumentException(
            "Cannot move files that are not already in hdfs. Input is not HDFS: "
                + input);
      }
      processPlainFileSources(inputURI);
    } else {
      throw new IllegalArgumentException(
          "Cannot process files from this URI scheme: " + inputScheme);
    }

    Statistics statistics = FileSystem.getStatistics(outputPath.toUri()
        .getScheme(), hdfs.getClass());
    if (statistics != null) {
      LOG.info("HDFS bytes read: " + statistics.getBytesRead());
      LOG.info("HDFS bytes written: " + statistics.getBytesWritten());
      LOG.info("HDFS read ops: " + statistics.getReadOps());
      System.out
          .println("HDFS large read ops: " + statistics.getLargeReadOps());
      LOG.info("HDFS write ops: " + statistics.getWriteOps());
    }

    return 0;
  }
1083569
public static void main(String[] args) throws Exception {
    parseArgs(args);

    GenMunger rawMunger;
    rawMunger = (hex.genmodel.GenMunger) Class.forName(assemblyClassName).newInstance();

    BufferedReader input = new BufferedReader(new FileReader(inputCSVFileName));
    BufferedWriter output = new BufferedWriter(new FileWriter(outputCSVFileName));

    // Emit outputCSV column names.
    String[] rawHeader = rawMunger.outNames();
    StringBuilder header = new StringBuilder();
    for(int i=0;i<rawHeader.length;++i) {
      header.append("\"").append(rawHeader[i]).append("\"");
      if( i < rawHeader.length - 1 ) header.append(",");
    }
    output.write(header.toString());
    output.write("\n");

    // Loop over inputCSV one row at a time.
    int lineNum = 0;
    String line;
    try {
      while ((line = input.readLine()) != null) {
        lineNum++;

        // skip the header.
        if (lineNum == 1)
          continue;

        // Parse the CSV line.  Somewhat handles quoted commas.  But this ain't no parser test!
        RowData row;
        try {
          row = parseDataRow(line, rawMunger);
        } catch( NumberFormatException nfe) {
          nfe.printStackTrace();
          System.out.println("Failed to parse row: " + lineNum );
          throw new RuntimeException();
        }
        RowData mungedRow = rawMunger.fit(row);

        for(int i=0; i<rawMunger.outNames().length;++i) {
          Object val = mungedRow==null?Double.NaN:mungedRow.get(rawMunger.outNames()[i]);
          if( val instanceof Double ) output.write(String.valueOf(val));
          else                        output.write("\"" + val + "\"");
          if( i < rawMunger.outNames().length - 1) output.write(",");
        }
        output.write("\n");
      }
    }
    catch (Exception e) {
      System.out.println("Caught exception on line " + lineNum);
      System.out.println("");
      e.printStackTrace();
      System.exit(1);
    }
    finally {

      // Clean up.
      output.close();
      input.close();
    }

    // Predictions were successfully generated.  Calling program can now compare them with something.
    System.exit(0);
  }
562149
public static void main(String[] args) throws Exception {
    Builder builder = Builder.newBuilder();

    Streamlet<Integer> zeroes = builder.newSource(() -> 0);

    builder.newSource(() -> ThreadLocalRandom.current().nextInt(1, 11))
        .setName("random-ints")
        .map(i -> i + 1)
        .setName("add-one")
        .union(zeroes)
        .setName("unify-streams")
        .filter(i -> i != 2)
        .setName("remove-twos")
        .log();

    Config config = Config.newBuilder()
        .setNumContainers(NUM_CONTAINERS)
        .setPerContainerRamInGigabytes(GIGABYTES_OF_RAM)
        .setPerContainerCpu(CPU)
        .build();

    // Fetches the topology name from the first command-line argument
    String topologyName = StreamletUtils.getTopologyName(args);

    // Finally, the processing graph and configuration are passed to the Runner, which converts
    // the graph into a Heron topology that can be run in a Heron cluster.
    new Runner().run(topologyName, config, builder);
  }
7026
public void configure(InputStream is) throws IOException {
        BufferedReader reader = new BufferedReader(new InputStreamReader(is));
        int lineNumber = 0;

        while (true) {
            String line = reader.readLine();
            if (line == null) break;

            line = line.trim();
            lineNumber++;

            if (line.startsWith("#") || line.length() == 0) continue;

            if (line.startsWith(LOAD_PREFIX)) {
                String loadPath = line.substring(LOAD_PREFIX.length()).trim();
                loadPath = assignProperties(loadPath);
                loadFilteredPath(loadPath);
            } else if (line.startsWith(GRAB_PREFIX)) {
                String grabParams = line.substring(GRAB_PREFIX.length()).trim();
                grabList.add(assignProperties(grabParams));
            } else if (line.startsWith(MAIN_PREFIX)) {
                if (main != null)
                    throw new IOException("duplicate definition of main in line " + lineNumber + " : " + line);
                main = line.substring(MAIN_PREFIX.length()).trim();
            } else if (line.startsWith(PROP_PREFIX)) {
                String params = line.substring(PROP_PREFIX.length()).trim();
                int index = params.indexOf('=');
                if (index == -1) {
                    throw new IOException("unexpected property format - expecting name=value" + lineNumber + " : " + line);
                }
                String propName = params.substring(0, index);
                String propValue = assignProperties(params.substring(index + 1));
                System.setProperty(propName, propValue);
            } else {
                throw new IOException("unexpected line in " + lineNumber + " : " + line);
            }
        }

        if (requireMain && main == null) throw new IOException("missing main class definition in config file");
    }
145863
public final void arguments() throws RecognitionException {
		int arguments_StartIndex = input.index();

		try {
			if ( state.backtracking>0 && alreadyParsedRule(input, 140) ) { return; }

			// src/main/resources/org/drools/compiler/semantics/java/parser/Java.g:1297:5: ( '(' ( expressionList )? ')' )
			// src/main/resources/org/drools/compiler/semantics/java/parser/Java.g:1297:7: '(' ( expressionList )? ')'
			{
			match(input,36,FOLLOW_36_in_arguments6395); if (state.failed) return;
			// src/main/resources/org/drools/compiler/semantics/java/parser/Java.g:1297:11: ( expressionList )?
			int alt192=2;
			int LA192_0 = input.LA(1);
			if ( ((LA192_0 >= CharacterLiteral && LA192_0 <= DecimalLiteral)||LA192_0==FloatingPointLiteral||(LA192_0 >= HexLiteral && LA192_0 <= Identifier)||(LA192_0 >= OctalLiteral && LA192_0 <= StringLiteral)||LA192_0==29||LA192_0==36||(LA192_0 >= 40 && LA192_0 <= 41)||(LA192_0 >= 44 && LA192_0 <= 45)||LA192_0==53||LA192_0==65||LA192_0==67||(LA192_0 >= 70 && LA192_0 <= 71)||LA192_0==77||(LA192_0 >= 79 && LA192_0 <= 80)||LA192_0==82||LA192_0==85||LA192_0==92||LA192_0==94||(LA192_0 >= 97 && LA192_0 <= 98)||LA192_0==105||LA192_0==108||LA192_0==111||LA192_0==115||LA192_0==118||LA192_0==126) ) {
				alt192=1;
			}
			switch (alt192) {
				case 1 :
					// src/main/resources/org/drools/compiler/semantics/java/parser/Java.g:1297:11: expressionList
					{
					pushFollow(FOLLOW_expressionList_in_arguments6397);
					expressionList();
					state._fsp--;
					if (state.failed) return;
					}
					break;

			}

			match(input,37,FOLLOW_37_in_arguments6400); if (state.failed) return;
			}

		}
		catch (RecognitionException re) {
			reportError(re);
			recover(input,re);
		}
		finally {
			// do for sure before leaving
			if ( state.backtracking>0 ) { memoize(input, 140, arguments_StartIndex); }

		}
	}
700558
@SuppressWarnings("OptionalGetWithoutIsPresent")
  @Override
  public void run() throws Exception {
    final Optional<File> inputMapFile = parameters.getOptionalExistingFile(INPUT_MAP_PARAM);
    final Optional<File> inputListFile = parameters.getOptionalExistingFile(INPUT_LIST_PARAM);
    final Optional<File> outputMapFile = parameters.getOptionalCreatableFile(OUTPUT_MAP_PARAM);
    final Optional<File> outputListFile = parameters.getOptionalCreatableFile(OUTPUT_LIST_PARAM);
    final Optional<File> outputDirectory =
        parameters.getOptionalCreatableDirectory(OUTPUT_DIR_PARAM);
    final boolean inPlace = parameters.getOptionalBoolean(IN_PLACE_PARAM).or(false);
    final Optional<File> basePath = parameters.getOptionalExistingDirectory(BASE_PATH_PARAM);

    if (inputListFile.isPresent() == inputMapFile.isPresent()) {
      log.error("Exactly one input parameter given. Expected exactly one of {} and {}",
          INPUT_LIST_PARAM, INPUT_MAP_PARAM);
      System.exit(1);
    }

    if (!outputListFile.isPresent() && !outputMapFile.isPresent() && !inPlace) {
      log.error("No output parameter given. Expected {} or {} or for {} to be true",
          OUTPUT_LIST_PARAM, OUTPUT_MAP_PARAM, IN_PLACE_PARAM);
      System.exit(1);
    }

    if (outputMapFile.isPresent() && !inputMapFile.isPresent()) {
      log.error("Cannot use {} without {}", OUTPUT_MAP_PARAM, INPUT_MAP_PARAM);
      System.exit(1);
    }

    if (!inPlace && !basePath.isPresent()) {
      log.error("Cannot determine whow to output. Either {} must be true or {} must be "
          + "specified", IN_PLACE_PARAM, BASE_PATH_PARAM);
    }

    final ImmutableMap<Symbol, File> inputFileMap;
    if (inputMapFile.isPresent()) {
      inputFileMap = FileUtils.loadSymbolToFileMap(inputMapFile.get());
    } else {
      // if a map wasn't supplied, we make a fake map using the file path as the key.
      // we know the key won't matter, because we know map output wasn't requested or
      // the checks above would have caught the missing input map
      // .get() is safe by checks above
      inputFileMap = Maps.uniqueIndex(FileUtils.loadFileList(inputListFile.get()),
          compose(symbolizeFunction(), toAbsolutePathFunction()));
    }
    log.info("Cleaning {} input files {}", inputFileMap.size(),
        inPlace ? "in-place" : ("to " + outputDirectory.get().getAbsolutePath()));

    int totalCharactersReplaced = 0;
    int numFilesWithReplacements = 0;

    final ImmutableList.Builder<File> outputFiles = ImmutableList.builder();
    final ImmutableMap.Builder<Symbol, File> outputMap = ImmutableMap.builder();
    for (final Map.Entry<Symbol, File> e : inputFileMap.entrySet()) {
      final File inputFileName = e.getValue();
      final File outFile;
      if (inPlace) {
        outFile = inputFileName;
      } else {
        // write the output to a file with the same position relative to the output directory
        // as the input had relative to the input directory
        // get safe by checks above
        //noinspection OptionalGetWithoutIsPresent
        outFile = outputDirectory.get().toPath().resolve(
            basePath.get().toPath().relativize(inputFileName.toPath())).toFile();
      }
      //noinspection ResultOfMethodCallIgnored
      outFile.getParentFile().mkdirs();

      final String originalText = Files.asCharSource(inputFileName, Charsets.UTF_8).read();
      final int numReplacementCharsInOriginal =
          CodepointMatcher.forCharacter(UNICODE_REPLACEMENT_CHARACTER).countIn(originalText);
      final String cleanedText = not(basicMultilingualPlane()).replaceAll(
          originalText, UNICODE_REPLACEMENT_CHARACTER);
      final int numReplacementCharsInCleaned =
          CodepointMatcher.forCharacter(UNICODE_REPLACEMENT_CHARACTER).countIn(cleanedText);
      Files.asCharSink(outFile, Charsets.UTF_8).write(cleanedText);

      outputFiles.add(outFile);
      outputMap.put(e.getKey(), outFile);

      final int numCharsReplaced = numReplacementCharsInCleaned - numReplacementCharsInOriginal;

      checkState(numCharsReplaced >= 0,
          "Number of replacement characters went down.  Either this program is buggy or you have"
              + " a very broken and bizarre document");
      if (numCharsReplaced > 0) {
        log.info("Replaced {} non-BMP code points with the Unicode replacement character for"
            + "input file {}", numCharsReplaced, inputFileName);
        totalCharactersReplaced += numCharsReplaced;
        ++numFilesWithReplacements;
      }
    }

    log.info("Replaced {} non-BMP characters in {} files",
        totalCharactersReplaced, numFilesWithReplacements);
    if (outputListFile.isPresent()) {
      log.info("Writing list of transformed files to {}", outputListFile.get());
      FileUtils.writeFileList(outputFiles.build(), Files.asCharSink(outputListFile.get(),
          Charsets.UTF_8));
    }
    if (outputMapFile.isPresent()) {
      log.info("Writing map of transformed files to {}", outputMapFile.get());
      FileUtils.writeSymbolToFileMap(outputMap.build(), Files.asCharSink(outputMapFile.get(),
          Charsets.UTF_8));
    }
  }
432904
public void write(Writer writer, Map<String, String> outputProperties) throws TransformerException {
		Transformer transformer;
		try {
			transformer = TransformerFactory.newInstance().newTransformer();
		} catch (TransformerConfigurationException e) {
			//should never be thrown because we're not doing anything fancy with the configuration
			throw new RuntimeException(e);
		} catch (TransformerFactoryConfigurationError e) {
			//should never be thrown because we're not doing anything fancy with the configuration
			throw new RuntimeException(e);
		}

		/*
		 * Using Transformer#setOutputProperties(Properties) doesn't work for
		 * some reason for setting the number of indentation spaces.
		 */
		for (Map.Entry<String, String> entry : outputProperties.entrySet()) {
			String key = entry.getKey();
			String value = entry.getValue();
			transformer.setOutputProperty(key, value);
		}

		DOMSource source = new DOMSource(document);
		StreamResult result = new StreamResult(writer);
		transformer.transform(source, result);
	}