Writes the map file according to the given configuration using the given data processor.

@param configuration the configuration
@param dataProcessor the data processor
@throws IOException thrown if any IO error occurs
977656
public static void writeFile(MapWriterConfiguration configuration, TileBasedDataProcessor dataProcessor)
            throws IOException {
        EXECUTOR_SERVICE = Executors.newFixedThreadPool(configuration.getThreads());
        RandomAccessFile randomAccessFile = new RandomAccessFile(configuration.getOutputFile(), "rw");

        int amountOfZoomIntervals = dataProcessor.getZoomIntervalConfiguration().getNumberOfZoomIntervals();
        ByteBuffer containerHeaderBuffer = ByteBuffer.allocate(HEADER_BUFFER_SIZE);
        // CONTAINER HEADER
        int totalHeaderSize = writeHeaderBuffer(configuration, dataProcessor, containerHeaderBuffer);

        // set to mark where zoomIntervalConfig starts
        containerHeaderBuffer.reset();

        final LoadingCache<TDWay, Geometry> jtsGeometryCache = CacheBuilder.newBuilder()
                .maximumSize(JTS_GEOMETRY_CACHE_SIZE).concurrencyLevel(Runtime.getRuntime().availableProcessors() * 2)
                .build(new JTSGeometryCacheLoader(dataProcessor));

        // SUB FILES
        // for each zoom interval write a sub file
        long currentFileSize = totalHeaderSize;
        for (int i = 0; i < amountOfZoomIntervals; i++) {
            // SUB FILE INDEX AND DATA
            long subfileSize = writeSubfile(currentFileSize, i, dataProcessor, jtsGeometryCache, randomAccessFile,
                    configuration);
            // SUB FILE META DATA IN CONTAINER HEADER
            writeSubfileMetaDataToContainerHeader(dataProcessor.getZoomIntervalConfiguration(), i, currentFileSize,
                    subfileSize, containerHeaderBuffer);
            currentFileSize += subfileSize;
        }

        randomAccessFile.seek(0);
        randomAccessFile.write(containerHeaderBuffer.array(), 0, totalHeaderSize);

        // WRITE FILE SIZE TO HEADER
        long fileSize = randomAccessFile.length();
        randomAccessFile.seek(OFFSET_FILE_SIZE);
        randomAccessFile.writeLong(fileSize);

        randomAccessFile.close();

        CacheStats stats = jtsGeometryCache.stats();
        LOGGER.fine("Tag values stats:\n" + OSMUtils.logValueTypeCount());
        LOGGER.info("JTS Geometry cache hit rate: " + stats.hitRate());
        LOGGER.info("JTS Geometry total load time: " + stats.totalLoadTime() / 1000);

        LOGGER.info("Finished writing file.");
    }
757365
static <K, V, T, R> void writeTo(ObjectOutput output, Mutation<K, V, R> mutation) throws IOException {
      BaseMutation bm = (BaseMutation) mutation;
      DataConversion.writeTo(output, bm.keyDataConversion);
      DataConversion.writeTo(output, bm.valueDataConversion);
      byte type = mutation.type();
      output.writeByte(type);
      switch (type) {
         case ReadWrite.TYPE:
            output.writeObject(((ReadWrite<K, V, ?>) mutation).f);
            break;
         case ReadWriteWithValue.TYPE:
            ReadWriteWithValue<K, V, T, R> rwwv = (ReadWriteWithValue<K, V, T, R>) mutation;
            output.writeObject(rwwv.argument);
            output.writeObject(rwwv.f);
            break;
         case Write.TYPE:
            output.writeObject(((Write<K, V>) mutation).f);
            break;
         case WriteWithValue.TYPE:
            WriteWithValue<K, V, T> wwv = (WriteWithValue<K, V, T>) mutation;
            output.writeObject(wwv.argument);
            output.writeObject(wwv.f);
            break;
      }
   }
300711
public final static DataWriter createDataWriter(File file, IOType type) {
        if(type == IOType.MAPPED) {
            if(file.length() <= Integer.MAX_VALUE) {
                return new MappedWriter(file);
            } else {
                return new MultiMappedWriter(file);
            }
        } else {
            return new ChannelWriter(file);
        }
    }
587365
public void processConfiguration(Configuration configuration, Class<?> mapperInterface) {
        String prefix;
        if (mapperInterface != null) {
            prefix = mapperInterface.getCanonicalName();
        } else {
            prefix = "";
        }
        for (Object object : new ArrayList<Object>(configuration.getMappedStatements())) {
            if (object instanceof MappedStatement) {
                MappedStatement ms = (MappedStatement) object;
                if (ms.getId().startsWith(prefix)) {
                    processMappedStatement(ms);
                }
            }
        }
    }
382795
@SuppressWarnings({"unchecked"})
  public void map(Object key, Object value, OutputCollector output,
                  Reporter reporter) throws IOException {
    Mapper mapper = chain.getFirstMap();
    if (mapper != null) {
      mapper.map(key, value, chain.getMapperCollector(0, output, reporter),
                 reporter);
    }
  }
1265206
private void writeObject(ObjectOutputStream out) throws IOException {
		out.writeUTF(hadoopInputFormatName);
		jobConf.write(out);
		out.writeObject(converter);
	}
642222
public void write(Configuration config)
    throws IOException {

    pp.startDocument();
    pp.startElement("duke", null);

    // FIXME: here we should write the objects, but that's not
    // possible with the current API. we don't need that for the
    // genetic algorithm at the moment, but it would be useful.

    pp.startElement("schema", null);

    writeElement("threshold", "" + config.getThreshold());
    if (config.getMaybeThreshold() != 0.0)
      writeElement("maybe-threshold", "" + config.getMaybeThreshold());

    for (Property p : config.getProperties())
      writeProperty(p);

    pp.endElement("schema");

    String dbclass = config.getDatabase(false).getClass().getName();
    AttributeListImpl atts = new AttributeListImpl();
    atts.addAttribute("class", "CDATA", dbclass);
    pp.startElement("database", atts);
    pp.endElement("database");

    if (config.isDeduplicationMode())
      for (DataSource src : config.getDataSources())
        writeDataSource(src);
    else {
      pp.startElement("group", null);
      for (DataSource src : config.getDataSources(1))
        writeDataSource(src);
      pp.endElement("group");

      pp.startElement("group", null);
      for (DataSource src : config.getDataSources(2))
        writeDataSource(src);
      pp.endElement("group");
    }

    pp.endElement("duke");
    pp.endDocument();
  }
214237
public void write(DataOutput out) throws IOException
    {
        out.writeUTF(startToken);
        out.writeUTF(endToken);
        out.writeInt(dataNodes.length);
        for (String endpoint : dataNodes)
        {
            out.writeUTF(endpoint);
        }
    }
1002107
public void map(Object key, Object value, OutputCollector output, Reporter reporter) throws IOException {
    if (outerrThreadsThrowable != null) {
      mapRedFinished();
      throw new IOException ("MROutput/MRErrThread failed:"
                             + StringUtils.stringifyException(
                                                              outerrThreadsThrowable));
    }
    try {
      // 1/4 Hadoop in
      numRecRead_++;
      maybeLogRecord();
      if (debugFailDuring_ && numRecRead_ == 3) {
        throw new IOException("debugFailDuring_");
      }

      // 2/4 Hadoop to Tool
      if (numExceptions_ == 0) {
        if (!this.ignoreKey) {
          write(key);
          clientOut_.write(getInputSeparator());
        }
        write(value);
        if (!ignoreNewLine) {
          clientOut_.write('\n');
        }
        if(skipping) {
          //flush the streams on every record input if running in skip mode
          //so that we don't buffer other records surrounding a bad record. 
          clientOut_.flush();
        }
      } else {
        numRecSkipped_++;
      }
    } catch (IOException io) {
      numExceptions_++;
      if (numExceptions_ > 1 || numRecWritten_ < minRecWrittenToEnableSkip_) {
        // terminate with failure
        String msg = logFailure(io);
        appendLogToJobLog("failure");
        mapRedFinished();
        throw new IOException(msg);
      } else {
        // terminate with success:
        // swallow input records although the stream processor failed/closed
      }
    }
  }
92346
static public void dumpInfo(String file, PrintStream out, Configuration conf)
      throws IOException {
    final int maxKeySampleLen = 16;
    Path path = new Path(file);
    FileSystem fs = path.getFileSystem(conf);
    long length = fs.getFileStatus(path).getLen();
    FSDataInputStream fsdis = fs.open(path);
    TFile.Reader reader = new TFile.Reader(fsdis, length, conf);
    try {
      LinkedHashMap<String, String> properties =
          new LinkedHashMap<String, String>();
      int blockCnt = reader.readerBCF.getBlockCount();
      int metaBlkCnt = reader.readerBCF.metaIndex.index.size();
      properties.put("BCFile Version", reader.readerBCF.version.toString());
      properties.put("TFile Version", reader.tfileMeta.version.toString());
      properties.put("File Length", Long.toString(length));
      properties.put("Data Compression", reader.readerBCF
          .getDefaultCompressionName());
      properties.put("Record Count", Long.toString(reader.getEntryCount()));
      properties.put("Sorted", Boolean.toString(reader.isSorted()));
      if (reader.isSorted()) {
        properties.put("Comparator", reader.getComparatorName());
      }
      properties.put("Data Block Count", Integer.toString(blockCnt));
      long dataSize = 0, dataSizeUncompressed = 0;
      if (blockCnt > 0) {
        for (int i = 0; i < blockCnt; ++i) {
          BlockRegion region =
              reader.readerBCF.dataIndex.getBlockRegionList().get(i);
          dataSize += region.getCompressedSize();
          dataSizeUncompressed += region.getRawSize();
        }
        properties.put("Data Block Bytes", Long.toString(dataSize));
        if (reader.readerBCF.getDefaultCompressionName() != "none") {
          properties.put("Data Block Uncompressed Bytes", Long
              .toString(dataSizeUncompressed));
          properties.put("Data Block Compression Ratio", String.format(
              "1:%.1f", (double) dataSizeUncompressed / dataSize));
        }
      }

      properties.put("Meta Block Count", Integer.toString(metaBlkCnt));
      long metaSize = 0, metaSizeUncompressed = 0;
      if (metaBlkCnt > 0) {
        Collection<MetaIndexEntry> metaBlks =
            reader.readerBCF.metaIndex.index.values();
        boolean calculateCompression = false;
        for (Iterator<MetaIndexEntry> it = metaBlks.iterator(); it.hasNext();) {
          MetaIndexEntry e = it.next();
          metaSize += e.getRegion().getCompressedSize();
          metaSizeUncompressed += e.getRegion().getRawSize();
          if (e.getCompressionAlgorithm() != Compression.Algorithm.NONE) {
            calculateCompression = true;
          }
        }
        properties.put("Meta Block Bytes", Long.toString(metaSize));
        if (calculateCompression) {
          properties.put("Meta Block Uncompressed Bytes", Long
              .toString(metaSizeUncompressed));
          properties.put("Meta Block Compression Ratio", String.format(
              "1:%.1f", (double) metaSizeUncompressed / metaSize));
        }
      }
      properties.put("Meta-Data Size Ratio", String.format("1:%.1f",
          (double) dataSize / metaSize));
      long leftOverBytes = length - dataSize - metaSize;
      long miscSize =
          BCFile.Magic.size() * 2 + Long.SIZE / Byte.SIZE + Version.size();
      long metaIndexSize = leftOverBytes - miscSize;
      properties.put("Meta Block Index Bytes", Long.toString(metaIndexSize));
      properties.put("Headers Etc Bytes", Long.toString(miscSize));
      // Now output the properties table.
      int maxKeyLength = 0;
      Set<Map.Entry<String, String>> entrySet = properties.entrySet();
      for (Iterator<Map.Entry<String, String>> it = entrySet.iterator(); it
          .hasNext();) {
        Map.Entry<String, String> e = it.next();
        if (e.getKey().length() > maxKeyLength) {
          maxKeyLength = e.getKey().length();
        }
      }
      for (Iterator<Map.Entry<String, String>> it = entrySet.iterator(); it
          .hasNext();) {
        Map.Entry<String, String> e = it.next();
        out.printf("%s : %s\n", Align.format(e.getKey(), maxKeyLength,
            Align.LEFT), e.getValue());
      }
      out.println();
      reader.checkTFileDataIndex();
      if (blockCnt > 0) {
        String blkID = "Data-Block";
        int blkIDWidth = Align.calculateWidth(blkID, blockCnt);
        int blkIDWidth2 = Align.calculateWidth("", blockCnt);
        String offset = "Offset";
        int offsetWidth = Align.calculateWidth(offset, length);
        String blkLen = "Length";
        int blkLenWidth =
            Align.calculateWidth(blkLen, dataSize / blockCnt * 10);
        String rawSize = "Raw-Size";
        int rawSizeWidth =
            Align.calculateWidth(rawSize, dataSizeUncompressed / blockCnt * 10);
        String records = "Records";
        int recordsWidth =
            Align.calculateWidth(records, reader.getEntryCount() / blockCnt
                * 10);
        String endKey = "End-Key";
        int endKeyWidth = Math.max(endKey.length(), maxKeySampleLen * 2 + 5);

        out.printf("%s %s %s %s %s %s\n", Align.format(blkID, blkIDWidth,
            Align.CENTER), Align.format(offset, offsetWidth, Align.CENTER),
            Align.format(blkLen, blkLenWidth, Align.CENTER), Align.format(
                rawSize, rawSizeWidth, Align.CENTER), Align.format(records,
                recordsWidth, Align.CENTER), Align.format(endKey, endKeyWidth,
                Align.LEFT));

        for (int i = 0; i < blockCnt; ++i) {
          BlockRegion region =
              reader.readerBCF.dataIndex.getBlockRegionList().get(i);
          TFileIndexEntry indexEntry = reader.tfileIndex.getEntry(i);
          out.printf("%s %s %s %s %s ", Align.format(Align.format(i,
              blkIDWidth2, Align.ZERO_PADDED), blkIDWidth, Align.LEFT), Align
              .format(region.getOffset(), offsetWidth, Align.LEFT), Align
              .format(region.getCompressedSize(), blkLenWidth, Align.LEFT),
              Align.format(region.getRawSize(), rawSizeWidth, Align.LEFT),
              Align.format(indexEntry.kvEntries, recordsWidth, Align.LEFT));
          byte[] key = indexEntry.key;
          boolean asAscii = true;
          int sampleLen = Math.min(maxKeySampleLen, key.length);
          for (int j = 0; j < sampleLen; ++j) {
            byte b = key[j];
            if ((b < 32 && b != 9) || (b == 127)) {
              asAscii = false;
            }
          }
          if (!asAscii) {
            out.print("0X");
            for (int j = 0; j < sampleLen; ++j) {
              byte b = key[i];
              out.printf("%X", b);
            }
          } else {
            out.print(new String(key, 0, sampleLen));
          }
          if (sampleLen < key.length) {
            out.print("...");
          }
          out.println();
        }
      }

      out.println();
      if (metaBlkCnt > 0) {
        String name = "Meta-Block";
        int maxNameLen = 0;
        Set<Map.Entry<String, MetaIndexEntry>> metaBlkEntrySet =
            reader.readerBCF.metaIndex.index.entrySet();
        for (Iterator<Map.Entry<String, MetaIndexEntry>> it =
            metaBlkEntrySet.iterator(); it.hasNext();) {
          Map.Entry<String, MetaIndexEntry> e = it.next();
          if (e.getKey().length() > maxNameLen) {
            maxNameLen = e.getKey().length();
          }
        }
        int nameWidth = Math.max(name.length(), maxNameLen);
        String offset = "Offset";
        int offsetWidth = Align.calculateWidth(offset, length);
        String blkLen = "Length";
        int blkLenWidth =
            Align.calculateWidth(blkLen, metaSize / metaBlkCnt * 10);
        String rawSize = "Raw-Size";
        int rawSizeWidth =
            Align.calculateWidth(rawSize, metaSizeUncompressed / metaBlkCnt
                * 10);
        String compression = "Compression";
        int compressionWidth = compression.length();
        out.printf("%s %s %s %s %s\n", Align.format(name, nameWidth,
            Align.CENTER), Align.format(offset, offsetWidth, Align.CENTER),
            Align.format(blkLen, blkLenWidth, Align.CENTER), Align.format(
                rawSize, rawSizeWidth, Align.CENTER), Align.format(compression,
                compressionWidth, Align.LEFT));

        for (Iterator<Map.Entry<String, MetaIndexEntry>> it =
            metaBlkEntrySet.iterator(); it.hasNext();) {
          Map.Entry<String, MetaIndexEntry> e = it.next();
          String blkName = e.getValue().getMetaName();
          BlockRegion region = e.getValue().getRegion();
          String blkCompression =
              e.getValue().getCompressionAlgorithm().getName();
          out.printf("%s %s %s %s %s\n", Align.format(blkName, nameWidth,
              Align.LEFT), Align.format(region.getOffset(), offsetWidth,
              Align.LEFT), Align.format(region.getCompressedSize(),
              blkLenWidth, Align.LEFT), Align.format(region.getRawSize(),
              rawSizeWidth, Align.LEFT), Align.format(blkCompression,
              compressionWidth, Align.LEFT));
        }
      }
    } finally {
      IOUtils.cleanup(LOG, reader, fsdis);
    }
  }
1000485
protected void writeHeader()
		throws ConfigurationException, IOException
	{

		if (this.output != null) {
			close();
		}

		this.counter++;

		String filePath = PATH_OUTPUT_SQL_FILES + this.outputName +"_"+counter;

		this.output = OutputFactory.getOutputStream(filePath);

		this.dataArchive = new File(filePath);

		this.output.flush();
	}
1295073
public static void write(IDataSet dataSet, OutputStream out) throws IOException, DataSetException {
		logger.debug("write(dataSet={}, out={}) - start", dataSet, out);

		FlatXmlWriter datasetWriter = new FlatXmlWriter(out);
		datasetWriter.setIncludeEmptyTable(true);
		datasetWriter.write(dataSet);
	}
579104
public static void writeConfigFile(OutputStream outputStream, Class<?>[] classes, boolean sortClasses)
			throws SQLException, IOException {
		if (sortClasses) {
			// sort our class list to make the output more deterministic
			Class<?>[] sortedClasses = new Class<?>[classes.length];
			System.arraycopy(classes, 0, sortedClasses, 0, classes.length);
			Arrays.sort(sortedClasses, classComparator);
			classes = sortedClasses;
		}
		BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(outputStream), 4096);
		try {
			writeHeader(writer);
			for (Class<?> clazz : classes) {
				writeConfigForTable(writer, clazz, sortClasses);
			}
			// NOTE: done is here because this is public
			System.out.println("Done.");
		} finally {
			writer.close();
		}
	}
444206
private void write() throws IOException
   {
      // Following USACE specification from 140.194.76.129/publications/eng-regs/ER_1-1-11/ER_1-1-11.pdf
      writeFileCreationRecord(); // VOLM
      writeProjectProperties(m_projectFile.getProjectProperties()); // PROJ
      writeCalendars(m_projectFile.getCalendars()); // CLDR
      writeExceptions(m_projectFile.getCalendars()); // HOLI
      writeTasks(m_projectFile.getTasks()); // ACTV
      writePredecessors(m_projectFile.getTasks()); // PRED
      // skipped UNIT cost record for now
      writeProgress(m_projectFile.getTasks()); // PROG
      m_writer.println("END"); // last line, that's the end!!!
   }
998453
private static <T> void writeConfig(BufferedWriter writer, DatabaseTableConfig<T> config) throws IOException,
			SQLException {
		writer.append(CONFIG_FILE_START_MARKER);
		writer.newLine();
		if (config.getDataClass() != null) {
			writer.append(FIELD_NAME_DATA_CLASS).append('=').append(config.getDataClass().getName());
			writer.newLine();
		}
		if (config.getTableName() != null) {
			writer.append(FIELD_NAME_TABLE_NAME).append('=').append(config.getTableName());
			writer.newLine();
		}
		writer.append(CONFIG_FILE_FIELDS_START);
		writer.newLine();
		if (config.getFieldConfigs() != null) {
			for (DatabaseFieldConfig field : config.getFieldConfigs()) {
				DatabaseFieldConfigLoader.write(writer, field, config.getTableName());
			}
		}
		writer.append(CONFIG_FILE_FIELDS_END);
		writer.newLine();
		writer.append(CONFIG_FILE_END_MARKER);
		writer.newLine();
	}
875477
public void write(DataOutput out) throws IOException {
    out.writeInt(getLayoutVersion());
    out.writeInt(getNamespaceID());
    out.writeLong(getCTime());
    out.writeLong(mostRecentCheckpointTxId);
    out.writeLong(curSegmentTxId);
    imageDigest.write(out);
  }
936399
public static void writeObjectToFile(String path, Object toWrite, Configuration hadoopConfig) throws IOException {
        FileSystem fileSystem = FileSystem.get(hadoopConfig);
        try (BufferedOutputStream bos = new BufferedOutputStream(fileSystem.create(new Path(path)))) {
            ObjectOutputStream oos = new ObjectOutputStream(bos);
            oos.writeObject(toWrite);
        }
    }