Called once at initialization.

@param split          the split that defines the range of records to read
@param imageTransform the image transform to use to transform images while loading them
@throws java.io.IOException
495032
public void initialize(InputSplit split, ImageTransform imageTransform) throws IOException {
        this.imageLoader = null;
        this.imageTransform = imageTransform;
        initialize(split);
    }
550075
protected void splitPixels(int indexStart, int length) {
		// too short to split
		if( length < minimumSideLengthPixel)
			return;

		// end points of the line
		int indexEnd = (indexStart+length)%N;

		int splitOffset = selectSplitOffset(indexStart,length);

		if( splitOffset >= 0 ) {
//			System.out.println("  splitting ");
			splitPixels(indexStart, splitOffset);
			int indexSplit = (indexStart+splitOffset)%N;
			splits.add(indexSplit);
			splitPixels(indexSplit, circularDistance(indexSplit, indexEnd));
		}
	}
415457
protected boolean splitSegments() {
		boolean change = false;

		work.reset();
		for( int i = 0; i < splits.size-1; i++ ) {
			int start = splits.data[i];
			int end = splits.data[i+1];

			int bestIndex = selectSplitBetween(start, end);
			if( bestIndex >= 0 ) {
				change |= true;
				work.add(start);
				work.add(bestIndex);
			} else {
				work.add(start);
			}
		}
		work.add(splits.data[ splits.size-1] );

		// swap the two lists
		GrowQueue_I32 tmp = work;
		work = splits;
		splits = tmp;

		return change;
	}
224569
private boolean fillBuffer(int offset) throws IOException {
		int maxReadLength = this.readBuffer.length - offset;
		// special case for reading the whole split.
		if (this.splitLength == FileInputFormat.READ_WHOLE_SPLIT_FLAG) {
			int read = this.stream.read(this.readBuffer, offset, maxReadLength);
			if (read == -1) {
				this.stream.close();
				this.stream = null;
				return false;
			} else {
				this.readPos = offset;
				this.limit = read;
				return true;
			}
		}
		
		// else ..
		int toRead;
		if (this.splitLength > 0) {
			// if we have more data, read that
			toRead = this.splitLength > maxReadLength ? maxReadLength : (int) this.splitLength;
		}
		else {
			// if we have exhausted our split, we need to complete the current record, or read one
			// more across the next split.
			// the reason is that the next split will skip over the beginning until it finds the first
			// delimiter, discarding it as an incomplete chunk of data that belongs to the last record in the
			// previous split.
			toRead = maxReadLength;
			this.overLimit = true;
		}

		int read = this.stream.read(this.readBuffer, offset, toRead);

		if (read == -1) {
			this.stream.close();
			this.stream = null;
			return false;
		} else {
			this.splitLength -= read;
			this.readPos = offset; // position from where to start reading
			this.limit = read + offset; // number of valid bytes in the read buffer
			return true;
		}
	}
237847
public static void split2(InterleavedF32 interleaved , GrayF32 band0 , GrayF32 band1 ) {
		if( interleaved.numBands != 2 )
			throw new IllegalArgumentException("Input interleaved image must have 2 bands");
		InputSanityCheck.checkSameShape(band0, interleaved);
		InputSanityCheck.checkSameShape(band1, interleaved);

		for( int y = 0; y < interleaved.height; y++ ) {

			int indexTran = interleaved.startIndex + y*interleaved.stride;
			int indexReal = band0.startIndex + y*band0.stride;
			int indexImg = band1.startIndex + y*band1.stride;

			for( int x = 0; x < interleaved.width; x++, indexTran += 2 ) {

				band0.data[indexReal++] = interleaved.data[indexTran];
				band1.data[indexImg++] = interleaved.data[indexTran+1];
			}
		}
	}
135021
public static void rotateImage(File input, File output, String format, int degrees) 
	  throws IOException
	  {
	   	   
		  BufferedImage inputImage = ImageIO.read(input);	   

		    Graphics2D g = (Graphics2D) inputImage.getGraphics();
		    g.drawImage(inputImage, 0, 0, null);
	
		    AffineTransform at = new AffineTransform();
	
		    // scale image
		    //at.scale(2.0, 2.0);
	
		    // rotate 45 degrees around image center
		    at.rotate(degrees * Math.PI / 180.0, inputImage.getWidth() / 2.0, inputImage.getHeight() / 2.0);
		    //at.rotate(Math.toRadians(degrees), inputImage.getWidth() / 2.0, inputImage.getHeight() / 2.0);
	
		   
		    /*
		     * translate to make sure the rotation doesn't cut off any image data
		     */
		    AffineTransform translationTransform;
		    translationTransform = findTranslation(at, inputImage);
		    at.preConcatenate(translationTransform);
		   
		    
		    // instantiate and apply transformation filter
		    BufferedImageOp bio = new AffineTransformOp(at, AffineTransformOp.TYPE_BILINEAR);
		    
		    BufferedImage destinationBI = bio.filter(inputImage, null);
	
		    ImageIO.write(destinationBI, format, output);
		    
	  }
644354
@SuppressWarnings("unchecked")  // Generic array assignment
  public void readFields(DataInput in) throws IOException {
    int card = WritableUtils.readVInt(in);
    if (splits == null || splits.length != card) {
      splits = new InputSplit[card];
    }
    Class<? extends InputSplit>[] cls = new Class[card];
    try {
      for (int i = 0; i < card; ++i) {
        cls[i] =
          Class.forName(Text.readString(in)).asSubclass(InputSplit.class);
      }
      for (int i = 0; i < card; ++i) {
        splits[i] = ReflectionUtils.newInstance(cls[i], null);
        splits[i].readFields(in);
      }
    } catch (ClassNotFoundException e) {
      throw (IOException)new IOException("Failed split init").initCause(e);
    }
  }
866567
private void init(final Map<DistributedDataSetPartition, InputSplit[]> splitsPerPartition) {
    final Pair<InputSplit[], DistributedDataSetPartition[]>
                                      splitsAndPartitions = getSplitsAndPartitions(splitsPerPartition);
    final InputSplit[] splits = splitsAndPartitions.getFirst();
    final DistributedDataSetPartition[] partitions = splitsAndPartitions.getSecond();
    Validate.isTrue(splits.length == partitions.length);
    for (int splitNum = 0; splitNum < splits.length; splitNum++) {
      LOG.log(Level.FINE, "Processing split: " + splitNum);
      final InputSplit split = splits[splitNum];
      final NumberedSplit<InputSplit> numberedSplit = new NumberedSplit<>(split, splitNum,
          partitions[splitNum]);
      unallocatedSplits.add(numberedSplit);
      updateLocations(numberedSplit);
    }
    if (LOG.isLoggable(Level.FINE)) {
      for (final Map.Entry<String, BlockingQueue<NumberedSplit<InputSplit>>> locSplit : locationToSplits.entrySet()) {
        LOG.log(Level.FINE, locSplit.getKey() + ": " + locSplit.getValue().toString());
      }
    }
  }
1078110
@Override
	public void open(FileInputSplit split) throws IOException {
		super.open(split);
		LOG.debug("Initialize Bitcoin reader");
		// temporary measure to set buffer size to 1, otherwise we cannot guarantee that checkpointing works
		bbr = new BitcoinBlockReader(this.stream,this.maxSizeBitcoinBlock,1,this.specificMagicArray,this.useDirectBuffer,this.readAuxPOW);
	}
1527
public void reopen(FileInputSplit split, Tuple3<Long, Long, GenericDataType[]> state) throws IOException {
		this.customSchema = state.f2;
		this.open(split);
		this.getOfficeReader().getCurrentParser().setCurrentSheet(state.f0);
		this.getOfficeReader().getCurrentParser().setCurrentRow(state.f1);

	}
384822
@SuppressWarnings("unchecked")
    public <INKEY,INVALUE,OUTKEY,OUTVALUE,
        T extends org.apache.hadoop.mapreduce.InputSplit> 
    void run() throws Exception {
        Configuration conf = job.getConfiguration();
        reporter = new ContentPumpReporter();
        InputFormat<INKEY,INVALUE> inputFormat = 
            (InputFormat<INKEY, INVALUE>)ReflectionUtils.newInstance(
                job.getInputFormatClass(), conf);
        List<InputSplit> splits;
        T[] array;
        try {
            splits = inputFormat.getSplits(job);
            array = (T[])splits.toArray(
                    new org.apache.hadoop.mapreduce.InputSplit[splits.size()]);
            // sort the splits into order based on size, so that the biggest
            // goes first
            Arrays.sort(array, new SplitLengthComparator());
        } catch (Exception ex) {
            if (LOG.isDebugEnabled()) {
                LOG.debug("Error getting input splits: ", ex);
            } else {
                LOG.error("Error getting input splits: ");
                LOG.error(ex.getMessage());
            }
            job.setJobState(JobStatus.State.FAILED);
            return;
        }               
        OutputFormat<OUTKEY, OUTVALUE> outputFormat = 
            (OutputFormat<OUTKEY, OUTVALUE>)ReflectionUtils.newInstance(
                job.getOutputFormatClass(), conf);
        Class<? extends Mapper<?,?,?,?>> mapperClass = job.getMapperClass();
        Mapper<INKEY,INVALUE,OUTKEY,OUTVALUE> mapper = 
            (Mapper<INKEY,INVALUE,OUTKEY,OUTVALUE>)ReflectionUtils.newInstance(
                mapperClass, conf);
        try {
            outputFormat.checkOutputSpecs(job);
        } catch (Exception ex) {         
            if (LOG.isDebugEnabled()) {
                LOG.debug("Error checking output specification: ", ex);
            } else {
                LOG.error("Error checking output specification: ");
                LOG.error(ex.getMessage());
            }
            job.setJobState(JobStatus.State.FAILED);
            return;
        }
        progress = new AtomicInteger[splits.size()];
        for (int i = 0; i < splits.size(); i++) {
            progress[i] = new AtomicInteger();
        }
     
        job.setJobState(JobStatus.State.RUNNING);
        Monitor monitor = new Monitor();
        monitor.start();
        List<Future<Object>> taskList = new ArrayList<Future<Object>>();
        for (int i = 0; i < array.length && !ContentPump.shutdown; i++) {        
            InputSplit split = array[i];
            if (pool != null) {
                LocalMapTask<INKEY, INVALUE, OUTKEY, OUTVALUE> task = 
                    new LocalMapTask<INKEY, INVALUE, OUTKEY, OUTVALUE>(
                        inputFormat, outputFormat, conf, i, split, reporter,
                        progress[i]);
                availableThreads = assignThreads(i, array.length);
                Class<? extends Mapper<?, ?, ?, ?>> runtimeMapperClass = 
                    job.getMapperClass();
                if (availableThreads > 1 && 
                    availableThreads != threadsPerSplit) { 
                	// possible runtime adjustment
                    if (runtimeMapperClass != (Class)MultithreadedMapper.class) {
                	    runtimeMapperClass = (Class<? extends 
                	        Mapper<INKEY, INVALUE, OUTKEY, OUTVALUE>>)
                	        cmd.getRuntimeMapperClass(job, mapperClass, 
                		        threadsPerSplit, availableThreads);
                    }   
                    if (runtimeMapperClass != mapperClass) {
                	    task.setMapperClass(runtimeMapperClass);
                    }
                    if (runtimeMapperClass == (Class)MultithreadedMapper.class) {
                	    task.setThreadCount(availableThreads);
                	    if (LOG.isDebugEnabled()) {
                            LOG.debug("Thread Count for Split#" + i + " : "
                                    + availableThreads);
                        }
                    }
                }
                
                if (runtimeMapperClass == (Class)MultithreadedMapper.class) {
                    synchronized (pool) {
                        taskList.add(pool.submit(task));
                        pool.wait();
                    }
                } else {
                	pool.submit(task);
                }
            } else { // single-threaded
                JobID jid = new JobID();
                TaskID taskId = new TaskID(jid.getJtIdentifier(), jid.getId(), TaskType.MAP, i);
                TaskAttemptID taskAttemptId = new TaskAttemptID(taskId, 0);
                TaskAttemptContext context = 
                    ReflectionUtil.createTaskAttemptContext(conf, taskAttemptId);
                RecordReader<INKEY, INVALUE> reader = 
                    inputFormat.createRecordReader(split, context);
                RecordWriter<OUTKEY, OUTVALUE> writer = 
                    outputFormat.getRecordWriter(context);
                OutputCommitter committer = 
                    outputFormat.getOutputCommitter(context);
                TrackingRecordReader trackingReader = 
                    new TrackingRecordReader(reader, progress[i]);

                Mapper.Context mapperContext = 
                    ReflectionUtil.createMapperContext(mapper, conf, 
                        taskAttemptId, trackingReader, writer, committer, 
                        reporter, split);
                
                trackingReader.initialize(split, mapperContext);
                
                // no thread pool (only 1 thread specified)
                Class<? extends Mapper<?,?,?,?>> mapClass = 
                        job.getMapperClass();
                mapperContext.getConfiguration().setClass(
                   CONF_MAPREDUCE_JOB_MAP_CLASS , mapClass, Mapper.class);
                mapper = (Mapper<INKEY, INVALUE, OUTKEY, OUTVALUE>) 
                    ReflectionUtils.newInstance(mapClass,
                        mapperContext.getConfiguration());
                try {
                    mapper.run(mapperContext);
                } finally {
                    try {
                        trackingReader.close();
                    } catch (Throwable t) {
                        LOG.error("Error closing reader: " + t.getMessage());
                        if (LOG.isDebugEnabled()) {
                            LOG.debug(t);
                        }
                    }
                    try {
                        writer.close(mapperContext);
                    } catch (Throwable t) {
                        LOG.error("Error closing writer: " + t.getMessage());
                        if (LOG.isDebugEnabled()) {
                            LOG.debug(t);
                        }
                    } 
                    try {
                        committer.commitTask(context);
                    } catch (Throwable t) {
                        LOG.error("Error committing task: " + t.getMessage());
                        if (LOG.isDebugEnabled()) {
                            LOG.debug(t);
                        }
                    }
                }
            }
        }
        // wait till all tasks are done
        if (pool != null) {
            pool.shutdown();
            while (!pool.awaitTermination(1, TimeUnit.HOURS));
        } 
        job.setJobState(JobStatus.State.SUCCEEDED);
        monitor.interrupt();
        monitor.join(1000);
        
        // report counters
        Iterator<CounterGroup> groupIt = 
            reporter.counters.iterator();
        while (groupIt.hasNext()) {
            CounterGroup group = groupIt.next();
            LOG.info(group.getDisplayName() + ": ");
            Iterator<Counter> counterIt = group.iterator();
            while (counterIt.hasNext()) {
                Counter counter = counterIt.next();
                LOG.info(counter.getDisplayName() + ": " + 
                                counter.getValue());
            }
        }
        LOG.info("Total execution time: " + 
                 (System.currentTimeMillis() - startTime) / 1000 + " sec");
    }
493827
private void splitAt(long t) {
        if (data.containsKey(t)) return; // already split at this timestamp

        SortedMap<Long, int[]> head = data.headMap(t);

        int v = head.isEmpty() ? 0 : data.get(head.lastKey())[0];
        data.put(t, new int[]{v});
    }
851584
@Override
    public RecordReader createReader(InputSplit split) throws IOException, InterruptedException {
        RecordReader reader = new ListStringRecordReader();
        reader.initialize(split);
        return reader;
    }
891266
public static void splitFile(File file, long[] splitPoints) throws IOException {
        splitFile(file, splitPoints, file.getParent());
    }
56515
public static void splitFile(File file, long[] splitPoints, String storageFolder) throws IOException {
        splitPoints = ArrayUtils.concatArrays(new long[]{0}, splitPoints, new long[]{file.length()});
        String[] fileName = file.getName().split("\\.", 2);
        storageFolder += ValueConsts.SEPARATOR;
        logger.info("start to split file '" + file.getAbsolutePath() + "'");
        for (int i = 0; i < splitPoints.length - 1; i++) {
            long start = splitPoints[i], end = splitPoints[i + 1];
            long length = end - start;
            String filePath = storageFolder + fileName[0] + "_" + (i + 1) + "." + fileName[1];
            createNewFile(filePath);
            while (length > Integer.MAX_VALUE) {
                saveFile(filePath, readFile(file, start, Integer.MAX_VALUE), true);
                start += Integer.MAX_VALUE;
                length -= Integer.MAX_VALUE;
            }
            if (length > 0) {
                saveFile(filePath, readFile(file, start, (int) length), true);
            }
        }
    }
512343
private void split() throws IOException {
        hasSplit = true;
        IntegerBuffer before = new IntegerBuffer();
        IntegerBuffer after = new IntegerBuffer();

        int current;
        boolean ready = false;
        boolean stillNeedToSeeDoctype = true;
        while (!ready && (current = original.read()) != -1) {
            if (Character.isWhitespace((char) current)) {
                before.append(current);
            } else if (current == '<') {
                // could be XML declaration, comment, PI, DOCTYPE
                // or the first element
                int[] elementOrDeclOr = readUntilCloseCharIsReached();
                if (elementOrDeclOr.length > 0) {
                    if (elementOrDeclOr[0] == '?') {
                        // XML declaration or PI
                        before.append('<');
                        before.append(elementOrDeclOr);
                    } else if (elementOrDeclOr[0] != '!') {
                        // first element
                        after.append('<');
                        after.append(elementOrDeclOr);
                        stillNeedToSeeDoctype = false;
                        ready = true;
                    } else {
                        // comment or doctype
                        IntegerBuffer b =
                            new IntegerBuffer(elementOrDeclOr.length);
                        b.append(elementOrDeclOr);
                        if (b.indexOf(DOCTYPE_INTS) == -1) {
                            after.append('<');
                            after.append(elementOrDeclOr);
                        } else {
                            // swallow old declaration
                            stillNeedToSeeDoctype = false;
                        }
                        ready = true;
                    }
                } else {
                    after.append('<');
                    stillNeedToSeeDoctype = false;
                    ready = true;
                }
            } else {
                after.append(current);
                stillNeedToSeeDoctype = false;
                ready = true;
            }
        }

        // need to eliminate original DOCTYPE if it exists
        while (stillNeedToSeeDoctype && (current = original.read()) != -1) {
            if (Character.isWhitespace((char) current)) {
                after.append(current);
            } else if (current == '<') {
                int[] elementOrDeclOr = readUntilCloseCharIsReached();
                if (elementOrDeclOr.length > 0) {
                    if (elementOrDeclOr[0] == '?') {
                        // PI
                        after.append('<');
                        after.append(elementOrDeclOr);
                    } else if (elementOrDeclOr[0] != '!') {
                        // first element
                        after.append('<');
                        after.append(elementOrDeclOr);
                        stillNeedToSeeDoctype = false;
                    } else {
                        // comment or doctype
                        IntegerBuffer b =
                            new IntegerBuffer(elementOrDeclOr.length);
                        b.append(elementOrDeclOr);
                        if (b.indexOf(DOCTYPE_INTS) == -1) {
                            after.append('<');
                            after.append(elementOrDeclOr);
                        } else {
                            // swallow old declaration
                            stillNeedToSeeDoctype = false;
                        }
                    }
                } else {
                    after.append('<');
                    stillNeedToSeeDoctype = false;
                }
            } else {
                after.append(current);
                stillNeedToSeeDoctype = false;
            }
        }

        beforeDoctype = before.size() > 0
            ? new IntBufferReadable(before) : null;
        afterDoctype = after.size() > 0
            ? new IntBufferReadable(after) : null;
    }
1091724
public static void splitLine(final String line, final char delimiter,
      final String[] splitted) {
    int idxComma, idxToken = 0, fromIndex = 0;
    while ((idxComma = line.indexOf(delimiter, fromIndex)) != -1) {
      splitted[idxToken++] = line.substring(fromIndex, idxComma);
      fromIndex = idxComma + 1;
    }
    splitted[idxToken] = line.substring(fromIndex);
  }