Iterate over a fileobj and yields on each mpeg sync.

    When yielding the fileobj offset is right before the sync and can be
    changed between iterations without affecting the iteration process.

    Might raise IOError.
965484
def iter_sync(fileobj, max_read):
    

    read = 0
    size = 2
    last_byte = b""
    is_second = lambda b: ord(b) & 0xe0 == 0xe0

    while read < max_read:
        data_offset = fileobj.tell()
        new_data = fileobj.read(min(max_read - read, size))
        if not new_data:
            return
        read += len(new_data)

        if last_byte == b"\xff" and is_second(new_data[0:1]):
            fileobj.seek(data_offset - 1, 0)
            yield

        size *= 2
        last_byte = new_data[-1:]

        find_offset = 0
        while True:
            index = new_data.find(b"\xff", find_offset)
            # if not found or the last byte -> read more
            if index == -1 or index == len(new_data) - 1:
                break

            if is_second(new_data[index + 1:index + 2]):
                fileobj.seek(data_offset + index, 0)
                yield
            find_offset = index + 1

        fileobj.seek(data_offset + len(new_data), 0)
97714
def _chunk_iter(self):
        """"""
        for chunk_offset in self._chunk_offsets():
            yield self._download_chunk(chunk_offset=chunk_offset,
                                       chunk_size=self._chunk_size)
1093415
def tracebacks_from_file(fileobj, reverse=False):
    

    if reverse:
        lines = deque()

        for line in BackwardsReader(fileobj):
            lines.appendleft(line)
            if tb_head in line:
                yield next(tracebacks_from_lines(lines))
                lines.clear()
    else:
        for traceback in tracebacks_from_lines(fileobj):
            yield traceback
562753
function (files, callback) {
		//loop on all files and write them
		async.each(files, function (fileObj, mCb) {
			var data = swagger.cloneObj(fileObj.data);
			
			//if tokens, replace all occurences with corresponding values
			if (fileObj.tokens) {
				for (var i in fileObj.tokens) {
					var regexp = new RegExp("%" + i + "%", "g");
					data = data.replace(regexp, fileObj.tokens[i]);
				}
			}
			if (fileObj.purify) {
				data = data.replace(/\\"/g, '"').replace(/["]+/g, '"').replace(/"__dirname/g, '__dirname');
				data = data.replace(/("group": "__empty__")/g, '"group": ""');
				data = data.replace(/("prefix": "(\s?|\s+),)/g, '"prefix": "",');
				data = data.replace(/("l": "__empty__")/g, '"l": ""');
				//"__dirname + \"/lib/mw/_get.js\""
				//"__dirname + "/lib/mw/_get.js""
				//"__dirname + "/lib/mw/_get.js"
				//__dirname + "/lib/mw/_get.js"
			}
			console.log("creating file:", fileObj.file);
			fs.writeFile(fileObj.file, data, "utf8", mCb);
		}, function (error) {
			if (error) {
				return callback({"code": 854, "msg": error.message});
			}
			return callback(null, true);
		});
	}
283393
public static long syncFile(InputStream in, RandomAccessFile out) throws IOException {
		byte[] inBuff = new byte[BLOCK_SIZE];
		byte[] outBuff = new byte[BLOCK_SIZE];
		long pos = 0;
		long bytesWritten = 0;
		int numBytes;
		while((numBytes=in.read(inBuff, 0, BLOCK_SIZE))!=-1) {
			if(DEBUG) System.err.println(pos+": Read " + numBytes + " bytes of input");
			out.seek(pos);
			long blockEnd = pos + numBytes;
			if(out.length()>=blockEnd) {
				// Read block from out
				if(DEBUG) System.err.println(pos+": Reading " + numBytes + " bytes of output");
				out.readFully(outBuff, 0, numBytes);
				if(!AoArrays.equals(inBuff, outBuff, 0, numBytes)) {
					if(DEBUG) System.err.println(pos+": Updating " + numBytes + " bytes of output");
					out.seek(pos);
					if(!DRY_RUN) out.write(inBuff, 0, numBytes);
					bytesWritten += numBytes;
				} else {
					if(DEBUG) System.err.println(pos+": Data matches, not writing");
				}
			} else {
				// At end, write entire block
				if(DEBUG) System.err.println(pos+": Appending " + numBytes +" bytes to output");
				if(!DRY_RUN) out.write(inBuff, 0, numBytes);
				bytesWritten += numBytes;
			}
			pos = blockEnd;
		}
		if(out.length()!=pos) {
			if(DEBUG) System.err.println(pos+": Truncating output to " + pos + " bytes");
			assert out.length()>pos;
			if(!DRY_RUN) {
				try {
					out.setLength(pos);
				} catch(IOException e) {
					System.err.println("Warning: Unable to truncate output to " + pos +" bytes");
				}
			}
		}
		return bytesWritten;
	}
357370
def iter_encode(self, boundary, blocksize=4096):
        """"""
        total = self.get_size(boundary)
        current = 0
        if self.value is not None:
            block = self.encode(boundary)
            current += len(block)
            yield block
            if self.cb:
                self.cb(self, current, total)
        else:
            block = self.encode_hdr(boundary)
            current += len(block)
            yield block
            if self.cb:
                self.cb(self, current, total)
            last_block = ""
            encoded_boundary = "--%s" % encode_and_quote(boundary)
            boundary_exp = re.compile("^%s$" % re.escape(encoded_boundary),
                                      re.M)
            while True:
                block = self.fileobj.read(blocksize)
                if not block:
                    current += 2
                    yield "\r\n"
                    if self.cb:
                        self.cb(self, current, total)
                    break
                last_block += block
                if boundary_exp.search(last_block):
                    raise ValueError("boundary found in file data")
                last_block = last_block[-len(encoded_boundary) - 2:]
                current += len(block)
                yield block
                if self.cb:
                    self.cb(self, current, total)
609230
public synchronized ByteBuffer getRecordBuffer(int index) throws IOException {
    if (index < 0 || index >= numrecs) {
      throw new IOException("Access beyond end of file.");
    }
    // Adjust buffer view
    synchronized (map) {
      map.limit(recordsize * (index + 1));
      map.position(recordsize * index);
      return map.slice();
    }
  }
508614
def iter_chunks_class(self):
        
        for m in self.get_metadata():
            try:
                yield self.chunkclass(self.get_chunk(m.x, m.z))
            except RegionFileFormatError:
                pass
1119947
def get_binary_iterator(self):
        
        CHUNK_SIZE = 1024
        file_object = open(self.path)

        while True:
            data = file_object.read(CHUNK_SIZE)
            if not data:
                break
            yield data

        file_object.close()
546562
def wrap_with_rewind *io, &block
      io.each do |i|
        i.rewind
      end
      yield
      io.each do |i|
        i.rewind
      end
    end
195146
def readlines(self, offset=0):
        """"""
        try:
            with open(self._filepath) as fp:
                # For full read go through existing lines in file
                if self._full_read:
                    fp.seek(offset)
                    for row in fp:
                        yield row

                # Okay now dynamically tail the file
                if self._tail:
                    while True:
                        current = fp.tell()
                        row = fp.readline()
                        if row:
                            yield row
                        else:
                            fp.seek(current)
                            time.sleep(self._sleep)

        except IOError as err:
            print('Error reading the file {0}: {1}'.format(self._filepath, err))
            return
164434
synchronized void sync()
    {
        try
        {
            // check we have more work to do
            if (allocatePosition.get() <= lastSyncedOffset + SYNC_MARKER_SIZE)
                return;

            // allocate a new sync marker; this is both necessary in itself, but also serves to demarcate
            // the point at which we can safely consider records to have been completely written to
            int nextMarker;
            nextMarker = allocate(SYNC_MARKER_SIZE);
            boolean close = false;
            if (nextMarker < 0)
            {
                // ensure no more of this CLS is writeable, and mark ourselves for closing
                discardUnusedTail();
                close = true;

                // wait for modifications guards both discardedTailFrom, and any outstanding appends
                waitForModifications();

                if (discardedTailFrom < buffer.capacity() - SYNC_MARKER_SIZE)
                {
                    // if there's room in the discard section to write an empty header, use that as the nextMarker
                    nextMarker = discardedTailFrom;
                }
                else
                {
                    // not enough space left in the buffer, so mark the next sync marker as the EOF position
                    nextMarker = buffer.capacity();
                }
            }
            else
            {
                waitForModifications();
            }

            assert nextMarker > lastSyncedOffset;

            // write previous sync marker to point to next sync marker
            // we don't chain the crcs here to ensure this method is idempotent if it fails
            int offset = lastSyncedOffset;
            final PureJavaCrc32 crc = new PureJavaCrc32();
            crc.updateInt((int) (id & 0xFFFFFFFFL));
            crc.updateInt((int) (id >>> 32));
            crc.updateInt(offset);
            buffer.putInt(offset, nextMarker);
            buffer.putInt(offset + 4, crc.getCrc());

            // zero out the next sync marker so replayer can cleanly exit
            if (nextMarker < buffer.capacity())
            {
                buffer.putInt(nextMarker, 0);
                buffer.putInt(nextMarker + 4, 0);
            }

            // actually perform the sync and signal those waiting for it
            buffer.force();

            if (close)
                nextMarker = buffer.capacity();

            lastSyncedOffset = nextMarker;
            syncComplete.signalAll();

            CLibrary.trySkipCache(fd, offset, nextMarker);
            if (close)
                internalClose();
        }
        catch (Exception e) // MappedByteBuffer.force() does not declare IOException but can actually throw it
        {
            throw new FSWriteError(e, getPath());
        }
    }
557434
public void reprocessInput() {
		if ( inputMethod == InputMethod.VIDEO ) {
			openVideo(true,inputFilePath);
		} else if( inputMethod == InputMethod.IMAGE ) {
			BufferedImage buff = inputStreams.get(0).getBufferedImage();
			openImage(true,new File(inputFilePath).getName(),buff);// TODO still does a pointless image conversion
		} else if( inputMethod == InputMethod.IMAGE_SET ) {
			openImageSet(inputFileSet);
		}
	}
538254
function iterateRecurse (dir, options, cb) {
  try {
    var files = fs.readdirSync(dir)
    files.forEach(function (file, index) {
      var parsedPath = path.parse(file)
      var name = parsedPath.name
      var ext = parsedPath.ext
      var srcpath = path.join(dir, file)

      if (options.skip_ && name.charAt(0) === '_') {
        logger.debug('Skipping file/folder: ' + srcpath)
      } else {
        if (fs.lstatSync(srcpath).isDirectory()) {
          if (options.dirs) {
            cb(srcpath, name)
          }

          // Do recursive search when needed
          if (options.recursive) {
            iterateRecurse(srcpath, options, cb)
          }
        } else {
          for (var i = 0; i < options.ext.length; i++) {
            if (ext === options.ext[i]) {
              cb(srcpath, name, ext)
            }
          }
        }
      }
    })
  } catch (e) {
    logger.debug('Error iterating files: ' + e)
  }
}
599735
def BytesIO(*args, **kwargs):
    """"""
    raw = sync_io.BytesIO(*args, **kwargs)
    return AsyncBytesIOWrapper(raw)
958333
def copyfileobj(src, dst, length=None):
    
    if length == 0:
        return
    if length is None:
        while True:
            buf = src.read(16*1024)
            if not buf:
                break
            dst.write(buf)
        return

    BUFSIZE = 16 * 1024
    blocks, remainder = divmod(length, BUFSIZE)
    for b in range(blocks):
        buf = src.read(BUFSIZE)
        if len(buf) < BUFSIZE:
            raise IOError("end of file reached")
        dst.write(buf)

    if remainder != 0:
        buf = src.read(remainder)
        if len(buf) < remainder:
            raise IOError("end of file reached")
        dst.write(buf)
    return
882400
public boolean processFile(Path srcPath, ParityFilePair parityPair,
      Decoder decoder, Boolean fromStripeStore, Context context) 
          throws IOException,
      InterruptedException {
    LOG.info("Processing file " + srcPath);
    Progressable progress = context;
    if (progress == null) {
      progress = RaidUtils.NULL_PROGRESSABLE;
    }
    
    DistributedFileSystem srcFs = getDFS(srcPath);
    FileStatus srcStat = srcFs.getFileStatus(srcPath);
    long blockSize = srcStat.getBlockSize();
    long srcFileSize = srcStat.getLen();
    String uriPath = srcPath.toUri().getPath();

    int numBlocksReconstructed = 0;
    List<LocatedBlockWithMetaInfo> lostBlocks = lostBlocksInFile(srcFs,
        uriPath, srcStat);
    if (lostBlocks.size() == 0) {
      LOG.warn("Couldn't find any lost blocks in file " + srcPath + 
          ", ignoring...");
      return false;
    }
    List<Block> blocksLostChecksum = new ArrayList<Block>();
    List<Block> blocksLostStripe = new ArrayList<Block>();
    
    for (LocatedBlockWithMetaInfo lb: lostBlocks) {
      Block lostBlock = lb.getBlock();
      long lostBlockOffset = lb.getStartOffset();

      LOG.info("Found lost block " + lostBlock +
          ", offset " + lostBlockOffset);
      Long oldCRC = decoder.retrieveChecksum(lostBlock, 
          srcPath, lostBlockOffset, srcFs, context);
      if (abortReconstruction(oldCRC, decoder)) {
        blocksLostChecksum.add(lostBlock);
        continue;
      }
      StripeInfo si = decoder.retrieveStripe(lostBlock, 
          srcPath, lostBlockOffset, srcFs, context, false);
      if (si == null && decoder.stripeStore != null) {
        blocksLostStripe.add(lostBlock);
        continue;
      }

      final long blockContentsSize =
        Math.min(blockSize, srcFileSize - lostBlockOffset);
      File localBlockFile =
        File.createTempFile(lostBlock.getBlockName(), ".tmp");
      localBlockFile.deleteOnExit();

      try {
        CRC32 crc = null;
        
        if (fromStripeStore) {
          crc = decoder.recoverBlockToFileFromStripeInfo(srcFs, srcPath,
              lostBlock, localBlockFile, blockSize,
              lostBlockOffset, blockContentsSize, si, context);
        } else {
          crc = decoder.recoverBlockToFile(srcFs, srcStat,
                        parityPair.getFileSystem(),
                        parityPair.getPath(), blockSize,
                        lostBlockOffset, localBlockFile,
                        blockContentsSize, si, context);
        }
        LOG.info("Recovered crc: " + ((crc == null)?null: crc.getValue()) +
            " expected crc:" + oldCRC);
        if (crc != null && oldCRC != null &&
            crc.getValue() != oldCRC) {
          // checksum doesn't match, it's dangerous to send it
          IOException ioe = new IOException("Block " + lostBlock.toString() +
              " new checksum " + crc.getValue() +
              " doesn't match the old one " + oldCRC);
          LogUtils.logRaidReconstructionMetrics(LOGRESULTS.FAILURE, 0,
              decoder.codec, srcPath, lostBlockOffset,
              LOGTYPES.OFFLINE_RECONSTRUCTION_CHECKSUM_VERIFICATION,
              srcFs, ioe, context);
          throw ioe;
        }
        // Now that we have recovered the file block locally, send it.
        computeMetadataAndSendReconstructedBlock(localBlockFile,
            lostBlock, blockContentsSize,
            lb.getLocations(),
            lb.getDataProtocolVersion(), 
            lb.getNamespaceID(), 
            progress);
        
        numBlocksReconstructed++;

      } finally {
        localBlockFile.delete();
      }
      progress.progress();
    }
    
    LOG.info("Reconstructed " + numBlocksReconstructed + " blocks in " + srcPath);
    checkLostBlocks(blocksLostChecksum, blocksLostStripe, srcPath, decoder.codec);
    return true;
  }