Read PIPES using BSD Kqueue
193771
def _read_select_kqueue(k_queue):
        
        npipes = len(NonBlockingStreamReader._streams)
        # Create list of kevent objects
        # pylint: disable=no-member
        kevents = [select.kevent(s.stream.fileno(),
                                 filter=select.KQ_FILTER_READ,
                                 flags=select.KQ_EV_ADD | select.KQ_EV_ENABLE)
                   for s in NonBlockingStreamReader._streams]
        while NonBlockingStreamReader._run_flag:
            events = k_queue.control(kevents, npipes, 0.5)  # Wake up twice in second
            for event in events:
                if event.filter == select.KQ_FILTER_READ:  # pylint: disable=no-member
                    NonBlockingStreamReader._read_fd(event.ident)
            # Check if new pipes added.
            if npipes != len(NonBlockingStreamReader._streams):
                return
1145921
def read_input(buf, has_header = True):
    """"""

    # Use stdin if there is no supplied buffer
    if buf is None:
        buf = sys.stdin

    # Attempt to read a header if necessary
    header = {}
    if has_header:
        # Until we get a blank line, read "attr:val" lines, 
        # setting the values in 'header'
        last_attr = None
        while True:
            line = buf.readline()

            # remove lastcharacter (which is a newline)
            line = line[:-1] 

            # When we encounter a newline, we are done with the header
            if len(line) == 0:
                break

            colon = line.find(':')

            # If we can't find a colon, then it might be that we are
            # on a new line, and it belongs to the previous attribute
            if colon < 0:
                if last_attr:
                    header[last_attr] = header[last_attr] + '\n' + urllib.parse.unquote(line)
                else:
                    continue

            # extract it and set value in settings
            last_attr = attr = line[:colon]
            val  = urllib.parse.unquote(line[colon+1:])
            header[attr] = val

    return buf, header
1002606
def _read_stdin():
    
    line = sys.stdin.readline()
    while line:
        yield line
        line = sys.stdin.readline()
1099262
def fastq_convert_pipe_cl(in_file, data):
    
    cmd = _seqtk_fastq_prep_cl(data, in_file)
    if not cmd:
        cat_cmd = "zcat" if in_file.endswith(".gz") else "cat"
        cmd = cat_cmd + " " + in_file
    return "<(%s)" % cmd
618226
function queue(context, file, fileSet, next) {
  var origin = file.history[0]
  var map = fileSet.complete
  var complete = true

  if (!map) {
    map = {}
    fileSet.complete = map
  }

  debug('Queueing `%s`', origin)

  map[origin] = next

  fileSet.valueOf().forEach(each)

  if (!complete) {
    debug('Not flushing: some files cannot be flushed')
    return
  }

  fileSet.complete = {}

  fileSet.pipeline.run(fileSet, done)

  function each(file) {
    var key = file.history[0]

    if (stats(file).fatal) {
      return
    }

    if (typeof map[key] === 'function') {
      debug('`%s` can be flushed', key)
    } else {
      debug('Interupting flush: `%s` is not finished', key)
      complete = false
    }
  }

  function done(error) {
    debug('Flushing: all files can be flushed')

    /* Flush. */
    for (origin in map) {
      map[origin](error)
    }
  }
}
1111435
def read_from_buffer(cls, buf, identifier_str=None):
        """"""
        try:
            return cls._read_from_buffer(buf, identifier_str)
        except Exception as e:
            cls._load_error(e, identifier_str)
1130463
def readlines(self):
      """"""

      # If the file is not open, there's nothing to return
      if not self._fh:
         raise StopIteration

      at_eof = False
      while True:
         # Clean the buffer sometimes.
         if self._bufoffset > (self._maxreadsize / 2):
            self._buf = self._buf[self._bufoffset:]
            self._bufoffset = 0

         # Fill up the buffer if necessary.
         if len(self._buf) < self._maxreadsize:
            at_eof = not self._read(self._maxreadsize)

         # Look for the next line.
         try:
            next_newline = self._buf.index("\n", self._bufoffset)
            line = self._buf[self._bufoffset:next_newline]
            self._bufoffset = next_newline + 1
            # Save the current file offset for yielding and advance the file offset.
            offset = self._offset
            self._offset += len(line) + 1
            if self._longline:
               # This is the remaining chunk of a long line, we're not going
               # to yield it.
               self._longline = False
            else:
               yield line, offset

         except ValueError:
            # Reached the end of the buffer without finding any newlines.
            if not at_eof:
               # Line is longer than the half the buffer size? - Nope
               logger.warning("Skipping over longline at %s:%d", self._path,
                                                                 self._offset)
               self._bufoffset = len(self._buf) - 1
               self._longline = True
            raise StopIteration
2745
def read_pipe(pipe_out):
    
    out = b''
    while more_data(pipe_out):
        out += os.read(pipe_out, 1024)

    return out.decode('utf-8')
75283
def run(stream=sys.stdin.buffer):
    
    payload = decode.decode(stream)
    sys.stdout.write(payload.decode('utf8'))
1338841
def bwasw(args, opts):
    
    dbfile, readfile = args
    dbfile = check_index(dbfile)

    samfile, _, unmapped = get_samfile(readfile, dbfile,
                                       bam=opts.bam, unmapped=opts.unmapped)
    if not need_update(dbfile, samfile):
        logging.error("`{0}` exists. `bwa bwasw` already run.".format(samfile))
        return "", samfile

    cmd = "bwa bwasw " + " ".join(args)
    cmd += " -t {0}".format(opts.cpus)
    cmd += " " + opts.extra
    return cmd, samfile
631713
def read_message_bytes_from_pipe(pipe_handle):
    
    overlapped = OVERLAPPED()
    overlapped.hEvent = create_event()

    try:
        buff = create_string_buffer(BUFSIZE + 1)
        c_read = DWORD()

        success = windll.kernel32.ReadFile(
            pipe_handle,
            buff,
            DWORD(BUFSIZE),
            byref(c_read),
            byref(overlapped))

        if success:
            buff[c_read.value] = b'\0'
            raise Return(buff.value)

        error_code = windll.kernel32.GetLastError()

        if error_code == ERROR_IO_PENDING:
            yield From(wait_for_event(overlapped.hEvent))

            success = windll.kernel32.GetOverlappedResult(
                pipe_handle,
                byref(overlapped),
                byref(c_read),
                BOOL(False))

            if success:
                buff[c_read.value] = b'\0'
                raise Return(buff.value)

            else:
                error_code = windll.kernel32.GetLastError()
                if error_code == ERROR_BROKEN_PIPE:
                    raise BrokenPipeError

                elif error_code == ERROR_MORE_DATA:
                    more_data = yield From(read_message_bytes_from_pipe(pipe_handle))
                    raise Return(buff.value + more_data)
                else:
                    raise Exception(
                        'reading overlapped IO failed. error_code=%r' % error_code)

        elif error_code == ERROR_BROKEN_PIPE:
            raise BrokenPipeError

        elif error_code == ERROR_MORE_DATA:
            more_data = yield From(read_message_bytes_from_pipe(pipe_handle))
            raise Return(buff.value + more_data)

        else:
            raise Exception('Reading pipe failed, error_code=%s' % error_code)
    finally:
        windll.kernel32.CloseHandle(overlapped.hEvent)
1180584
function fsSize(callback) {

  return new Promise((resolve) => {
    process.nextTick(() => {
      let data = [];
      if (_linux || _freebsd || _openbsd || _darwin) {
        let cmd = '';
        if (_darwin) cmd = 'df -lkP | grep ^/';
        if (_linux) cmd = 'df -lkPT | grep ^/';
        if (_freebsd || _openbsd) cmd = 'df -lkPT';
        exec(cmd, function (error, stdout) {
          if (!error) {
            let lines = stdout.toString().split('\n');
            //lines.splice(0, 1);
            lines.forEach(function (line) {
              if (line !== '') {
                line = line.replace(/ +/g, ' ').split(' ');
                if (line && (line[0].startsWith('/')) || (line[6] && line[6] === '/')) {
                  data.push({
                    'fs': line[0],
                    'type': ((_linux || _freebsd || _openbsd) ? line[1] : 'HFS'),
                    'size': parseInt(((_linux || _freebsd || _openbsd) ? line[2] : line[1])) * 1024,
                    'used': parseInt(((_linux || _freebsd || _openbsd) ? line[3] : line[2])) * 1024,
                    'use': parseFloat((100.0 * ((_linux || _freebsd || _openbsd) ? line[3] : line[2]) / ((_linux || _freebsd || _openbsd) ? line[2] : line[1])).toFixed(2)),
                    'mount': line[line.length - 1]
                  });
                }
              }
            });
          }
          if (callback) {
            callback(data);
          }
          resolve(data);
        });
      }
      if (_sunos) {
        if (callback) { callback(data); }
        resolve(data);
      }
      if (_windows) {
        try {
          util.wmic('logicaldisk get Caption,FileSystem,FreeSpace,Size').then((stdout, error) => {
            let lines = stdout.split('\r\n').filter(line => line.trim() !== '').filter((line, idx) => idx > 0);
            lines.forEach(function (line) {
              if (line !== '') {
                line = line.trim().split(/\s\s+/);
                data.push({
                  'fs': line[0],
                  'type': line[1],
                  'size': line[3],
                  'used': parseInt(line[3]) - parseInt(line[2]),
                  'use': parseFloat((100.0 * (parseInt(line[3]) - parseInt(line[2]))) / parseInt(line[3])),
                  'mount': line[0]
                });
              }
            });
            if (callback) {
              callback(data);
            }
            resolve(data);
          });
        } catch (e) {
          if (callback) { callback(data); }
          resolve(data);
        }
      }
    });
  });
}
966991
def main(args):

  
  sys.stderr.write("reading reference genome\n")
  ref = FASTAData(open(args.genome).read())
  sys.stderr.write("read in "+str(len(ref.keys()))+" chromosomes\n")

  """ Next make the transcriptome """
  txome = {}
  sys.stderr.write("write the transcriptome\n")
  inf = None
  if args.gpd[-3:] == '.gz':
    inf = gzip.open(args.gpd)
  else:
    inf = open(args.gpd)
  stream = GPDStream(inf)
  tof = open(args.tempdir+'/transcriptome.fa','w')
  z = 0
  for gpd in stream:
    z += 1
    if gpd.transcript_name in txome:
      sys.stderr.write("WARNING already have a transcript "+gpd.transcript_name+" ignoring line "+str(z)+" of the gpd\n")
      continue
    txome[gpd.transcript_name] = gpd.gene_name
    tof.write('>'+gpd.transcript_name+"\n"+str(gpd.get_sequence(ref))+"\n")
  tof.close()
  inf.close()
  sys.stderr.write("wrote "+str(len(txome.keys()))+" transcripts\n")

  """Build the salmon index"""
  sys.stderr.write("building a salmon index\n")
  cmd = 'salmon index -p '+str(args.numThreads)+' -t '+args.tempdir+'/transcriptome.fa -i '+args.tempdir+'/salmon_index'
  p = Popen(cmd.split())
  p.communicate()
  sys.stderr.write("finished building the index\n")

  """Use the index to quanitfy"""
  sys.stderr.write("quanitfy reads\n")
  reads = ''
  if args.rU:
    reads = '-r '+args.rU
  else:
    reads = '-1 '+args.r1+' -2 '+args.r2
  cmd = 'salmon quant -p '+str(args.numThreads)+' -i '+args.tempdir+'/salmon_index -l A '+reads+' -o '+args.tempdir+'/output_quant'
  p = Popen(cmd.split())
  p.communicate()
  sys.stderr.write("finished quanitfying\n")

  """Now parse the salmon output to add gene name"""
  salmon = {}
  with open(args.tempdir+'/output_quant/quant.sf') as inf:
    header = inf.readline()
    for line in inf:
      f = line.rstrip().split("\t")
      # by each transcript name hold a data strcture of the information
      salmon[f[0]] = {'name':f[0],'length':int(f[1]),'EffectiveLength':float(f[2]),'TPM':float(f[3]),'NumReads':float(f[4])}
  genes = {}
  for name in salmon:
    gene = txome[name]
    if gene not in genes: genes[gene] = []
    genes[gene].append(salmon[name])
  genetot = {}
  for gene in genes:
    tot = sum([x['TPM'] for x in genes[gene]])
    genetot[gene] = tot
  ordered_gene_names = sorted(genetot.keys(), key=lambda x: genetot[x],reverse=True)
      
  """Collected enough information to make output"""
  sys.stderr.write("generating output\n")
  of = sys.stdout
  if args.output != '-':
    of = open(args.output,'w')
  of.write("geneName\ttranscriptName\tlength\tEffectiveLength\ttxTPM\tNumReads\tgeneTPM\n")
  for gene in ordered_gene_names:
    txs = sorted(genes[gene],key=lambda x: x['TPM'],reverse=True)
    for tx in txs:
      of.write(gene+"\t"+tx['name']+"\t"+str(tx['length'])+"\t"+str(tx['EffectiveLength'])+"\t"+str(tx['TPM'])+"\t"+str(tx['NumReads'])+"\t"+str(genetot[gene])+"\n")
  of.close()

  # Temporary working directory step 3 of 3 - Cleanup
  if not args.specific_tempdir:
    rmtree(args.tempdir)
1264411
public static void pipe(Reader reader, Writer writer, int buffersize) throws IOException {
        char[] buffer = new char[buffersize];
        while (reader.read(buffer) != -1) {
            writer.write(buffer);
        }
    }
322312
def pipe_wait(popens):
    
    # Avoid mutating the passed copy
    popens = copy.copy(popens)
    results = [0] * len(popens)
    while popens:
        last = popens.pop(-1)
        results[len(popens)] = last.wait()
    return results
1244576
def _AsynchronouslyGetProcessOutput(formattedCmd, printStdOut, printStdErr, **kwargs):
    '''  '''
    opts = filterKWArgsForFunc(kwargs, subprocess.Popen)
    opts['stdout'] = subprocess.PIPE
    opts['stderr'] = subprocess.PIPE
    process = subprocess.Popen(formattedCmd, **opts)

    # Launch the asynchronous readers of the process' stdout and stderr.
    stdout_queue = Queue.Queue()
    stdout_reader = _AsynchronousFileReader(process.stdout, stdout_queue)
    stdout_reader.start()
    stderr_queue = Queue.Queue()
    stderr_reader = _AsynchronousFileReader(process.stderr, stderr_queue)
    stderr_reader.start()

    stdOutLines = []
    stdErrLines = []
    # Check the queues if we received some output (until there is nothing more to get).
    while not stdout_reader.eof() or not stderr_reader.eof():
        # Show what we received from standard output.
        while not stdout_queue.empty():
            line = stdout_queue.get()
            stdOutLines.append(line)
            if printStdOut:
                print line.rstrip()
 
        # Show what we received from standard error.
        while not stderr_queue.empty():
            line = stderr_queue.get()
            stdErrLines.append(line)
            if printStdErr:
                print colored(line.rstrip(),'red')
 
        # Sleep a bit before asking the readers again.
        time.sleep(.05)
 
    # Let's be tidy and join the threads we've started.
    stdout_reader.join()
    stderr_reader.join()
 
    # Close subprocess' file descriptors.
    process.stdout.close()
    process.stderr.close()
    process.wait()

    stdOut = ''.join(stdOutLines)
    stdErr = ''.join(stdErrLines)
    return (process.returncode, stdOut, stdErr)
796347
function (pipe) {
        var tasks = [];

        _.forEach(plumber.sieve(pipe.content, 'statics'), function (file, key) {
            tasks.push(glob('**/*.*', {cwd: path.resolve(pipe.contentDir, key)})
                .then(function (files) {
                    pipe.content[key].files = files;
                }));
        });

        return Bluebird.all(tasks).return(pipe);
    }