Returns the next previous chunk in the sentence with the given type.
973632
def previous(self, type=None):
        
        i = self.start - 1
        s = self.sentence
        while i > 0:
            if s[i].chunk is not None and type in (s[i].chunk.type, None):
                return s[i].chunk
            i -= 1
632370
def next(self):
        
        if self.current_token == len(self.tokens):
            return None
        token = self.tokens[self.current_token]
        if token["type"] == "newline":
            self.line += 1
            self.line_start = token["start"]
        self.current_token += 1
        if token["type"] == "unknown":
            self.error("Unknown token")
        return token
410276
def next(self):
        
        if self._has_finished:
            raise StopIteration

        if not len(self._buffer):
                    
            results = self.fetch_next_block()
            self._buffer.extend(results)
            
        if not len(self._buffer):
            raise StopIteration
            
        return self._buffer.popleft()
1040991
def _next(self):
        
        c = self._get()
        if c == '/':
            p = self._peek()
            if p == '/':
                c = self._get()
                while c > '\n':
                    c = self._get()
                return c
            if p == '*':
                c = self._get()
                while 1:
                    c = self._get()
                    if c == '*':
                        if self._peek() == '/':
                            self._get()
                            return ' '
                    if c == '\000':
                        raise UnterminatedComment()

        return c
587417
def next (self):
        
        header, packet = self.read()

        if packet is None:
            raise StopIteration

        return header, packet
1218767
def end_of_chunk(prev_tag, tag, prev_type, type_):
    
    chunk_end = False

    if prev_tag == 'E': chunk_end = True
    if prev_tag == 'S': chunk_end = True

    if prev_tag == 'B' and tag == 'B': chunk_end = True
    if prev_tag == 'B' and tag == 'S': chunk_end = True
    if prev_tag == 'B' and tag == 'O': chunk_end = True
    if prev_tag == 'I' and tag == 'B': chunk_end = True
    if prev_tag == 'I' and tag == 'S': chunk_end = True
    if prev_tag == 'I' and tag == 'O': chunk_end = True

    if prev_tag != 'O' and prev_tag != '.' and prev_type != type_:
        chunk_end = True

    return chunk_end
1266254
def this_and_prev(iterable):
    """"""
    try:
        item = next(iterable)
        while True:
            next_item = next(iterable)
            yield item, next_item
            item = next_item
    except StopIteration:
        return
195458
def next_chunk(self):
        
        if self._unconsumed:
            data = self._unconsumed.pop()
        else:
            data = self._iterator.next()  # Might raise StopIteration
        self._pos += len(data)
        return data
708692
private LogChunkType readNextChunk()
    {
        // clear the current chunk
        currentChunk = Slices.EMPTY_SLICE;

        // read the next block if necessary
        if (currentBlock.available() < HEADER_SIZE) {
            if (!readNextBlock()) {
                if (eof) {
                    return EOF;
                }
            }
        }

        // parse header
        int expectedChecksum = currentBlock.readInt();
        int length = currentBlock.readUnsignedByte();
        length = length | currentBlock.readUnsignedByte() << 8;
        byte chunkTypeId = currentBlock.readByte();
        LogChunkType chunkType = getLogChunkTypeByPersistentId(chunkTypeId);

        // verify length
        if (length > currentBlock.available()) {
            int dropSize = currentBlock.available() + HEADER_SIZE;
            reportCorruption(dropSize, "Invalid chunk length");
            currentBlock = Slices.EMPTY_SLICE.input();
            return BAD_CHUNK;
        }

        // skip zero length records
        if (chunkType == ZERO_TYPE && length == 0) {
            // Skip zero length record without reporting any drops since
            // such records are produced by the writing code.
            currentBlock = Slices.EMPTY_SLICE.input();
            return BAD_CHUNK;
        }

        // Skip physical record that started before initialOffset
        if (endOfBufferOffset - HEADER_SIZE - length < initialOffset) {
            currentBlock.skipBytes(length);
            return BAD_CHUNK;
        }

        // read the chunk
        currentChunk = currentBlock.readBytes(length);

        if (verifyChecksums) {
            int actualChecksum = getChunkChecksum(chunkTypeId, currentChunk);
            if (actualChecksum != expectedChecksum) {
                // Drop the rest of the buffer since "length" itself may have
                // been corrupted and if we trust it, we could find some
                // fragment of a real log record that just happens to look
                // like a valid log record.
                int dropSize = currentBlock.available() + HEADER_SIZE;
                currentBlock = Slices.EMPTY_SLICE.input();
                reportCorruption(dropSize, "Invalid chunk checksum");
                return BAD_CHUNK;
            }
        }

        // Skip unknown chunk types
        // Since this comes last so we the, know it is a valid chunk, and is just a type we don't understand
        if (chunkType == UNKNOWN) {
            reportCorruption(length, String.format("Unknown chunk type %d", chunkType.getPersistentId()));
            return BAD_CHUNK;
        }

        return chunkType;
    }
158199
public String nextParagraph(int totalSentences) {
        StringBuilder out = new StringBuilder();

        List<String> lastWords = new ArrayList<String>();
        lastWords.add(null);
        lastWords.add(null);
        int numSentences = 0;
        boolean inSentence = false;
        boolean inQuote = false;
        while(numSentences < totalSentences) {
            List<String> words = pairTable.get(lastWords);
            if(words == null) {
                // no hit for the digram, try just the last word
                words = singleWordTable.get(lastWords.get(1));
            }
            if(words == null) {
                // hit end of paragraph pair, nothing directly following. start again
                words = pairTable.get(Arrays.<String>asList(null, null));
            }
            String nextWord = words.get(random.nextInt(words.size()));
            if(nextWord.length() == 1 && sentenceEndChars.indexOf(nextWord.charAt(0)) != -1) {
                out.append(nextWord);
                if(inQuote) {
                    out.append('"');
                }
                numSentences++;
                inSentence = false;
                inQuote = false;
                lastWords.remove(0);
                lastWords.add(null); // look up
            } else {
                if(!inSentence) {
                    // start a new sentence
                    if(out.length() > 0) {
                        out.append(" ");
                    }
                    inSentence = true;
                } else {
                    // need a word separator
                    out.append(" ");
                }
                out.append(nextWord);
                if(nextWord.indexOf('"') != -1) {
                    inQuote = true;
                }
            }
            lastWords.remove(0);
            lastWords.add(nextWord);

        }
        return out.toString();
    }
1323086
def get_chunks(self,chunk_type):
        
        for nonter,this_type in self.label_for_nonter.items():
            if this_type == chunk_type:
                subsumed = self.terms_subsumed_by_nonter.get(nonter)
                if subsumed is not None:
                    yield sorted(list(subsumed))
615294
def chunks(self):
        
        if not hasattr(self, '_it'):
            class ChunkIterator(object):
                def __iter__(iter):
                    return iter

                def __next__(iter):
                    try:
                        chunk = self._next_chunk()
                    except StopIteration:
                        if self.loop:
                            self._init_stretching()
                            return iter.__next__()

                        raise

                    return chunk
                next = __next__

            self._it = ChunkIterator()

        return self._it
705414
def _get_next(self, history):
        """"""
        orig_history = history
        if not history:
            return helper.START
        history = history[-(self._n-1):]
        while history and not self._T.get(history):
            history = history[1:]
        kv = [(k, v) for k, v in self._T.items(history)
              if not (k in reserved_words or k == history)]
        total = sum(v for k, v in kv)
        while total == 0 and len(history) > 0:
            history = history[1:]
            kv = [(k, v) for k, v in self._T.items(history)
                  if not (k in reserved_words or k == history)]
            total = sum(v for k, v in kv)
        assert total > 0, "Sorry there is no n-gram with {!r}".format(orig_history)
        d = defaultdict(float)
        total = self._T.get(history)
        for k, v in kv:
            k = k[len(history):]
            d[k] += (v+1)/(total + N_VALID_CHARS-1)
        return d
572176
def getTypeLastNode(self, type_):
        

        lastNode = ""
        if type_ and (type(type_) is not list) and (type(type_) is not dict):
            aType = type_.split('>')
            lastNode = aType[len(aType) - 1]
        return lastNode
1292843
def get_next(self, num_bytes=None):
        """"""
        if num_bytes is None:
            return self._data[self._offset:]
        else:
            return self._data[self._offset:self._offset + num_bytes]
1123667
def next(self):
        
        line = self.buffer.readline()

        while line == '\n':       # Skip forward to the next line with content.
            line = self.buffer.readline()

        if line == '':            # End of file.
            self.at_eof = True
            return None, None

        match = re.match('([A-Z]{2}|[C][1])\W(.*)', line)
        if match is not None:
            self.current_tag, data = match.groups()
        else:
            self.current_tag = self.last_tag
            data = line.strip()
        return self.current_tag, _cast(data)
430477
def previous(self, type=None):
        
        i = self.index - 1
        s = self.sentence
        while i > 0:
            if type in (s[i].type, None):
                return s[i]
            i -= 1