The terminator might have some additional newlines before it.
        There is at least one application that sends additional newlines
        before headers (the python setuptools package).
331992
def _find_terminator(self, iterator):
        
        for line in iterator:
            if not line:
                break
            line = line.strip()
            if line:
                return line
        return b''
765672
def tic(self):
        
        if self.verbose:
            self.flush()
            self.write('\ntic(%r)' % self.label)
            if self.newline:
                self.write('\n')
            self.flush()
        self.tstart = self._time()
        return self
652683
def _write_header(self):
        """"""
        for line in self.header.lines:
            print(line.serialize(), file=self.stream)
        if self.header.samples.names:
            print(
                "\t".join(list(parser.REQUIRE_SAMPLE_HEADER) + self.header.samples.names),
                file=self.stream,
            )
        else:
            print("\t".join(parser.REQUIRE_NO_SAMPLE_HEADER), file=self.stream)
501805
def preparse_iter(self):
        
        to_yield = {}
        last_directive = None
        lines_processed = 0
        for line in self.lines_iter():
            if not line:
                continue
            if line.startswith(u'#'):
                comment = line.lstrip('#').strip()
                # Directives have to precede any instructions
                if lines_processed == 1:
                    if comment.startswith(u'escape='):
                        self.escape_char = comment.split(u'=', 1)[1]
                        continue
                to_yield.setdefault('comments', []).append(comment)
            else:
                # last_directive being set means the previous line ended with a
                # newline escape
                if last_directive:
                    directive, payload = last_directive, line
                else:
                    directive, payload = line.split(u' ', 1)
                if line.endswith(self.escape_char):
                    payload = payload.rstrip(self.escape_char)
                    last_directive = directive
                else:
                    last_directive = None
                to_yield['directive'] = directive
                to_yield['payload'] = payload.strip()
                yield to_yield
                to_yield = {}
362905
def __header(self, line):
        
        self.cpu_number = len(line.split())
        return self.cpu_number
508324
def newline(self, *args, **kwargs):
        
        levelOverride = kwargs.get('level') or self._lastlevel
        self._log(levelOverride, '', 'newline', args, kwargs)
63015
def main(argv=None):
    ''''''

    t = CrfTokenizer()
    print t.tokenize("This is a sentence.")
    print t.tokenize("Buy???This...Now!!!")
    print t.tokenize("The <bold>only</bold> source.")
    print t.tokenize("The<bold>only</bold>source.")
    print t.tokenize("Big&gt;little.")
    print t.tokenize("Big & little.")
    print t.tokenize("blond&curly.")
    print t.tokenize("&brokenHtml")
    t.setGroupPunctuation(True)
    t.setRecognizeHtmlTags(True)
    t.setRecognizeHtmlEntities(True)
    print t.tokenize("Buy???This...Now!!!")
    print t.tokenize("The <bold>only</bold> source.")
    print t.tokenize("The<bold>only</bold>source.")
    print t.tokenize("Big&gt;little.")
    print t.tokenize("Big & little.")
    print t.tokenize("blond&curly.")
    print t.tokenize("&brokenHtml")
    t.setSkipHtmlTags(True)
    t.setSkipHtmlEntities(True)
    print t.tokenize("Buy???This...Now!!!")
    print t.tokenize("The <bold>only</bold> source.")
    print t.tokenize("The<bold>only</bold>source.")
    print t.tokenize("Big&gt;little.")
    print t.tokenize("Big & little.")
    print t.tokenize("blond&curly.")
    print t.tokenize("&brokenHtml")
    t.setTokenPrefix("X:")
    print t.tokenize("Tokenize with prefixes.")
    t.setTokenPrefix(None)
    print t.tokenize("No more  prefixes.")
    t.setRecognizePunctuation(False)
    print t.tokenize("This is a sentence.")
    print t.tokenize("Buy???This...Now!!!")
    print t.tokenize("The <bold>only</bold> source.")
    print t.tokenize("The<bold>only</bold>source.")
    print t.tokenize("Big&gt;little.")
    print t.tokenize("Big & little.")
    print t.tokenize("blond&curly.")
    print t.tokenize("&brokenHtml")
    print t.tokenize("A line break goes here\n\t \rand a new line starts")
    t.setRecognizeLinebreaks(True)
    print t.tokenize("A line break goes here\n\r \rand a new line starts")
218187
private void parseHeaderLine(Map<String, String> headers, String header) {
        final int colonOffset = header.indexOf(':');
        if (colonOffset == -1) {
            // This header line is malformed, skip it.
            return;
        }
        String headerName = header.substring(0, colonOffset).trim().toLowerCase();
        String headerValue = header.substring(header.indexOf(':') + 1).trim();
        if (getHeader(headers, headerName) != null) {
            // More that one heder of that name exists,
            // append to the list.
            headers.put(headerName, getHeader(headers, headerName) + ',' + headerValue);
        } else {
            headers.put(headerName, headerValue);
        }
    }
656589
def fix_bad_headers(self):
        
        # WCSLIB doesn't like "nonstandard" units
        unit = self.header.get('CUNIT1', 'deg')
        if unit.upper() == 'DEGREE':
            # self.header.update('CUNIT1', 'deg')
            self.header['CUNIT1'] = 'deg'
        unit = self.header.get('CUNIT2', 'deg')
        if unit.upper() == 'DEGREE':
            # self.header.update('CUNIT2', 'deg')
            self.header['CUNIT2'] = 'deg'
779247
def add_ti_txt(self, lines, overwrite=False):
        

        address = None
        eof_found = False

        for line in StringIO(lines):
            # Abort if data is found after end of file.
            if eof_found:
                raise Error("bad file terminator")

            line = line.strip()

            if len(line) < 1:
                raise Error("bad line length")

            if line[0] == 'q':
                eof_found = True
            elif line[0] == '@':
                try:
                    address = int(line[1:], 16)
                except ValueError:
                    raise Error("bad section address")
            else:
                # Try to decode the data.
                try:
                    data = bytearray(binascii.unhexlify(line.replace(' ', '')))
                except (TypeError, binascii.Error):
                    raise Error("bad data")

                size = len(data)

                # Check that there are correct number of bytes per
                # line. There should TI_TXT_BYTES_PER_LINE. Only
                # exception is last line of section which may be
                # shorter.
                if size > TI_TXT_BYTES_PER_LINE:
                    raise Error("bad line length")

                if address is None:
                    raise Error("missing section address")

                self._segments.add(_Segment(address,
                                            address + size,
                                            data,
                                            self.word_size_bytes),
                                   overwrite)

                if size == TI_TXT_BYTES_PER_LINE:
                    address += size
                else:
                    address = None

        if not eof_found:
            raise Error("missing file terminator")
1372807
def header(self):
        
        
        cmd = [self.__samtools, 'view', '-H', self.__bam]

        stdout = subprocess.Popen(cmd, stdout=subprocess.PIPE).stdout

        for l in stdout:
            yield l.decode('utf-8').strip()

        stdout.close()
177963
public static void appendPair(StringBuffer b, String s1, String s2,
                                  String separator, String terminator) {

        b.append(s1);
        b.append(separator);
        b.append(s2);
        b.append(terminator);
    }
255207
def render(txt):
    

    # Removing links to other channels
    txt = re.sub(r'<#[^\|]*\|(.*)>', r'#\g<1>', txt)

    # Removing links to other users
    txt = re.sub(r'<(@.*)>', r'\g<1>', txt)

    # handle named hyperlinks
    txt = re.sub(r'<([^\|]*)\|([^\|]*)>', r'<a href="\g<1>" target="blank">\g<2></a>', txt)

    # handle unnamed hyperlinks
    txt = re.sub(r'<([^a|/a].*)>', r'<a href="\g<1>" target="blank">\g<1></a>', txt)

    # handle ordered and unordered lists
    for delimeter in LIST_DELIMITERS:
        slack_tag = delimeter
        class_name = LIST_DELIMITERS[delimeter]

        # Wrap any lines that start with the slack_tag in <li></li>
        list_regex = u'(?:^|\n){}\s?(.*)'.format(slack_tag)
        list_repl = r'<li class="list-item-{}">\g<1></li>'.format(class_name)
        txt = re.sub(list_regex, list_repl, txt)

    # hanlde blockquotes
    txt = re.sub(u'(^|\n)(?:&gt;){3}\s?(.*)$', r'\g<1><blockquote>\g<2></blockquote>', txt, flags=re.DOTALL)
    txt = re.sub(u'(?:^|\n)&gt;\s?(.*)\n?', r'<blockquote>\g<1></blockquote>', txt)

    # handle code blocks
    txt = re.sub(r'```\n?(.*)```', r'<pre>\g<1></pre>', txt, flags=re.DOTALL)
    txt = re.sub(r'\n(</pre>)', r'\g<1>', txt)

    # handle bolding, italics, and strikethrough
    for wrapper in FORMATTERS:
        slack_tag = wrapper
        html_tag = FORMATTERS[wrapper]

        # Grab all text in formatted characters on the same line unless escaped
        regex = r'(?<!\\)\{t}([^\{t}|\n]*)\{t}'.format(t=slack_tag)
        repl = r'<{t}>\g<1></{t}>'.format(t=html_tag)
        txt = re.sub(regex, repl, txt)

    # convert line breaks
    txt = txt.replace('\n', '<br />')

    # clean up bad HTML
    parser = CustomSlackdownHTMLParser(txt)
    txt = parser.clean()

    # convert multiple spaces
    txt = txt.replace(r'  ', ' &nbsp')

    return txt
264035
private void matchEndLine() {
		// First, parse all whitespace characters except for new lines
		index = skipLineSpace(index);

		// Second, check whether we've reached the end-of-file (as signaled by
		// running out of tokens), or we've encountered some token which not a
		// newline.
		if (index >= tokens.size()) {
			return; // EOF
		} else if (tokens.get(index).kind != NewLine) {
			syntaxError("expected end-of-line", tokens.get(index));
		} else {
			index = index + 1;
		}
	}
1298446
function Tokenizer(inp, options){
	this.inp = inp||'';
	// replace all other line terminators with \n (leave \r\n in tact though). we should probably remove the shadowInp when finished...
	// only replace \r if it is not followed by a \n else \r\n would become \n\n causing a double newline where it is just a single
	this.shadowInp = (inp||'').replace(Tokenizer.regexNormalizeNewlines, '\n');
	this.pos = 0;
	this.line = 0;
	this.column = 0;
	this.cache = {};

	this.errorStack = [];

	this.wtree = [];
	this.btree = [];

//	this.regexWhiteSpace = Tokenizer.regexWhiteSpace;
	this.regexLineTerminator = Tokenizer.regexLineTerminator; // used in fallback
	this.regexAsciiIdentifier = Tokenizer.regexAsciiIdentifier;
	this.hashAsciiIdentifier = Tokenizer.hashAsciiIdentifier;
//	this.regexHex = Tokenizer.regexHex;
	this.hashHex = Tokenizer.hashHex
	this.regexUnicodeEscape = Tokenizer.regexUnicodeEscape;
	this.regexIdentifierStop = Tokenizer.regexIdentifierStop;
	this.hashIdentifierStop = Tokenizer.hashIdentifierStop;
//	this.regexPunctuators = Tokenizer.regexPunctuators;
	this.regexNumber = Tokenizer.regexNumber;
	this.regexNewline = Tokenizer.regexNewline;

	this.regexBig = Tokenizer.regexBig;
	this.regexBigAlt = Tokenizer.regexBigAlt;

    // stuff for parsing tag literals
    this.regexTagName = Tokenizer.regexTagName;
    this.regexTagAttributes = Tokenizer.regexTagAttributes;
    this.regexTagUnarySuffix = Tokenizer.regexTagUnarySuffix;
    this.regexTagBinarySuffix = Tokenizer.regexTagBinarySuffix;
    this.regexTagBody = Tokenizer.regexTagBody;
    this.regexTagOpenOrClose = Tokenizer.regexTagOpenOrClose;
    this.regexTagClose = Tokenizer.regexTagClose;
	this.regexRemoveEscape = Tokenizer.regexRemoveEscape;

	this.tokenCount = 0;
	this.tokenCountNoWhite = 0;

	this.Unicode = window.Unicode;

	// if the Parser throws an error. it will set this property to the next match
	// at the time of the error (which was not what it was expecting at that point) 
	// and pass on an "error" match. the error should be scooped on the stack and 
	// this property should be returned, without looking at the input...
	this.errorEscape = null;

	// support tag literals
	this.tagLiterals = false || (options && options.tagLiterals);
}
1180309
def log_no_newline(self, msg):
      
      self.print2file(self.logfile, False, False, msg)
469940
def n_list(self, node):
        
        p = self.prec
        self.prec = 100
        lastnode = node.pop()
        lastnodetype = lastnode.kind

        # If this build list is inside a CALL_FUNCTION_VAR,
        # then the first * has already been printed.
        # Until I have a better way to check for CALL_FUNCTION_VAR,
        # will assume that if the text ends in *.
        last_was_star = self.f.getvalue().endswith('*')

        if lastnodetype.endswith('UNPACK'):
            # FIXME: need to handle range of BUILD_LIST_UNPACK
            have_star = True
            # endchar = ''
        else:
            have_star = False

        if lastnodetype.startswith('BUILD_LIST'):
            self.write('['); endchar = ']'
        elif lastnodetype.startswith('BUILD_TUPLE'):
            # Tuples can appear places that can NOT
            # have parenthesis around them, like array
            # subscripts. We check for that by seeing
            # if a tuple item is some sort of slice.
            no_parens = False
            for n in node:
                if n == 'expr' and n[0].kind.startswith('build_slice'):
                    no_parens = True
                    break
                pass
            if no_parens:
                endchar = ''
            else:
                self.write('('); endchar = ')'
                pass

        elif lastnodetype.startswith('BUILD_SET'):
            self.write('{'); endchar = '}'
        elif lastnodetype.startswith('BUILD_MAP_UNPACK'):
            self.write('{*'); endchar = '}'
        elif lastnodetype.startswith('ROT_TWO'):
            self.write('('); endchar = ')'
        else:
            raise TypeError('Internal Error: n_build_list expects list, tuple, set, or unpack')

        flat_elems = flatten_list(node)

        self.indent_more(INDENT_PER_LEVEL)
        sep = ''
        for elem in flat_elems:
            if elem in ('ROT_THREE', 'EXTENDED_ARG'):
                continue
            assert elem == 'expr'
            line_number = self.line_number
            value = self.traverse(elem)
            if line_number != self.line_number:
                sep += '\n' + self.indent + INDENT_PER_LEVEL[:-1]
            else:
                if sep != '': sep += ' '
            if not last_was_star:
                if have_star:
                    sep += '*'
                    pass
                pass
            else:
                last_was_star = False
            self.write(sep, value)
            sep = ','
        if lastnode.attr == 1 and lastnodetype.startswith('BUILD_TUPLE'):
            self.write(',')
        self.write(endchar)
        self.indent_less(INDENT_PER_LEVEL)

        self.prec = p
        self.prune()
        return