The terminator might have some additional newlines before it.
        There is at least one application that sends additional newlines
        before headers (the python setuptools package).
435589
def _find_terminator(self, iterator):
        
        for line in iterator:
            if not line:
                break
            line = line.strip()
            if line:
                return line
        return b""
543116
def _shorten_line_at_tokens_new(tokens, source, indentation,
                                max_line_length):
    
    # Yield the original source so to see if it's a better choice than the
    # shortened candidate lines we generate here.
    yield indentation + source

    parsed_tokens = _parse_tokens(tokens)

    if parsed_tokens:
        # Perform two reflows. The first one starts on the same line as the
        # prefix. The second starts on the line after the prefix.
        fixed = _reflow_lines(parsed_tokens, indentation, max_line_length,
                              start_on_prefix_line=True)
        if fixed and check_syntax(normalize_multiline(fixed.lstrip())):
            yield fixed

        fixed = _reflow_lines(parsed_tokens, indentation, max_line_length,
                              start_on_prefix_line=False)
        if fixed and check_syntax(normalize_multiline(fixed.lstrip())):
            yield fixed
596072
def error(self, **kwargs):
        '''
        
        '''
        exception_header_width = 100

        e = Error(**kwargs)
        e.module = self.__class__.__name__

        self.errors.append(e)

        if e.exception:
            sys.stderr.write("\n" + e.module + " Exception: " + str(e.exception) + "\n")
            sys.stderr.write("-" * exception_header_width + "\n")
            traceback.print_exc(file=sys.stderr)
            sys.stderr.write("-" * exception_header_width + "\n\n")
        elif e.description:
            sys.stderr.write("\n" + e.module + " Error: " + e.description + "\n\n")
875698
def p_list_terminator(p):
    ''''''
    if p[1] == ';':
        p[0] = ast.node(kind='operator', op=';', pos=p.lexspan(1))
662149
def CheckEnd(self, filename, clean_lines, linenum, error):
    """"""
    line = clean_lines.raw_lines[linenum]

    # Check how many lines is enclosed in this namespace.  Don't issue
    # warning for missing namespace comments if there aren't enough
    # lines.  However, do apply checks if there is already an end of
    # namespace comment and it's incorrect.
    #
    # TODO(unknown): We always want to check end of namespace comments
    # if a namespace is large, but sometimes we also want to apply the
    # check if a short namespace contained nontrivial things (something
    # other than forward declarations).  There is currently no logic on
    # deciding what these nontrivial things are, so this check is
    # triggered by namespace size only, which works most of the time.
    if (linenum - self.starting_linenum < 10
        and not Match(r'};*\s*(//|/\*).*\bnamespace\b', line)):
      return

    # Look for matching comment at end of namespace.
    #
    # Note that we accept C style "/* */" comments for terminating
    # namespaces, so that code that terminate namespaces inside
    # preprocessor macros can be cpplint clean.
    #
    # We also accept stuff like "// end of namespace <name>." with the
    # period at the end.
    #
    # Besides these, we don't accept anything else, otherwise we might
    # get false negatives when existing comment is a substring of the
    # expected namespace.
    if self.name:
      # Named namespace
      if not Match((r'};*\s*(//|/\*).*\bnamespace\s+' + re.escape(self.name) +
                    r'[\*/\.\\\s]*$'),
                   line):
        error(filename, linenum, 'readability/namespace', 5,
              'Namespace should be terminated with "// namespace %s"' %
              self.name)
    else:
      # Anonymous namespace
      if not Match(r'};*\s*(//|/\*).*\bnamespace[\*/\.\\\s]*$', line):
        # If "// namespace anonymous" or "// anonymous namespace (more text)",
        # mention "// anonymous namespace" as an acceptable form
        if Match(r'}.*\b(namespace anonymous|anonymous namespace)\b', line):
          error(filename, linenum, 'readability/namespace', 5,
                'Anonymous namespace should be terminated with "// namespace"'
                ' or "// anonymous namespace"')
        else:
          error(filename, linenum, 'readability/namespace', 5,
                'Anonymous namespace should be terminated with "// namespace"')
30315
function(line) {
    header.resolveTempo();
    multilineVars.is_in_header = false;	// We should have gotten a key header by now, but just in case, this is definitely out of the header.
    var i = 0;
    var startOfLine = multilineVars.iChar;
    //multilineVars.havent_set_length = false;	// To late to set this now.
    // see if there is nothing but a comment on this line. If so, just ignore it. A full line comment is optional white space followed by %
    while (tokenizer.isWhiteSpace(line.charAt(i)) && i < line.length) {
      i++;
    }
    if (i === line.length || line.charAt(i) === '%') {
      return;

    }
    // Start with the standard staff, clef and key symbols on each line
    var delayStartNewLine = multilineVars.start_new_line;
    if (multilineVars.continueall === undefined)
      multilineVars.start_new_line = true;
    else
      multilineVars.start_new_line = false;
    var tripletNotesLeft = 0;

    // See if the line starts with a header field
    var retHeader = header.letter_to_body_header(line, i);
    if (retHeader[0] > 0) {
      i += retHeader[0];
      // TODO-PER: Handle inline headers
    }
    var el = { };

    while (i < line.length)
    {
      var startI = i;
      if (line.charAt(i) === '%')
        break;

      var retInlineHeader = header.letter_to_inline_header(line, i);
      if (retInlineHeader[0] > 0) {
        i += retInlineHeader[0];
        // TODO-PER: Handle inline headers
        //multilineVars.start_new_line = false;
      } else {
        // Wait until here to actually start the line because we know we're past the inline statements.
        if (delayStartNewLine) {
          startNewLine();
          delayStartNewLine = false;
        }

        // We need to decide if the following characters are a bar-marking or a note-group.
        // Unfortunately, that is ambiguous. Both can contain chord symbols and decorations.
        // If there is a grace note either before or after the chord symbols and decorations, then it is definitely a note-group.
        // If there is a bar marker, it is definitely a bar-marking.
        // If there is either a core-note or chord, it is definitely a note-group.
        // So, loop while we find grace-notes, chords-symbols, or decorations. [It is an error to have more than one grace-note group in a row; the others can be multiple]
        // Then, if there is a grace-note, we know where to go.
        // Else see if we have a chord, core-note, slur, triplet, or bar.

        var ret;
        while (1) {
          ret = tokenizer.eatWhiteSpace(line, i);
          if (ret > 0) {
            i += ret;
          }
          if (i > 0 && line.charAt(i-1) === '\x12') {
            // there is one case where a line continuation isn't the same as being on the same line, and that is if the next character after it is a header.
            ret = header.letter_to_body_header(line, i);
            if (ret[0] > 0) {
              // TODO: insert header here
              i = ret[0];
              multilineVars.start_new_line = false;
            }
          }
          // gather all the grace notes, chord symbols and decorations
          ret = letter_to_spacer(line, i);
          if (ret[0] > 0) {
            i += ret[0];
          }

          ret = letter_to_chord(line, i);
          if (ret[0] > 0) {
            // There could be more than one chord here if they have different positions.
            // If two chords have the same position, then connect them with newline.
            if (!el.chord)
              el.chord = [];
            var chordName = tokenizer.translateString(ret[1]);
            chordName = chordName.replace(/;/g, "\n");
            var addedChord = false;
            for (var ci = 0; ci < el.chord.length; ci++) {
              if (el.chord[ci].position === ret[2]) {
                addedChord = true;
                el.chord[ci].name += "\n" + chordName;
              }
            }
            if (addedChord === false) {
              if (ret[2] === null && ret[3])
                el.chord.push({name: chordName, rel_position: ret[3]});
              else
                el.chord.push({name: chordName, position: ret[2]});
            }

            i += ret[0];
            var ii = tokenizer.skipWhiteSpace(line.substring(i));
            if (ii > 0)
              el.force_end_beam_last = true;
            i += ii;
          } else {
            if (nonDecorations.indexOf(line.charAt(i)) === -1)
              ret = letter_to_accent(line, i);
            else ret = [ 0 ];
            if (ret[0] > 0) {
              if (ret[1] === null) {
                if (i + 1 < line.length)
                  startNewLine();	// There was a ! in the middle of the line. Start a new line if there is anything after it.
              } else if (ret[1].length > 0) {
                if (ret[1].indexOf("style=") === 0) {
                  el.style = ret[1].substr(6);
                } else {
                  if (el.decoration === undefined)
                    el.decoration = [];
                  el.decoration.push(ret[1]);
                }
              }
              i += ret[0];
            } else {
              ret = letter_to_grace(line, i);
              // TODO-PER: Be sure there aren't already grace notes defined. That is an error.
              if (ret[0] > 0) {
                el.gracenotes = ret[1];
                i += ret[0];
              } else
                break;
            }
          }
        }

        ret = letter_to_bar(line, i);
        if (ret[0] > 0) {
          // This is definitely a bar
          if (el.gracenotes !== undefined) {
            // Attach the grace note to an invisible note
            el.rest = { type: 'spacer' };
            el.duration = 0.125; // TODO-PER: I don't think the duration of this matters much, but figure out if it does.
            multilineVars.addFormattingOptions(el, tune.formatting, 'note');
            tune.appendElement('note', startOfLine+i, startOfLine+i+ret[0], el);
            multilineVars.measureNotEmpty = true;
            el = {};
          }
          var bar = {type: ret[1]};
          if (bar.type.length === 0)
            warn("Unknown bar type", line, i);
          else {
            if (multilineVars.inEnding && bar.type !== 'bar_thin') {
              bar.endEnding = true;
              multilineVars.inEnding = false;
            }
            if (ret[2]) {
              bar.startEnding = ret[2];
              if (multilineVars.inEnding)
                bar.endEnding = true;
              multilineVars.inEnding = true;
            }
            if (el.decoration !== undefined)
              bar.decoration = el.decoration;
            if (el.chord !== undefined)
              bar.chord = el.chord;
            if (bar.startEnding && multilineVars.barFirstEndingNum === undefined)
              multilineVars.barFirstEndingNum = multilineVars.currBarNumber;
            else if (bar.startEnding && bar.endEnding && multilineVars.barFirstEndingNum)
              multilineVars.currBarNumber = multilineVars.barFirstEndingNum;
            else if (bar.endEnding)
              multilineVars.barFirstEndingNum = undefined;
            if (bar.type !== 'bar_invisible' && multilineVars.measureNotEmpty) {
              multilineVars.currBarNumber++;
              if (multilineVars.barNumbers && multilineVars.currBarNumber % multilineVars.barNumbers === 0)
                multilineVars.barNumOnNextNote = multilineVars.currBarNumber;
            }
            multilineVars.addFormattingOptions(el, tune.formatting, 'bar');
            tune.appendElement('bar', startOfLine+i, startOfLine+i+ret[0], bar);
            multilineVars.measureNotEmpty = false;
            el = {};
          }
          i += ret[0];
        } else if (line[i] === '&') {	// backtrack to beginning of measure
          warn("Overlay not yet supported", line, i);
          i++;

        } else {
          // This is definitely a note group
          //
          // Look for as many open slurs and triplets as there are. (Note: only the first triplet is valid.)
          ret = letter_to_open_slurs_and_triplets(line, i);
          if (ret.consumed > 0) {
            if (ret.startSlur !== undefined)
              el.startSlur = ret.startSlur;
            if (ret.triplet !== undefined) {
              if (tripletNotesLeft > 0)
                warn("Can't nest triplets", line, i);
              else {
                el.startTriplet = ret.triplet;
                tripletNotesLeft = ret.num_notes === undefined ? ret.triplet : ret.num_notes;
              }
            }
            i += ret.consumed;
          }

          // handle chords.
          if (line.charAt(i) === '[') {
            var chordStartChar = i;
            i++;
            var chordDuration = null;

            var done = false;
            while (!done) {
              var chordNote = getCoreNote(line, i, {}, false);
              if (chordNote !== null) {
                if (chordNote.end_beam) {
                  el.end_beam = true;
                  delete chordNote.end_beam;
                }
                if (el.pitches === undefined) {
                  el.duration = chordNote.duration;
                  el.pitches = [ chordNote ];
                } else	// Just ignore the note lengths of all but the first note. The standard isn't clear here, but this seems less confusing.
                  el.pitches.push(chordNote);
                delete chordNote.duration;

                if (multilineVars.inTieChord[el.pitches.length]) {
                  chordNote.endTie = true;
                  multilineVars.inTieChord[el.pitches.length] = undefined;
                }
                if (chordNote.startTie)
                  multilineVars.inTieChord[el.pitches.length] = true;

                i  = chordNote.endChar;
                delete chordNote.endChar;
              } else if (line.charAt(i) === ' ') {
                // Spaces are not allowed in chords, but we can recover from it by ignoring it.
                warn("Spaces are not allowed in chords", line, i);
                i++;
              } else {
                if (i < line.length && line.charAt(i) === ']') {
                  // consume the close bracket
                  i++;

                  if (multilineVars.next_note_duration !== 0) {
                    el.duration = el.duration * multilineVars.next_note_duration;
                    multilineVars.next_note_duration = 0;
                  }

                  if (multilineVars.inTie) {
                    Common.each(el.pitches, function(pitch) { pitch.endTie = true; });
                    multilineVars.inTie = false;
                  }

                  if (tripletNotesLeft > 0) {
                    tripletNotesLeft--;
                    if (tripletNotesLeft === 0) {
                      el.endTriplet = true;
                    }
                  }

                  var postChordDone = false;
                  while (i < line.length && !postChordDone) {
                    switch (line.charAt(i)) {
                      case ' ':
                      case '\t':
                        addEndBeam(el);
                        break;
                      case ')':
                        if (el.endSlur === undefined) el.endSlur = 1; else el.endSlur++;
                        break;
                      case '-':
                        Common.each(el.pitches, function(pitch) { pitch.startTie = {}; });
                        multilineVars.inTie = true;
                        break;
                      case '>':
                      case '<':
                        var br2 = getBrokenRhythm(line, i);
                        i += br2[0] - 1;	// index gets incremented below, so we'll let that happen
                        multilineVars.next_note_duration = br2[2];
                        if (chordDuration)
                          chordDuration = chordDuration * br2[1];
                        else
                          chordDuration = br2[1];
                        break;
                      case '1':
                      case '2':
                      case '3':
                      case '4':
                      case '5':
                      case '6':
                      case '7':
                      case '8':
                      case '9':
                      case '/':
                        var fraction = tokenizer.getFraction(line, i);
                        chordDuration = fraction.value;
                        i = fraction.index;
                        if (line.charAt(i) === '-' || line.charAt(i) === ')' || line.charAt(i) === ' ' || line.charAt(i) === '<' || line.charAt(i) === '>')
                          i--; // Subtracting one because one is automatically added below
                        else
                          postChordDone = true;
                        break;
                      default:
                        postChordDone = true;
                        break;
                    }
                    if (!postChordDone) {
                      i++;
                    }
                  }
                } else
                  warn("Expected ']' to end the chords", line, i);

                if (el.pitches !== undefined) {
                  if (chordDuration !== null) {
                    el.duration = el.duration * chordDuration;
                  }
                  if (multilineVars.barNumOnNextNote) {
                    el.barNumber = multilineVars.barNumOnNextNote;
                    multilineVars.barNumOnNextNote = null;
                  }
                  multilineVars.addFormattingOptions(el, tune.formatting, 'note');
                  tune.appendElement('note', startOfLine+chordStartChar, startOfLine+i, el);
                  multilineVars.measureNotEmpty = true;
                  el = {};
                }
                done = true;
              }
            }

          } else {
            // Single pitch
            var el2 = {};
            var core = getCoreNote(line, i, el2, true);
            if (el2.endTie !== undefined) multilineVars.inTie = true;
            if (core !== null) {
              if (core.pitch !== undefined) {
                el.pitches = [ { } ];
                // TODO-PER: straighten this out so there is not so much copying: getCoreNote shouldn't change e'
                if (core.accidental !== undefined) el.pitches[0].accidental = core.accidental;
                el.pitches[0].pitch = core.pitch;
                if (core.endSlur !== undefined) el.pitches[0].endSlur = core.endSlur;
                if (core.endTie !== undefined) el.pitches[0].endTie = core.endTie;
                if (core.startSlur !== undefined) el.pitches[0].startSlur = core.startSlur;
                if (el.startSlur !== undefined) el.pitches[0].startSlur = el.startSlur;
                if (core.startTie !== undefined) el.pitches[0].startTie = core.startTie;
                if (el.startTie !== undefined) el.pitches[0].startTie = el.startTie;
              } else {
                el.rest = core.rest;
                if (core.endSlur !== undefined) el.endSlur = core.endSlur;
                if (core.endTie !== undefined) el.rest.endTie = core.endTie;
                if (core.startSlur !== undefined) el.startSlur = core.startSlur;
                if (core.startTie !== undefined) el.rest.startTie = core.startTie;
                if (el.startTie !== undefined) el.rest.startTie = el.startTie;
              }

              if (core.chord !== undefined) el.chord = core.chord;
              if (core.duration !== undefined) el.duration = core.duration;
              if (core.decoration !== undefined) el.decoration = core.decoration;
              if (core.graceNotes !== undefined) el.graceNotes = core.graceNotes;
              delete el.startSlur;
              if (multilineVars.inTie) {
                if (el.pitches !== undefined) {
                  el.pitches[0].endTie = true;
                  multilineVars.inTie = false;
                } else if (el.rest.type !== 'spacer') {
                  el.rest.endTie = true;
                  multilineVars.inTie = false;
                }
              }
              if (core.startTie || el.startTie)
                multilineVars.inTie = true;
              i  = core.endChar;

              if (tripletNotesLeft > 0) {
                tripletNotesLeft--;
                if (tripletNotesLeft === 0) {
                  el.endTriplet = true;
                }
              }

              if (core.end_beam)
                addEndBeam(el);

              // If there is a whole rest, then it should be the duration of the measure, not it's own duration. We need to special case it.
              if (el.rest && el.rest.type === 'rest' && el.duration === 1) {
                el.rest.type = 'whole';

                el.duration = durationOfMeasure(multilineVars);
              }

              if (multilineVars.barNumOnNextNote) {
                el.barNumber = multilineVars.barNumOnNextNote;
                multilineVars.barNumOnNextNote = null;
              }
              multilineVars.addFormattingOptions(el, tune.formatting, 'note');
              tune.appendElement('note', startOfLine+startI, startOfLine+i, el);
              multilineVars.measureNotEmpty = true;
              el = {};
            }
          }

          if (i === startI) {	// don't know what this is, so ignore it.
            if (line.charAt(i) !== ' ' && line.charAt(i) !== '`')
              warn("Unknown character ignored", line, i);
            i++;
          }
        }
      }
    }
  }
1251562
def describe(cls) -> None:
        
        max_lengths = []
        for attr_name in cls.attr_names():
            attr_func = "%ss" % attr_name
            attr_list = list(map(str, getattr(cls, attr_func)())) + [attr_name]
            max_lengths.append(max(list(map(len, attr_list))))
        row_format = "{:>%d} | {:>%d} | {:>%d}" % tuple(max_lengths)
        headers = [attr_name.capitalize() for attr_name in cls.attr_names()]
        header_line = row_format.format(*headers)
        output = "Class: %s\n" % cls.__name__
        output += header_line + "\n"
        output += "-"*(len(header_line)) + "\n"
        for item in cls:
            format_list = [str(getattr(item, attr_name))
                           for attr_name in cls.attr_names()]
            output += row_format.format(*format_list) + "\n"
        print(output)
203164
def write_headers(
        self,
        start_line: Union[httputil.RequestStartLine, httputil.ResponseStartLine],
        headers: httputil.HTTPHeaders,
        chunk: bytes = None,
    ) -> "Future[None]":
        """"""
        lines = []
        if self.is_client:
            assert isinstance(start_line, httputil.RequestStartLine)
            self._request_start_line = start_line
            lines.append(utf8("%s %s HTTP/1.1" % (start_line[0], start_line[1])))
            # Client requests with a non-empty body must have either a
            # Content-Length or a Transfer-Encoding.
            self._chunking_output = (
                start_line.method in ("POST", "PUT", "PATCH")
                and "Content-Length" not in headers
                and (
                    "Transfer-Encoding" not in headers
                    or headers["Transfer-Encoding"] == "chunked"
                )
            )
        else:
            assert isinstance(start_line, httputil.ResponseStartLine)
            assert self._request_start_line is not None
            assert self._request_headers is not None
            self._response_start_line = start_line
            lines.append(utf8("HTTP/1.1 %d %s" % (start_line[1], start_line[2])))
            self._chunking_output = (
                # TODO: should this use
                # self._request_start_line.version or
                # start_line.version?
                self._request_start_line.version == "HTTP/1.1"
                # 1xx, 204 and 304 responses have no body (not even a zero-length
                # body), and so should not have either Content-Length or
                # Transfer-Encoding headers.
                and start_line.code not in (204, 304)
                and (start_line.code < 100 or start_line.code >= 200)
                # No need to chunk the output if a Content-Length is specified.
                and "Content-Length" not in headers
                # Applications are discouraged from touching Transfer-Encoding,
                # but if they do, leave it alone.
                and "Transfer-Encoding" not in headers
            )
            # If connection to a 1.1 client will be closed, inform client
            if (
                self._request_start_line.version == "HTTP/1.1"
                and self._disconnect_on_finish
            ):
                headers["Connection"] = "close"
            # If a 1.0 client asked for keep-alive, add the header.
            if (
                self._request_start_line.version == "HTTP/1.0"
                and self._request_headers.get("Connection", "").lower() == "keep-alive"
            ):
                headers["Connection"] = "Keep-Alive"
        if self._chunking_output:
            headers["Transfer-Encoding"] = "chunked"
        if not self.is_client and (
            self._request_start_line.method == "HEAD"
            or cast(httputil.ResponseStartLine, start_line).code == 304
        ):
            self._expected_content_remaining = 0
        elif "Content-Length" in headers:
            self._expected_content_remaining = int(headers["Content-Length"])
        else:
            self._expected_content_remaining = None
        # TODO: headers are supposed to be of type str, but we still have some
        # cases that let bytes slip through. Remove these native_str calls when those
        # are fixed.
        header_lines = (
            native_str(n) + ": " + native_str(v) for n, v in headers.get_all()
        )
        lines.extend(l.encode("latin1") for l in header_lines)
        for line in lines:
            if b"\n" in line:
                raise ValueError("Newline in header: " + repr(line))
        future = None
        if self.stream.closed():
            future = self._write_future = Future()
            future.set_exception(iostream.StreamClosedError())
            future.exception()
        else:
            future = self._write_future = Future()
            data = b"\r\n".join(lines) + b"\r\n\r\n"
            if chunk:
                data += self._format_chunk(chunk)
            self._pending_write = self.stream.write(data)
            future_add_done_callback(self._pending_write, self._on_write_complete)
        return future
1260117
public void print(String separator, String terminator) {
    for (int i = 0; i < maxColumnLength.size(); i++) {
      maxColumnLength.set(i, (((maxColumnLength.get(i) / TAB_SIZE) + 1) * TAB_SIZE));
    }

    for (int r = 0; r < rows.size(); r++) {
      String[] s = rows.get(r);

      for (int i = 0; i < s.length; i++) {
        out.print(s[i]);

        if (i < s.length - 1) {
          for (int l = (s[i] == null ? 0 : s[i].length()); l < maxColumnLength.get(i); l++) {
            out.print(' ');
          }
        }
      }
      if (separator != null && r < rows.size() - 1) { out.println(separator); } else if (terminator != null
          && r == rows.size() - 1) { out.println(terminator); } else { out.println(); }
    }
  }
545482
private void beforeName() throws IOException {
    int context = peek();
    if (context == NONEMPTY_OBJECT) { // first in object
      out.write(',');
    } else if (context != EMPTY_OBJECT) { // not in an object!
      throw new IllegalStateException("Nesting problem.");
    }
    newline();
    replaceTop(DANGLING_NAME);
  }
209141
private void writeLines(final HttpServletResponse response, final ArrayList<String> lines, final String header)
      throws IOException {
    LOG.log(Level.INFO, "HttpServerReefEventHandler writeLines is called");

    final PrintWriter writer = response.getWriter();

    writer.println("<h1>" + header + "</h1>");

    for (final String line : lines) {
      writer.println(line);
      writer.write("<br/>");
    }
    writer.write("<br/>");
  }
1298446
function Tokenizer(inp, options){
	this.inp = inp||'';
	// replace all other line terminators with \n (leave \r\n in tact though). we should probably remove the shadowInp when finished...
	// only replace \r if it is not followed by a \n else \r\n would become \n\n causing a double newline where it is just a single
	this.shadowInp = (inp||'').replace(Tokenizer.regexNormalizeNewlines, '\n');
	this.pos = 0;
	this.line = 0;
	this.column = 0;
	this.cache = {};

	this.errorStack = [];

	this.wtree = [];
	this.btree = [];

//	this.regexWhiteSpace = Tokenizer.regexWhiteSpace;
	this.regexLineTerminator = Tokenizer.regexLineTerminator; // used in fallback
	this.regexAsciiIdentifier = Tokenizer.regexAsciiIdentifier;
	this.hashAsciiIdentifier = Tokenizer.hashAsciiIdentifier;
//	this.regexHex = Tokenizer.regexHex;
	this.hashHex = Tokenizer.hashHex
	this.regexUnicodeEscape = Tokenizer.regexUnicodeEscape;
	this.regexIdentifierStop = Tokenizer.regexIdentifierStop;
	this.hashIdentifierStop = Tokenizer.hashIdentifierStop;
//	this.regexPunctuators = Tokenizer.regexPunctuators;
	this.regexNumber = Tokenizer.regexNumber;
	this.regexNewline = Tokenizer.regexNewline;

	this.regexBig = Tokenizer.regexBig;
	this.regexBigAlt = Tokenizer.regexBigAlt;

    // stuff for parsing tag literals
    this.regexTagName = Tokenizer.regexTagName;
    this.regexTagAttributes = Tokenizer.regexTagAttributes;
    this.regexTagUnarySuffix = Tokenizer.regexTagUnarySuffix;
    this.regexTagBinarySuffix = Tokenizer.regexTagBinarySuffix;
    this.regexTagBody = Tokenizer.regexTagBody;
    this.regexTagOpenOrClose = Tokenizer.regexTagOpenOrClose;
    this.regexTagClose = Tokenizer.regexTagClose;
	this.regexRemoveEscape = Tokenizer.regexRemoveEscape;

	this.tokenCount = 0;
	this.tokenCountNoWhite = 0;

	this.Unicode = window.Unicode;

	// if the Parser throws an error. it will set this property to the next match
	// at the time of the error (which was not what it was expecting at that point) 
	// and pass on an "error" match. the error should be scooped on the stack and 
	// this property should be returned, without looking at the input...
	this.errorEscape = null;

	// support tag literals
	this.tagLiterals = false || (options && options.tagLiterals);
}
451489
public void transfromHeadline ()
  {
    if (m_nHeadlineDepth > 0)
      return;
    int nLevel = 0;
    final Line aLine = m_aLines;
    if (aLine.m_bIsEmpty)
      return;
    int nStart = aLine.m_nLeading;
    while (nStart < aLine.m_sValue.length () && aLine.m_sValue.charAt (nStart) == '#')
    {
      nLevel++;
      nStart++;
    }
    while (nStart < aLine.m_sValue.length () && aLine.m_sValue.charAt (nStart) == ' ')
      nStart++;
    if (nStart >= aLine.m_sValue.length ())
    {
      aLine.setEmpty ();
    }
    else
    {
      int nEnd = aLine.m_sValue.length () - aLine.m_nTrailing - 1;
      while (aLine.m_sValue.charAt (nEnd) == '#')
        nEnd--;
      while (aLine.m_sValue.charAt (nEnd) == ' ')
        nEnd--;
      aLine.m_sValue = aLine.m_sValue.substring (nStart, nEnd + 1);
      aLine.m_nLeading = aLine.m_nTrailing = 0;
    }
    m_nHeadlineDepth = Math.min (nLevel, 6);
  }
565675
def _skip_newlines(self):
        """"""
        while self._cur_token['type'] is TT.lbreak and not self._finished:
            self._increment()
501805
def preparse_iter(self):
        
        to_yield = {}
        last_directive = None
        lines_processed = 0
        for line in self.lines_iter():
            if not line:
                continue
            if line.startswith(u'#'):
                comment = line.lstrip('#').strip()
                # Directives have to precede any instructions
                if lines_processed == 1:
                    if comment.startswith(u'escape='):
                        self.escape_char = comment.split(u'=', 1)[1]
                        continue
                to_yield.setdefault('comments', []).append(comment)
            else:
                # last_directive being set means the previous line ended with a
                # newline escape
                if last_directive:
                    directive, payload = last_directive, line
                else:
                    directive, payload = line.split(u' ', 1)
                if line.endswith(self.escape_char):
                    payload = payload.rstrip(self.escape_char)
                    last_directive = directive
                else:
                    last_directive = None
                to_yield['directive'] = directive
                to_yield['payload'] = payload.strip()
                yield to_yield
                to_yield = {}
66046
def ensure_headers
      missing = missing_headers
      return if missing.size == 0

      message = "Compilation cannot proceed as the following header files are missing:\n" + missing.join("\n") 
      raise Error.new(message)
    end
169439
public void newLineIfNotEmpty() {
		for (int i = segments.size() - 1; i >= 0; i--) {
			String segment = segments.get(i);
			if (lineDelimiter.equals(segment)) {
				segments.subList(i + 1, segments.size()).clear();
				cachedToString = null;
				return;
			}
			for (int j = 0; j < segment.length(); j++) {
				if (!WhitespaceMatcher.isWhitespace(segment.charAt(j))) {
					newLine();
					return;
				}
			}
		}
		segments.clear();
		cachedToString = null;
	}