Drop comment tokens from a `tokenize` stream.

    Comments on lines 1-2 are kept, to preserve hashbang and encoding.
    Trailing whitespace is remove from all lines.
550762
def strip_comments(tokens):
    
    prev_typ = None
    prev_end_col = 0
    for typ, tok, (start_row, start_col), (end_row, end_col), line in tokens:
        if typ in (tokenize.NL, tokenize.NEWLINE):
            if prev_typ in (tokenize.NL, tokenize.NEWLINE):
                start_col = 0
            else:
                start_col = prev_end_col
            end_col = start_col + 1
        elif typ == tokenize.COMMENT and start_row > 2:
            continue
        prev_typ = typ
        prev_end_col = end_col
        yield typ, tok, (start_row, start_col), (end_row, end_col), line
1365591
function(content) {
			if (!reHasComment.test(content)) {
				return content;
			}

			var stream = stringStream(content);
			var replaceRanges = [];
			var ch, ch2;

			while ((ch = stream.next())) {
				if (ch === '/') {
					ch2 = stream.peek();
					if (ch2 === '*') { // multiline CSS comment
						stream.start = stream.pos - 1;

						if (stream.skipTo('*/')) {
							stream.pos += 2;
						} else {
							// unclosed comment
							stream.skipToEnd();
						}

						replaceRanges.push([stream.start, stream.pos]);
					} else if (ch2 === '/') {
						// preprocessor’s single line comments
						stream.start = stream.pos - 1;
						while ((ch2 = stream.next())) {
							if (ch2 === '\n' || ch2 == '\r') {
								break
							}
						}

						replaceRanges.push([stream.start, stream.pos]);
					}
				} else {
					stream.skipQuoted();
				}
			}

			return utils.replaceWith(content, replaceRanges, ' ');
		}
479754
def _starts_till(src, probe, strip_comments=True):
    '''
    
    '''
    def _strip_comments(txt):
        '''
        Strip possible comments.
        Usually comments are one or two symbols at the beginning of the line, separated with space
        '''
        buff = txt.split(" ", 1)
        return len(buff) == 2 and len(buff[0]) < 2 and buff[1] or txt

    def _to_words(txt):
        '''
        Split by words
        '''
        return txt and [w for w in txt.strip().split(" ") if w.strip()] or txt

    no_match = -1
    equal = 0
    if not src or not probe:
        return no_match

    src = src.rstrip('\n\r')
    probe = probe.rstrip('\n\r')
    if src == probe:
        return equal

    src = _to_words(strip_comments and _strip_comments(src) or src)
    probe = _to_words(strip_comments and _strip_comments(probe) or probe)

    a_buff, b_buff = len(src) < len(probe) and (src, probe) or (probe, src)
    b_buff = ' '.join(b_buff)
    for idx in range(len(a_buff)):
        prb = ' '.join(a_buff[:-(idx + 1)])
        if prb and b_buff.startswith(prb):
            return idx

    return no_match
303676
public void skipWhitespace(List<Token> tokens) {
		while (pos < input.length()
				&& (input.charAt(pos) == '\n' || input.charAt(pos) == '\t')) {
			pos++;
		}
	}
257740
def transform_comments(separator, &block)
      if @comments.empty?
        nil
      else
        block ||= lambda { |c| c }
        @comments.map(&block).join(separator)
      end
    end
964998
def comment_lines(lines):
    """"""
    ret = []
    for line in lines:
        ws_prefix, rest, ignore = RE_LINE_SPLITTER_COMMENT.match(line).groups()
        ret.append(ws_prefix + '#' + rest)
    return ''.join(ret)
386852
def tables_with_comments(stream)
      tables_without_comments(stream)
      @connection.tables.sort.each do |table_name|
        dump_comments(table_name, stream)
      end

      unless (index_comments = @connection.index_comments).empty?
        index_comments.each_pair do |index_name, comment|
          stream.puts "  set_index_comment '#{index_name}', '#{format_comment(comment)}'"
        end
      end
    end
955808
private void skipEmptyLines() {
		int tmp = index;
		do {
			tmp = skipLineSpace(tmp);
			if (tmp < tokens.size() && tokens.get(tmp).kind != Token.Kind.NewLine) {
				return; // done
			} else if (tmp >= tokens.size()) {
				index = tmp;
				return; // end-of-file reached
			}
			// otherwise, skip newline and continue
			tmp = tmp + 1;
			index = tmp;
		} while (true);
		// deadcode
	}
1250880
def commentless(data):
    
    it = iter(data)
    while True:
        line = next(it)
        while ":" in line or not line.lstrip().startswith(".."):
            yield line
            line = next(it)
        indent = indent_size(line)
        it = itertools.dropwhile(lambda el: indent_size(el) > indent
                                            or not el.strip(), it)
221611
def process_composite_comment(self, level, comments, key):
        
        if key not in comments:
            comment = ""
        else:
            value = comments[key]
            spacer = self.whitespace(level, 0)

            if isinstance(value, list):
                comments = [self.format_comment(spacer, v) for v in value]
                comment = self.newlinechar.join(comments)
            else:
                comment = self.format_comment(spacer, value)

        return comment
80995
protected function consumeMultipleLineComment()
    {
        $this->get();
        $comment = '';
        for(;;) {
            $get = $this->get();
            if ($get === '*') {
                if ($this->peek() === '/') { // end of comment reached
                    $this->get();
                    if (0 === strpos($comment, '!')) {
                        // preserved by YUI Compressor
                        if (!$this->keptComment) {
                            // don't prepend a newline if two comments right after one another
                            $this->keptComment = "\n";
                        }
                        $this->keptComment .= "/*!" . substr($comment, 1) . "*/\n";
                    } else if (preg_match('/^@(?:cc_on|if|elif|else|end)\\b/', $comment)) {
                        // IE conditional
                        $this->keptComment .= "/*{$comment}*/";
                    }
                    return;
                }
            } elseif ($get === null) {
                throw new UnterminatedCommentException(
                    "JSMin: Unterminated comment at byte {$this->inputIndex}: /*{$comment}");
            }
            $comment .= $get;
        }
    }
625941
def strip_comments(text):
    
    regex = r'\s*(#|\/{2}).*$'
    regex_inline = r'(:?(?:\s)*([A-Za-z\d\.{}]*)|((?<=\").*\"),?)(?:\s)*(((#|(\/{2})).*)|)$'  # noqa
    lines = text.split('\n')

    for index, line in enumerate(lines):
        if re.search(regex, line):
            if re.search(r'^' + regex, line, re.IGNORECASE):
                lines[index] = ""
            elif re.search(regex_inline, line):
                lines[index] = re.sub(regex_inline, r'\1', line)

    return '\n'.join(lines)
958900
def _end_comment_type_from_line(line):
    
    regex = re.compile(r"^{0}".format(_FTR_COMMENT))
    match = regex.match(line)
    if match:
        return " */"

    return ""
825940
def _tokenize(self, stream):
        

        self._tokens = []
        self._reset_token()
        self._state = self.ST_TOKEN

        for chunk in iter(lambda: stream.read(8192), ''):
            for char in chunk:
                if char in self.NEWLINES:
                    self._process_newline(char)

                else:
                    state = self._state

                    if state == self.ST_STRING:
                        self._process_string(char)

                    elif state == self.ST_TOKEN:
                        self._process_tokens(char)
489128
def skip(self):
        """"""
        buflen = len(self.buf)

        while True:
            self.buf = self.buf.lstrip()
            if self.buf == '':
                self.readline()
                buflen = len(self.buf)
            else:
                self.offset += (buflen - len(self.buf))
                break

        # do not keep comments in the syntax tree
        if not self.keep_comments:
            # skip line comment
            if self.buf[0] == '/':
                if self.buf[1] == '/':
                    self.readline()
                    return self.skip()
            # skip block comment
                elif self.buf[1] == '*':
                    i = self.buf.find('*/')
                    while i == -1:
                        self.readline()
                        i = self.buf.find('*/')
                    self.set_buf(i+2)
                    return self.skip()
427298
def insert_spacing! lines, comments
      comment_count = comments.size

      return if comments.empty?
      return if lines.size == 1
      return if lines[comment_count] == "\n"

      lines.insert comment_count, "\n"
    end
570218
function removeThisFunction (tokens, index) {
  for (var i = index + 1; i < tokens.length; i++) {
    var token = tokens[i]
    
    if (token.type === "operator") {
      if(token.data === ";") {
        //Remove the trailing newline
        var nextToken = tokens[i+1]
        if(nextToken.type === "whitespace" && nextToken.data[0] === "\n") {
          nextToken.data = nextToken.data.substr(1)
        }

        //Remove the previous tabs or spaces
        var prevToken = tokens[index-1]
        if(prevToken.type === "whitespace") {
          var indentation = prevToken.data.match("[\t ]*$")[0]
          prevToken.data = prevToken.data.substr(0, prevToken.data.length - indentation.length)
        }

        // Pull out the function
        tokens.splice(index, i - index + 1)
        return
      }
    }
  }
}