Mark whitespaces and comments
332406
function markSC(tokens) {
    let tokensLength = tokens.length;
    let ws = -1; // Flag for whitespaces
    let sc = -1; // Flag for whitespaces and comments
    let t; // Current token

    // For every token in the token list, mark spaces and line breaks
    // as spaces (set both `ws` and `sc` flags). Mark multiline comments
    // with `sc` flag.
    // If there are several spaces or tabs or line breaks or multiline
    // comments in a row, group them: take the last one's index number
    // and save it to the first token in the group as a reference:
    // e.g., `ws_last = 7` for a group of whitespaces or `sc_last = 9`
    // for a group of whitespaces and comments.
    for (var i = 0; i < tokensLength; i++) {
      t = tokens[i];
      switch (t.type) {
        case TokenType.Space:
        case TokenType.Tab:
        case TokenType.Newline:
          t.ws = true;
          t.sc = true;

          if (ws === -1) ws = i;
          if (sc === -1) sc = i;

          break;
        case TokenType.CommentML:
        case TokenType.CommentSL:
          if (ws !== -1) {
            tokens[ws].ws_last = i - 1;
            ws = -1;
          }

          t.sc = true;

          break;
        default:
          if (ws !== -1) {
            tokens[ws].ws_last = i - 1;
            ws = -1;
          }

          if (sc !== -1) {
            tokens[sc].sc_last = i - 1;
            sc = -1;
          }
      }
    }

    if (ws !== -1) tokens[ws].ws_last = i - 1;
    if (sc !== -1) tokens[sc].sc_last = i - 1;
  }
371969
function whitespaceEndPos( $tpl, &$text, $currentPosition, $textLength )
    {
        if ( $currentPosition >= $textLength )
            return $currentPosition;
        while( $currentPosition < $textLength and
               ( $text[$currentPosition] === ' '
                 or $text[$currentPosition] === "\n"
                 or $text[$currentPosition] === "\t"
                 or $text[$currentPosition] === "\r" ) )
        {
            ++$currentPosition;
        }
        return $currentPosition;
    }
352849
function (commentLine) {
    if (state === STATES.IN_COMMENT) {
        commentLine = removePrecedingAsterisks(commentLine);
        // If there is a single space after the * then remove it. Otherwise assume that the user is indenting.
        if (commentLine.indexOf("  ") !== 0 && commentLine.indexOf(" ") === 0) {
            commentLine = commentLine.substr(1);
        }
    }

    return commentLine;
}
334118
protected List<String> trim(List<String> comments) {
        List<String> trimComments = new ArrayList<>();
        int n = 0;
        boolean tryRemoveWhitespace = true;
        while (tryRemoveWhitespace) {
            boolean allLinesAreShorter = true;
            for (String comment : comments) {
                if (comment.length() <= n) {
                    continue;
                }
                if (comment.charAt(n) != ' ') {
                    tryRemoveWhitespace = false;
                }
                allLinesAreShorter = false;
            }
            if (allLinesAreShorter) {
                break;
            }
            if (tryRemoveWhitespace) {
                n++;
            }
        }
        for (String comment : comments) {
            if (comment.length() > n) {
                String substring = comment.substring(n);
                trimComments.add(substring);
            } else {
                trimComments.add("");
            }
        }
        return trimComments;
    }
928358
protected function whitespace() {
        $gotWhite = false;
        while (preg_match(self::$whitePattern, $this->buffer, $m, null, $this->count)) {
            if ($this->insertComments) {
                if (isset($m[1]) && empty($this->commentsSeen[$this->count])) {
                    $this->append(array("comment", $m[1]));
                    $this->commentsSeen[$this->count] = true;
                }
            }
            $this->count += strlen($m[0]);
            $gotWhite = true;
        }
        return $gotWhite;
    }
45043
protected void burnWhitespaces() {
        while (reader.hasNextCharacter()) {
            final char character = reader.peekCharacter();
            if (Strings.isWhitespace(character) || character == commentMarker && reader.hasNextCharacter(1)
                    && reader.peekCharacter(1) == commentSecondary) {
                next();
            } else {
                return;
            }
        }
    }
153123
private void flushCharacters()
    {
        int indentLength = 0;
        int len = charBuffer.length();
        /*
         * Count characters from end of ignorable whitespace to first end of line we hit
         */
        for ( int i = len - 1; i >= 0; i-- )
        {
            char ch = charBuffer.charAt( i );
            switch ( ch )
            {
                case '\n':
                case '\r':
                    lastIndent = new Indent( charLineNumber, indentLength );
                    charBuffer.setLength( 0 );
                    return;
                case ' ':
                case '\t':
                    indentLength++;
                    break;
                default:
                    /*
                     * No end of line foundIndent in the trailing whitespace. Leave the foundIndent from previous
                     * ignorable whitespace unchanged
                     */
                    charBuffer.setLength( 0 );
                    return;
            }
        }
    }
403138
function(source) {

    // Remove comment markup from comments
    var comment = source.split(/\/\/[ ]?/).join('').split('/*').join('').split('*/').join(''),
      additionalKssParams = {},
      _this = this;

    comment = standardizeLinebreaks(comment);
    comment = trimLinebreaks(comment);

    comment.split('\n\n').forEach(function(markUpBlock) {

      var varName = markUpBlock.match(/([^:^\n]*):[\s\S]*\n/);
      if (varName && varName[1] !== undefined) {
        varName = varName[1].trim();
      }
      if (varName && varName.substring(0, 3) === 'sg-') {
        additionalKssParams[varName] = _this.getValue(varName, markUpBlock);
      }

    });

    return additionalKssParams;
  }
100507
def highlight_block(self, text):
        """"""
        text = to_text_string(text)
        if text.startswith(("c", "C")):
            self.setFormat(0, len(text), self.formats["comment"])
            self.highlight_spaces(text)
        else:
            FortranSH.highlight_block(self, text)
            self.setFormat(0, 5, self.formats["comment"])
            self.setFormat(73, max([73, len(text)]),
                           self.formats["comment"])
1112962
def whitespace_tokenize(text):
    """"""
    text = text.strip()
    if not text:
        return []
    tokens = text.split()
    return tokens
413538
private void commentLinesAfter( String content, int line )
  {
    int offset = _root.getElement( line ).getEndOffset();
    // End of comment not found, nothing to do
    int endDelimiter = indexOf( content, getEndDelimiter(), offset );
    if( endDelimiter < 0 )
    {
      return;
    }
    // Matching start/end of comment found, comment the lines
    int startDelimiter = lastIndexOf( content, getStartDelimiter(), endDelimiter );
    if( startDelimiter < 0 || startDelimiter <= offset )
    {
      setCharacterAttributes( offset, endDelimiter - offset + 1, _comment, false );
    }
  }
1208160
function(all, backslash, comment){
      var spaces = "                                                                                                                                                                                                     ";
      var padding = comment ? spaces.substring(0, comment.length) : "";
      return backslash + " \x12" + padding;
    }
1206298
def combine_comments(comments):
    '''
    
    '''
    if not isinstance(comments, list):
        comments = [comments]
    ret = []
    for comment in comments:
        if not isinstance(comment, six.string_types):
            comment = str(comment)
        # Normalize for any spaces (or lack thereof) after the #
        ret.append('# {0}\n'.format(comment.lstrip('#').lstrip()))
    return ''.join(ret)
201464
def comment_lines(lines, prefix):
    """"""
    if not prefix:
        return lines
    return [prefix + ' ' + line if line else prefix for line in lines]
556591
function stripComments(token) {
  return strip

  /* Strip comments. */
  function strip(value) {
    var index = value.indexOf(token)

    if (index !== -1) {
      value = value.substr(0, index)
    }

    return value
  }
}
364103
def CleanComments(line, quote=False):
    
    if line.find('#') == -1 and line.find('"') == -1:
        if quote:
            return '', quote
        else:
            return line, quote
    # else have to check for comment
    prior = []
    prev = ''
    for char in line:
        try:
            if char == '"':
                if prev != '\\':
                    quote = not quote
                    prior.append(char)
                continue
            elif char == '#' and not quote:
                break
            if not quote:
                prior.append(char)
        finally:
            prev = char

    # rstrip removes trailing space between end of command and the comment # start

    return ''.join(prior).rstrip(), quote
550762
def strip_comments(tokens):
    
    prev_typ = None
    prev_end_col = 0
    for typ, tok, (start_row, start_col), (end_row, end_col), line in tokens:
        if typ in (tokenize.NL, tokenize.NEWLINE):
            if prev_typ in (tokenize.NL, tokenize.NEWLINE):
                start_col = 0
            else:
                start_col = prev_end_col
            end_col = start_col + 1
        elif typ == tokenize.COMMENT and start_row > 2:
            continue
        prev_typ = typ
        prev_end_col = end_col
        yield typ, tok, (start_row, start_col), (end_row, end_col), line