Merges two sorted lists into a larger sorted list in O(n) time.
@param {Token[]} tokens The list of tokens.
@param {Token[]} comments The list of comments.
@returns {Token[]} A sorted list of tokens and comments.
@private
481218
function sortedMerge(tokens, comments) {
    const result = [];
    let tokenIndex = 0;
    let commentIndex = 0;

    while (tokenIndex < tokens.length || commentIndex < comments.length) {
        if (commentIndex >= comments.length || tokenIndex < tokens.length && tokens[tokenIndex].range[0] < comments[commentIndex].range[0]) {
            result.push(tokens[tokenIndex++]);
        } else {
            result.push(comments[commentIndex++]);
        }
    }

    return result;
}
238661
private void convertTokenDefinitions(Map<String, ParseTreeNode> helpers,
	    Map<String, ParseTreeNode> tokens) throws GrammarException,
	    TreeException {
	tokenDefinitions = new TokenDefinitionSet();
	for (ParseTreeNode tokenDefinitionAST : parserTree.getChild("Tokens")
		.getChild("TokenDefinitions").getChildren("TokenDefinition")) {
	    TokenDefinition convertedTokenDefinition = getTokenDefinition(
		    tokenDefinitionAST, helpers, tokens);
	    tokenDefinitions.addDefinition(convertedTokenDefinition);
	}
    }
629177
public static List<TokensLine> convert(List<TokenEntry> tokens) {
    List<TokensLine> result = new ArrayList<>();
    StringBuilder sb = new StringBuilder();
    int startLine = Integer.MIN_VALUE;
    int startIndex = 0;
    int currentIndex = 0;
    for (TokenEntry token : tokens) {
      if (token != TokenEntry.EOF) {
        String value = token.getValue();
        int line = token.getBeginLine();
        if (line != startLine) {
          addNewTokensLine(result, startIndex, currentIndex, startLine, sb);
          startIndex = currentIndex + 1;
          startLine = line;
        }
        currentIndex++;
        sb.append(value);
      }
    }
    addNewTokensLine(result, startIndex, currentIndex, startLine, sb);
    return result;
  }
1318153
public final String[] getTokensWithMultiWords(final String[] tokens) {
    final Span[] multiWordSpans = multiWordsToSpans(tokens);
    final List<String> tokenList = new ArrayList<String>(Arrays.asList(tokens));
    int counter = 0;
    for (final Span mwSpan : multiWordSpans) {
      final int fromIndex = mwSpan.getStart() - counter;
      final int toIndex = mwSpan.getEnd() - counter;
      // System.err.println(fromIndex + " " + toIndex);
      // add to the counter the length of the sublist removed
      // to allow the fromIndex and toIndex to match wrt to the tokenList
      // indexes
      counter = counter + tokenList.subList(fromIndex, toIndex).size() - 1;
      // create the multiword joining the sublist
      final String multiWord = Joiner.on("#").join(
          tokenList.subList(fromIndex, toIndex));
      // remove the sublist containing the tokens to be replaced in the span
      tokenList.subList(fromIndex, toIndex).clear();
      // add the multiword containing the tokens in one Span
      tokenList.add(fromIndex, multiWord);
    }
    return tokenList.toArray(new String[tokenList.size()]);
  }
321568
function hasCommentInsideNonSpecifiers(node, sourceCode) {
  const tokens = sourceCode.getTokens(node)
  const openBraceIndex = tokens.findIndex(token => isPunctuator(token, '{'))
  const closeBraceIndex = tokens.findIndex(token => isPunctuator(token, '}'))
  // Slice away the first token, since we're no looking for comments _before_
  // `node` (only inside). If there's a `{...}` part, look for comments before
  // the `{`, but not before the `}` (hence the `+1`s).
  const someTokens = openBraceIndex >= 0 && closeBraceIndex >= 0
    ? tokens.slice(1, openBraceIndex + 1).concat(tokens.slice(closeBraceIndex + 1))
    : tokens.slice(1)
  return someTokens.some(token => sourceCode.getCommentsBefore(token).length > 0)
}
1126832
function importNonBlocks(blocks, tokens) {
    var temp = [];
    utils.each(blocks, function (block) { temp.push(block); });
    utils.each(temp.reverse(), function (block) {
      if (block.name !== 'block') {
        tokens.unshift(block);
      }
    });
  }
455113
private void addTokens(final Token curToken, final List<Token> tokens) {
    if (curToken.tokenLength() != 0) {
      if (this.unTokenizable) {
        tokens.add(curToken);
      } else if (!this.unTokenizable) {
        if (!replacement.matcher(curToken.getTokenValue()).matches()) {
          tokens.add(curToken);
        }
      }
    }
  }
952102
def tokens(cls, tokens):
        
        return cls(Lnk.TOKENS, tuple(map(int, tokens)))
1185828
def join(tokens, start, result):
    """"""
    texts = []
    if len(result) > 0:
        for e in result:
            for child in e.iter():
                if child.text is not None:
                    texts.append(child.text)
        return [E(result[0].tag, ' '.join(texts))]
1150875
function joinDeclarations(declarations) {
            const declaration = declarations[0];
            const body = Array.isArray(declaration.parent.parent.body) ? declaration.parent.parent.body : [];
            const currentIndex = body.findIndex(node => node.range[0] === declaration.parent.range[0]);
            const previousNode = body[currentIndex - 1];

            return fixer => {
                const type = sourceCode.getTokenBefore(declaration);
                const prevSemi = sourceCode.getTokenBefore(type);
                const res = [];

                if (previousNode && previousNode.kind === sourceCode.getText(type)) {
                    if (prevSemi.value === ";") {
                        res.push(fixer.replaceText(prevSemi, ","));
                    } else {
                        res.push(fixer.insertTextAfter(prevSemi, ","));
                    }
                    res.push(fixer.replaceText(type, ""));
                }

                return res;
            };
        }
1167657
public final List<SequenceLabel> getSequencesFromSpans(final String[] tokens,
      final Span[] seqSpans) {
    final List<SequenceLabel> sequences = new ArrayList<SequenceLabel>();
    for (final Span seqSpan : seqSpans) {
      final String seqString = seqSpan.getCoveredText(tokens);
      final String seqType = seqSpan.getType();
      final SequenceLabel sequence = this.sequenceFactory
          .createSequence(seqString, seqType, seqSpan);
      sequences.add(sequence);
    }
    return sequences;
  }
240268
@Override
	public List<Token> tokenize(String s) {
		List<Token> result = new ArrayList<Token>();
		result.add(new UndefinedToken(s));

		for (PredefinedTokenDefinition predefinedTokenDefinition : _predefinedTokenDefitions) {
			Set<Pattern> patterns = predefinedTokenDefinition.getTokenRegexPatterns();
			for (Pattern pattern : patterns) {
				for (ListIterator<Token> it = result.listIterator(); it.hasNext();) {
					Token token = it.next();
					if (token instanceof UndefinedToken) {
						List<Token> replacementTokens = tokenizeInternal(token.getString(), predefinedTokenDefinition,
								pattern);
						if (replacementTokens.size() > 1) {
							it.remove();
							for (Token newToken : replacementTokens) {
								it.add(newToken);
							}
						}
					}
				}
			}
		}

		return result;
	}
1254450
function(comment){
    var ast = [];
    if (!comment.buffer) return;
    if (this.pp) push.apply(ast, this.prettyIndent(1, true));
    push.apply(ast, this.buffer('<!--' + (comment.val || '')));
    push.apply(ast, this.visit(comment.block, comment));
    if (this.pp) push.apply(ast, this.prettyIndent(1, true));
    push.apply(ast, this.buffer('-->'));
    return ast;
  }
1072916
def combine_comments(comments):
    '''
    
    '''
    if isinstance(comments, list):
        for idx in range(len(comments)):
            if not isinstance(comments[idx], six.string_types):
                comments[idx] = six.text_type(comments[idx])
    else:
        if not isinstance(comments, six.string_types):
            comments = [six.text_type(comments)]
        else:
            comments = [comments]
    return ' '.join(comments).strip()
297835
public Parse parse(Parse tokens) {
    Parse p = parse(tokens,1)[0];
    setParents(p);
    return p;
  }
1138379
function(comment){
    var ast = [];
    if (!comment.buffer) return;
    if (this.pp) push.apply(ast, this.prettyIndent(1, true));
    push.apply(ast, this.buffer('<!--' + comment.val + '-->'));
    return ast;
  }
724623
function Tokenizer(str, tags) {
            this.str = str;
            this.index = 0;
            this.len = str.length;
            this.lineno = 0;
            this.colno = 0;

            this.in_code = false;

            tags = tags || {};
            this.tags = {
                BLOCK_START: tags.blockStart || BLOCK_START,
                BLOCK_END: tags.blockEnd || BLOCK_END,
                VARIABLE_START: tags.variableStart || VARIABLE_START,
                VARIABLE_END: tags.variableEnd || VARIABLE_END,
                COMMENT_START: tags.commentStart || COMMENT_START,
                COMMENT_END: tags.commentEnd || COMMENT_END
            };
        }