Get tokens for original & new filenames.

@param string[] $diffLineList
@param int $currentLine

@return Token[]
326443
private function getFilenameTokens(array $diffLineList, int $currentLine): array
    {
        $filenameTokens = [];

        // Get hunk metadata
        $hunkTokens = $this->getHunkStartTokens($diffLineList[$currentLine+2]);

        // In some cases we may have a diff with no contents (e.g. diff of svn propedit)
        if (count($hunkTokens)) {
            // Simple change
            if (4 == count($hunkTokens)) {
                $originalFilename = $this->diffNormalizer->getFilename($diffLineList[$currentLine]);
                $newFilename = $this->diffNormalizer->getFilename($diffLineList[$currentLine + 1]);

            // File deletion
            } elseif (Token::FILE_DELETION_LINE_COUNT === $hunkTokens[0]->getType()) {
                $originalFilename = $this->diffNormalizer->getFilename($diffLineList[$currentLine]);
                $newFilename = '';

            // File creation
            } else {
                $originalFilename = '';
                $newFilename = $this->diffNormalizer->getFilename($diffLineList[$currentLine + 1]);
            }

            $filenameTokens = [
                new Token(Token::ORIGINAL_FILENAME, $originalFilename),
                new Token(Token::NEW_FILENAME, $newFilename)
            ];
        }

        return $filenameTokens;
    }
838342
protected function parseListContent(TokenList $tokenList) {
        $lastToken = $tokenList->last();
        if ($lastToken->getName() === $this->name) {
            $tokenList->removeTokenAt(-1); // Remove last LIST_BLOCK (content)

            $content = $lastToken->get('content', '');

            $containsParagraphs = $tokenList->last()->get('forceParagraph', false);

            if (preg_match('/(?:\n(?:[ \t]*(?=\n))?){2,}/i', $content) === 1) {
                $containsParagraphs = true;

                // Remove starting spaces or tab
                $content = explode("\n", $content);
                $pad = null;
                foreach ($content as $index => &$item) {
                    // Find out lin starting space length up to 4 characters
                    if ($pad === null && $index > 0 && preg_match('/^[ \t]*$/i', $item) !== 1) {
                        if (preg_match('/^[ ]{0,4}/', $item, $matches) === 1) {
                            $pad = strlen($matches[0]);
                        } else {
                            $pad = 4;
                        }
                    }

                    // Remove line starting space or tab characters
                    if ($pad !== null) {
                        $item = preg_replace('/^(?: {' . $pad . '}|\t)/', '', $item);
                    }
                }

                $content = implode("\n", $content);
            }

            $content = "\n" . $content . "\n";

            $contentTokenList = $this->parseContent($content, null, true);

            // Remove beginning and ending new line feed
            if ($contentTokenList->first()->getName() === EolRule::NAME
                && $contentTokenList->last()->getName() === EolRule::NAME
            ) {
                $contentTokenList->removeTokenAt(0);  // Remove first EOL
                $contentTokenList->removeTokenAt(-1); // Remove last EOL
            }

            // Remove paragraphs
            if (!$containsParagraphs
                && $contentTokenList->first()->getName() === PRule::NAME . AbstractBlockRule::OPEN
                && $contentTokenList->last()->getName() === PRule::NAME . AbstractBlockRule::CLOSE
            ) {
                $contentTokenList->removeTokenAt(0);  // Remove first P
                $contentTokenList->removeTokenAt(-1); // Remove last P
            }

            // Add new line indicator for paragraphs
            for ($i = 0; $i < count($contentTokenList); $i++) {
                $token = $contentTokenList->getTokenAt($i);
                if ($token->getName() === EolRule::NAME) {
                    $contentTokenList->removeTokenAt($i);
                    $contentTokenList->insertTokenAt($this->name . '_ITEM_EOL', $token->getPosition(), $i);
                }
            }

            // Change LIST_BLOCK to BASE token
            $startRule = $this->set->getStartRule();

            for ($i = 0; $i < count($contentTokenList); $i++) {
                if ($contentTokenList->getTokenAt($i)->getName() === $this->name) {
                    $token = $contentTokenList->getTokenAt($i);
                    $contentTokenList->removeTokenAt($i);
                    $contentTokenList->insertTokenAt($startRule->getName(), $token->getPosition(), $i)
                        ->set('content', $token->get('content', ''));
                }
            }

            $tokenList->merge($contentTokenList);
        }
    }
1281269
public function getDiffs() {
    $diffs = array();
    foreach ($this->diffs as $diff) {
      if ($diff[1] !== self::OLD) {
        $diffs[] = $diff;
      } 
    }
    return $diffs;
  }
171911
public function getReferencedFilesFromDiff(array $diff_files, bool $include_referencing_files = true)
    {
        $all_inherited_files_to_check = $diff_files;

        while ($diff_files) {
            $diff_file = array_shift($diff_files);

            $dependent_files = $this->file_reference_provider->getFilesInheritingFromFile($diff_file);

            $new_dependent_files = array_diff($dependent_files, $all_inherited_files_to_check);

            $all_inherited_files_to_check = array_merge($all_inherited_files_to_check, $new_dependent_files);
            $diff_files = array_merge($diff_files, $new_dependent_files);
        }

        $all_files_to_check = $all_inherited_files_to_check;

        if ($include_referencing_files) {
            foreach ($all_inherited_files_to_check as $file_name) {
                $dependent_files = $this->file_reference_provider->getFilesReferencingFile($file_name);
                $all_files_to_check = array_merge($dependent_files, $all_files_to_check);
            }
        }

        return array_combine($all_files_to_check, $all_files_to_check);
    }
549022
def get_files(file_tokens, cwd=None):
    
    if not file_tokens:
        return []

    token = file_tokens.pop()
    try:
        filename = token.filename
    except AttributeError:
        filename = ''

    if cwd:
        input = Input(token.alias, filename, cwd=cwd)
    else:
        input = Input(token.alias, filename)

    return [input] + get_files(file_tokens)
1323212
private function countLinesBetweenAnnotations(Annotation $current, Annotation $previous)
    {
        $tokens = $this->tokens;

        \preg_match('~(\\s+)$~', $tokens[$previous->getEndPointer()]['content'], $matches);

        $lines = $matches[1] ?? '';
        $lines .= TokenHelper::getContent(
            $this->phpcsFile,
            $previous->getEndPointer() + 1,
            $current->getStartPointer() - 1
        );

        return $this->countLines($lines);
    }
424991
public function loadListCurrentAcl(): array
    {
        $path = $this->getPathListCurrentAcl();
        if (file_exists($path)) {
            return Neon::decode(file_get_contents($path));
        }
        return [];
    }
194396
public String getCurrentShrinkedLine() {
		if (!isCurrentLine()) {
			return null;
		}
		String string = FlatFileUtils.trim(currentLine,
				getTagWidth(currentLine));
		if (string.equals("")) {
			return null;
		}
		return FlatFileUtils.shrink(string);
	}
52404
public function getLinePositions($diff)
    {
        if (empty($diff)) {
            return null;
        }

        $rtn = array();

        $diffLines = explode(PHP_EOL, $diff);

        while (count($diffLines)) {
            $line = array_shift($diffLines);

            if (substr($line, 0, 2) == '@@') {
                array_unshift($diffLines, $line);
                break;
            }
        }

        $lineNumber = 0;
        $position = 0;

        foreach ($diffLines as $diffLine) {
            if (preg_match('/@@\s+\-[0-9]+\,[0-9]+\s+\+([0-9]+)\,([0-9]+)/', $diffLine, $matches)) {
                $lineNumber = (int)$matches[1] - 1;
            }

            $rtn[$lineNumber] = $position;

            $lineNumber++;
            $position++;
        }

        return $rtn;
    }
183816
private function getHunkMeta(array $hunkTokenList): array
    {
        switch (true) {
            case Token::FILE_DELETION_LINE_COUNT === $hunkTokenList[0]->getType():
                $originalStart = 1;
                $originalCount = intval($hunkTokenList[0]->getValue());
                $newStart = intval($hunkTokenList[1]->getValue());
                $newCount = intval($hunkTokenList[2]->getValue());
                $tokensReadCount = 3;
                break;

            case Token::FILE_CREATION_LINE_COUNT === $hunkTokenList[2]->getType():
                $originalStart = intval($hunkTokenList[0]->getValue());
                $originalCount = intval($hunkTokenList[1]->getValue());
                $newStart = 1;
                $newCount = intval($hunkTokenList[2]->getValue());
                $tokensReadCount = 3;
                break;

            default:
                $originalStart = intval($hunkTokenList[0]->getValue());
                $originalCount = intval($hunkTokenList[1]->getValue());
                $newStart = intval($hunkTokenList[2]->getValue());
                $newCount = intval($hunkTokenList[3]->getValue());
                $tokensReadCount = 4;
                break;
        }

        return [
            $originalStart,
            $originalCount,
            $newStart,
            $newCount,
            $tokensReadCount
        ];
    }
1027318
public function getModifiedFiles() {
		try {
			return $this->extractFromCommand ( 'git diff --name-status HEAD', function ($array) {
				$array = trim ( preg_replace ( '!\s+!', ' ', $array ) );
				return explode ( ' ', $array );
			} );
		} catch ( \Cz\Git\GitException $e ) {
			return [ ];
		}
	}
1301834
public static function readSymmetry($fileName, $current = 1, $lineNum = 3): array
    {
        $startLine = $current - $lineNum;
        $endLine = $current + $lineNum;

        if ((int)$current < ($lineNum + 1)) {
            $startLine = 1;
            $endLine = 9;
        }

        return self::readLines($fileName, $startLine, $endLine);
    }
234433
public function parse(array $diffLineList): Changeset
    {
        $tokenList = $this->tokenizer->tokenize($diffLineList);

        $fileList = [];
        $startIndex = 0;
        $tokenCount = count($tokenList);
        for ($i = 0; $i < $tokenCount; $i++) {
            // File begin
            if (Token::ORIGINAL_FILENAME === $tokenList[$i]->getType()) {
                $startIndex = $i;
            }

            // File end, hydrate object
            if ($this->fileEnd($tokenList, $i + 1, Token::ORIGINAL_FILENAME)) {
                $fileList[] = $this->parseFile(
                    array_slice($tokenList, $startIndex, ($i - $startIndex) + 1)
                );
            }
        }

        return new Changeset($fileList);
    }
1010139
private String[] getTokens(String line) {

    // these are fine because they do not affect offsets
    line = line.trim();
    line = doubleSpaces.matcher(line).replaceAll(" ");
    // remove non printable stuff
    line = asciiHex.matcher(line).replaceAll(" ");
    line = generalBlankPunctuation.matcher(line).replaceAll(" ");

    // separate question and exclamation marks
    line = qexc.matcher(line).replaceAll(" $1 ");
    // separate dash if before or after space
    line = spaceDashSpace.matcher(line).replaceAll(" $1 ");
    // tokenize everything but these characters [^\p{Alnum}s.'`,-?!/]
    line = specials.matcher(line).replaceAll(" $1 ");

    // do not separate multidots
    line = generateMultidots(line);

    // separate "," except if within numbers (1,200)
    line = noDigitComma.matcher(line).replaceAll("$1 $2");
    line = commaNoDigit.matcher(line).replaceAll("$1 $2");
    // separate pre and post digit
    line = digitCommaNoDigit.matcher(line).replaceAll("$1 $2 $3");
    line = noDigitCommaDigit.matcher(line).replaceAll("$1 $2 $3");

    // contractions it's, l'agila, c'est, don't
    line = treatContractions(line);
    // exceptions for period tokenization
    line = this.nonBreaker.TokenizerNonBreaker(line);

    // restore multidots
    line = restoreMultidots(line);
    // urls
    // TODO normalize URLs after tokenization for offsets
    line = detokenizeURLs(line);
    line = beginLink.matcher(line).replaceAll("$1://");
    line = endLink.matcher(line).replaceAll("$1$2");

    // these are fine because they do not affect offsets
    line = line.trim();
    line = doubleSpaces.matcher(line).replaceAll(" ");
    line = detokenParagraphs.matcher(line).replaceAll("$1$2");

    if (DEBUG) {
      System.out.println("->Tokens:" + line);
    }
    final String[] tokens = line.split(" ");
    return tokens;
  }
67227
private function _countLinesOfCode() {
		// We get the previous token (T_WHITESPACE, T_COMMENT, ... ignored);
		$currentToken = $this->tokenizer->getCurrentToken();
		$previousToken = $this->tokenizer->peekPrvsValidToken();

		// If the previous token is not the new line (empty line), we suppose we have some code
		if ($previousToken !== null) {
			if (!$this->tokenizer->checkToken($previousToken, T_NEW_LINE) && ($previousToken->line === $currentToken->line)) {
				$this->_ncssTotalLinesOfCode ++;
				$this->_ncssFileLinesOfCode ++;
			}
		}
	}
580694
protected function consumeFencedCode($lines, $current) {
		list($block, $i) = parent::consumeFencedCode($lines, $current);

		if (empty($block['language'])) {
			$block['language'] = 'php';
		}
		return [$block, $i];
	}
850572
public static function lineMatch(
        array $lines,
        string $prefix,
        string $suffix,
        bool $doTrim,
        bool $removePrefixSuffix = false
    ): array {
        $result = [];
        foreach ($lines as $line) {
            $isHit = true;
            if ($prefix !== '' && $prefix !== null && Str::startsWith(trim($line), $prefix)) {
                if ($removePrefixSuffix) {
                    $line = substr(trim($line), strlen($prefix));
                }
            } else {
                $isHit = false;
            }
            if ($suffix !== '' && $suffix !== null && Str::endsWith(trim($line), $suffix)) {
                if ($removePrefixSuffix) {
                    $line = substr(trim($line), 0, -strlen($suffix));
                }
            } else {
                $isHit = false;
            }
            if ($isHit) {
                if ($doTrim) {
                    $line = trim($line);
                }
                $result[] = $line;
            }
        }
        return $result;
    }