@param $originalTokens

@return array
337583
public static function removeTokenWhitespace(array &$originalTokens)
    {
        $tokens = [];
        foreach ($originalTokens as $i => &$token) {
            if ($token[Tokenizer::TOKEN_TYPE] !== Tokenizer::TOKEN_TYPE_WHITESPACE) {
                $token['i'] = $i;
                $tokens[] = $token;
            }
        }

        return $tokens;
    }
112784
public function getTokens() : array
    {
        return [
            'T_N' => self::T_N,
            'T_I' => self::T_I,
            'T_V' => self::T_V,
            'T_X' => self::T_X,
            'T_L' => self::T_L,
            'T_C' => self::T_C,
            'T_D' => self::T_D,
            'T_M' => self::T_M,
        ];
    }
193219
public function getTokenParsers()
    {
        return [
            new PageTokenParser,
            new PartialTokenParser,
            new ContentTokenParser,
            new PutTokenParser,
            new PlaceholderTokenParser,
            new DefaultTokenParser,
            new FrameworkTokenParser,
            new ComponentTokenParser,
            new FlashTokenParser,
            new ScriptsTokenParser,
            new StylesTokenParser,
        ];
    }
486036
public function getTokens()
    {
        $conditions = '';
        $params = array();
        $records = Oauth_TokenRecord::model()->findAll($conditions, $params);
        return Oauth_TokenModel::populateModels($records);
    }
148486
private function removeUnwantedTokens(array $subTokens): array
    {
        return array_filter($subTokens, function (array $subToken): bool {
            switch ($subToken['code']) {
                case T_VARIABLE:
                    $return = (new PropertyHelper($this->file))->isProperty($subToken['pointer']);
                    break;

                case T_USE:
                    $return = UseStatementHelper::isTraitUse($this->file, $subToken['pointer']);
                    break;

                default:
                    $return = true;
            }

            return $return;
        });
    }
654417
public function replaceTokens($tokens){
        $keys = array_keys($tokens);
        $values = array_values($tokens);

        return str_replace($keys, $values, $this->getExpression());
    }
608547
protected function filterTokens(array $tokens)
    {
        $filteredTokens = array_filter($tokens, function (Token $t) {
            return $t->getType() !== TokenType::Whitespace;
        });

        // Return the array values only, because array_filter preserves the keys
        return array_values($filteredTokens);
    }
57378
public function getTokens(): array
    {
        $response = $this
            ->httpClient
            ->get(
                sprintf(
                    '/%s/tokens',
                    $this->getAppUUID()->getId()
                ),
                'get',
                [],
                [],
                Http::getApisearchHeaders($this)
            );

        self::throwTransportableExceptionIfNeeded($response);

        return array_map(function (array $token) {
            return Token::createFromArray($token);
        }, $response['body']);
    }
258222
private function loadSubTokenPositions(): array
    {
        return TokenHelper::findNextAll(
            $this->file->getBaseFile(),
            $this->sortedTokens,
            $this->stackPos,
            $this->token['scope_closer']
        );
    }
982097
public function arrayUnmerge($original, $modified)
    {
        // return key/value pairs for keys in $modified but not in $original
        // return key/value pairs for keys in both $modified and $original, but values differ
        // ignore keys that are in $original but not in $modified

        if (empty($original) || !is_array($original)) {
            $original = array();
        }

        if (empty($modified) || !is_array($modified)) {
            $modified = array();
        }

        return array_udiff_assoc($modified, $original, array(__CLASS__, 'compareElements'));
    }
794867
public static function create(array $tokens) {
        HashMap::assert($tokens, 'tokens');

        if (empty($tokens['ORDER'])) return [];

        $headers = [];
        $orderQueryArr = array_map(function($order){
            $query = end($order['no_quotes']['parts']);
            if (empty($query)) return null;
            return !isset($order['direction']) ? $query : $query . ' ' . $order['direction'];
        },$tokens['ORDER']);
        array_push($headers, 'Order: '.implode(',',$orderQueryArr));
        return $headers;
    }
1086219
public function mappingExamples($tokens) {

    if ($this->ogVersion == 1) {
      $groups = og_get_all_group();
      $ogEntities = og_load_multiple($groups);
      $OGroles = og_roles(0);

      $rows = array();
      foreach ($ogEntities as $group) {
        foreach ($OGroles as $rid => $role) {
          $example =   "<code>ou=IT,dc=myorg,dc=mytld,dc=edu|gid=" . $group->gid . ',rid=' . $rid . '</code><br/>' .
            '<code>ou=IT,dc=myorg,dc=mytld,dc=edu|group-name=' . $group->label . ',role-name=' . $role . '</code>';
          $rows[] = array(
            $group->label,
            $group->gid,
            $role,
            $example,
          );
        }
      }

      $variables = array(
      'header' => array('Group Name', 'OG Group ID', 'OG Membership Type', 'example'),
      'rows' => $rows,
      'attributes' => array(),
      );
    }
    else {

      /**
       * OG 7.x-2.x mappings:
       * $entity_type = $group_type,
       * $bundle = $group_bundle
       * $etid = $gid where edid is nid, uid, etc.
       *
       * og group is: entity_type (eg node) x entity_id ($gid) eg. node:17
       * group identifier = group_type:gid; aka entity_type:etid e.g. node:17
       *
       * membership identifier is:  group_type:gid:entity_type:etid
       * in our case: group_type:gid:user:uid aka entity_type:etid:user:uid e.g. node:17:user:2
       *
       * roles are simply rids ((1,2,3) and names (non-member, member, and administrator member) in og_role table
       * og_users_roles is simply uid x rid x gid
       *
       * .. so authorization mappings should look like:
       *    <ldap group>|group_type:gid:rid such as staff|node:17:2
       */

      $og_fields = field_info_field(OG_GROUP_FIELD);
      $rows = array();
      $role_name = OG_AUTHENTICATED_ROLE;

      if (!empty($og_fields['bundles'])) {
        foreach ($og_fields['bundles'] as $entity_type => $bundles) {

          foreach ($bundles as $i => $bundle) {

            $query = new EntityFieldQuery();
            $query->entityCondition('entity_type', $entity_type)
              ->entityCondition('bundle', $bundle)
              ->range(0, 5)
              ->addMetaData('account', user_load(1)); // run the query as user 1
            $result = $query->execute();
            $entities = entity_load($entity_type, array_keys($result[$entity_type]));
            $i=0;
            foreach ($entities as $entity_id => $entity) {
              $i++;
              $rid = ldap_authorization_og2_rid_from_role_name($entity_type, $bundle, $entity_id, OG_AUTHENTICATED_ROLE);
              $title = (is_object($entity) && property_exists($entity, 'title')) ? $entity->title : '';
              $middle = ($title && $i < 3) ? $title : $entity_id;
              $group_role_identifier = ldap_authorization_og_authorization_id($middle, $rid, $entity_type);
              $example = "<code>ou=IT,dc=myorg,dc=mytld,dc=edu|$group_role_identifier</code>";
              $rows[] = array("$entity_type $title - $role_name", $example);

            }

          }
        }
      }

      $variables = array(
        'header' => array('Group Entity - Group Title - OG Membership Type', 'example'),
        'rows' => $rows,
        'attributes' => array(),
      );
    }

    $table = theme('table', $variables);
    $link = l(t('admin/config/people/ldap/authorization/test/og_group'), 'admin/config/people/ldap/authorization/test/og_group');

$examples =
<<<EOT

<br/>
Examples for some (or all) existing OG Group IDs can be found in the table below.
This is complex.  To test what is going to happen, uncheck "When a user logs on" in IV.B.
and use $link to see what memberships sample users would receive.

$table

EOT;
    $examples = t($examples, $tokens);
    return $examples;
  }
662456
private function queryTokens()
    {
        $tokens = $this->getTokens();

        $queryNorm = $this->getQueryNorm($tokens);
        $results = array();
        foreach ($tokens as $token) {
            $results[$token] = $this->getResultsForToken($token, $queryNorm);
        }
        return $results;
    }
127279
public function getTagCounts(array $tagTokens): array
    {
        $tagCounts = [];

        foreach ($tagTokens as $tagToken) {
            $tagName = $tagToken['content'];

            if (!array_key_exists($tagName, $tagCounts)) {
                $tagCounts[$tagName] = 0;
            }

            ++$tagCounts[$tagName];
        }

        return $tagCounts;
    }
850778
private function grabAttribsBeforeMethodToken(Tokens $tokens, $index)
    {
        static $tokenAttribsMap = [
            T_PRIVATE   => 'visibility',
            T_PROTECTED => 'visibility',
            T_PUBLIC    => null, // destroy T_PUBLIC token. This is literally the only change from the original
            T_ABSTRACT  => 'abstract',
            T_FINAL     => 'final',
            T_STATIC    => 'static',
        ];

        return $this->grabAttribsBeforeToken(
            $tokens,
            $index,
            $tokenAttribsMap,
            [
                'abstract'   => null,
                'final'      => null,
                'visibility' => null,
                'static'     => null,
            ]
        );
    }
207823
private function getClassNamesFromTokens(array $tokens) : array
    {
        for ($i = 0;$i < count($tokens);$i++) {
            // Skip literals
            if (is_string($tokens[$i])) {
                continue;
            }

            $className = '';

            switch ($tokens[$i][0]) {
                case T_NAMESPACE:
                    $namespace = '';

                    // Collect all the namespace parts and separators
                    while (isset($tokens[++$i][1])) {
                        if (in_array($tokens[$i][0], [T_NS_SEPARATOR, T_STRING])) {
                            $namespace .= $tokens[$i][1];
                        }
                    }

                    break;
                case T_CLASS:
                    $isClassConstant = false;

                    // Scan previous tokens to see if they're double colons, which would mean this is a class constant
                    for ($j = $i - 1;$j >= 0;$j--) {
                        if (!isset($tokens[$j][1])) {
                            break;
                        }

                        if ($tokens[$j][0] === T_DOUBLE_COLON) {
                            $isClassConstant = true;
                            break 2;
                        } elseif ($tokens[$j][0] === T_WHITESPACE) {
                            // Since we found whitespace, then we know this isn't a class constant
                            break;
                        }
                    }

                    // Get the class name
                    while (isset($tokens[++$i][1])) {
                        if ($tokens[$i][0] === T_STRING) {
                            $className .= $tokens[$i][1];
                            break;
                        }
                    }

                    $classNames[] = ltrim($namespace . '\\' . $className, '\\');
                    break 2;
            }
        }

        return $classNames;
    }
1025
protected function getTokenStrings(int $row, array $tokens): array
    {
        $pieces = [];
        foreach ($tokens as $key => $token) {
            if ($token['line'] > $row) {
                break;
            }
            if ($token['line'] < $row) {
                continue;
            }
            if ($this->verbose) {
                $type = $token['type'];
                $content = $token['content'];
                $content = '`' . str_replace(["\r\n", "\n", "\r", "\t"], ['\r\n', '\n', '\r', '\t'], $content) . '`';

                unset($token['type']);
                unset($token['content']);
                $token['content'] = $content;

                $tokenList = [];
                foreach ($token as $k => $v) {
                    if (is_array($v)) {
                        if (empty($v)) {
                            continue;
                        }
                        $v = json_encode($v);
                    }
                    $tokenList[] = $k . '=' . $v;
                }
                $pieces[] = $type . ' (' . $key . ') ' . implode(', ', $tokenList);
            } else {
                $pieces[] = $token['type'];
            }
        }
        if ($this->verbose) {
            return $pieces;
        }
        return [implode(' ', $pieces)];
    }