use set of tokens for (more or less) O(1) lookup on multiple channels
437841
def parse_outgoing_tokens(tokens)
      case tokens
      when NilClass
        []
      when String
        tokens.split(/[,\s]+/)
      when Array
        tokens
      end.to_set
    end
491375
function getIdentifiersFromTokens (tokens) {
  const identifiers = {}

  /**
   * Predicate to determine if token is an identifier.
   *
   * @private
   * @param {Object} token
   * @return {Boolean}
   */

  const isIdentifier = token => 'Identifier' == token.type

  /**
   * Mark token as a function identifier.
   *
   * @private
   * @param {Object} token
   * @param {Number} index
   * @return {Object} token
   */

  const markFunction = (token, index) => {
    const next = tokens[index + 1] || null
    token.isFunction = ( 'Identifier' == token.type
                        && 'object' == typeof next && next
                        && 'Punctuator' == next.type
                        && '(' == next.value
                          ? true : false )
    return token
  }

  /**
   * Mark token as a object identifier.
   *
   * @private
   * @param {Object} token
   * @param {Number} index
   * @return {Object} token
   */

  const markObject = (token, index) => {
    const next = tokens[index + 1] || null
    token.isObject = ( 'Identifier' == token.type
                      && 'object' == typeof next && next
                      && 'Punctuator' == next.type
                      && '.' == next.value
                        ? true : false )
    return token
  }

  /**
   * Assign token value to identifierss map.
   *
   * @private
   * @param {Object} map
   * @param {Object} token
   * @return {Object} map
   */

  const assign = (map, token) => {
    const value = token.value
    if (token.isFunction)
      map[value] = _ => ''
    else if (token.isObject)
      map[value] = {}
    else
      map[value] = ''
    return map
  }

  // resolve identifierss and return map
  return ( tokens
          .map((t, i) => markFunction(t, i))
          .map((t, i) => markObject(t, i))
          .filter(t => isIdentifier(t))
          .reduce((map, t) => assign(map, t), identifiers) )
}
1195010
function step1c(token) {
    if(categorizeGroups(token).substr(-2, 1) == 'V') {
        if(token.substr(-1) == 'y')
            return token.replace(/y$/, 'i');
    }
   
    return token;
}
1185828
def join(tokens, start, result):
    """"""
    texts = []
    if len(result) > 0:
        for e in result:
            for child in e.iter():
                if child.text is not None:
                    texts.append(child.text)
        return [E(result[0].tag, ' '.join(texts))]
963798
def channel_name_to_id(self, channel_name, force_lookup=False):
        
        if force_lookup or not self.channel_name_id_map:
            channels = self.channels_list()['channels']
            self.channel_name_id_map = {channel['name']: channel['id'] for channel in channels}
        channel = channel_name.startswith('#') and channel_name[1:] or channel_name
        return self.channel_name_id_map.get(channel)
1028537
function(options, streamChannels) {
    streamChannels.trackedKeywords = [];
    streamChannels.channels = {};
    streamChannels.channelsKeywordsLowerCased = {};
    streamChannels.channelsKeywordsLowerCasedRegExpSafe = {};
    streamChannels.channelsKeywordsLowerCasedRegExp = {};

    if (options.track instanceof Array || typeof options.track === 'string') {
      options.track = {
        "default": options.track
      };
    }

    for (var channel in options.track) {
      streamChannels.channels[channel] = keywordsToArray(options.track[channel], []);//process the options.track[channel] to make sure it will be an array of keywords
      streamChannels.channelsKeywordsLowerCased[channel] = streamChannels.channels[channel].map(function(item){
        return item.toLowerCase();
      });
      streamChannels.channelsKeywordsLowerCasedRegExpSafe[channel] = streamChannels.channelsKeywordsLowerCased[channel].map(function(item){
        return regExpEscape(item);//escape the lower cased keywords so they will be regexp proof
      });
      streamChannels.channelsKeywordsLowerCasedRegExp[channel] = streamChannels.channelsKeywordsLowerCased[channel].length > 0 ? new RegExp(streamChannels.channelsKeywordsLowerCasedRegExpSafe[channel].join('|'),'g') : null;//create the full text search regexp on lower cased keywords
      streamChannels.channels[channel].forEach(function(item){
        if(streamChannels.trackedKeywords.indexOf(item) === -1){
          streamChannels.trackedKeywords.push(item);
        }
      });
    }

    /*
     * Prepares keywords to an array of keywords like 'foo,bar' -> ['foo','bar'] also ['foo,bar','toto'] -> ['foo','bar','toto'] also 'foo,bar,toto' -> ['foo','bar','toto']
     * @param {Array|String} keywords
     * @returns {Array}
     */
    function keywordsToArray(keywords, result) {
      if (typeof keywords === 'string') {
        result = result.concat(keywords.split(','));
      }
      else if (keywords instanceof Array) {
        if (keywords.length > 0) {
          for (var i = 0; i < keywords.length; i++) {
            result = keywordsToArray(keywords[i], result);
          }
        }
      }
      return result;
    }

    return streamChannels;
  }
222488
def build_channel(namespace, name, user_ids):
    
    ids = ','.join(map(str, user_ids))
    return "{0}:{1}#{2}".format(namespace, name, ids)
1168416
def set_to_tuple(tokens):
    """"""
    internal_assert(len(tokens) == 1, "invalid set maker tokens", tokens)
    if "comp" in tokens or "list" in tokens:
        return "(" + tokens[0] + ")"
    elif "test" in tokens:
        return "(" + tokens[0] + ",)"
    else:
        raise CoconutInternalException("invalid set maker item", tokens[0])
5347
function (source, nextToken) {
      var tm = this,
          tokenizerRe,
          lastIndex,
          matches,
          textType;

      // Initialize for first run
      // if no template passed assume first pass
      // and use the main template.
      if (typeof source === "undefined") {
        source = tm.template;
      }

      // if no token passed, this is an open tag lookup
      if (typeof nextToken === "undefined") {
        tm._tokens = [];
        nextToken = tm.lex.tagOpen;
      }

      while (source) {
        // Use created token.
        tokenizerRe = nextToken.compiledRe;
        matches = tokenizerRe.exec(source);

        lastIndex = tokenizerRe.lastIndex;
        tokenizerRe.lastIndex = 0;

        // This guy matched. So let's push the tokens.
        if (matches) {

          // Flags to see if code or text
          if (nextToken === tm.lex.tagOpen) {
            // if the match was done using the tagOpen, then everything we matched is normal text.
            textType = "text";
          } else {
            // assume the match was a closing tag, so everything we have is code.
            textType = "code";
          }

          // if the token is a tagOpen, or tagClose, we need to push it with the text value matched.
          if (nextToken.hasText) {
            tm._tokens.push({
              type : textType,
              value : matches[1]
            });
          } else {
            // push the token name into the array to mark the open/close/print status on it.
            tm._tokens.push({
              type : nextToken.name
            });
          }

          // A print indicator just shows open prints, so we buffer the open print.
          if (nextToken === tm.lex.printIndicator) {
            tm._openPrints.push(tm._openBrackets);
          }

          // Closing an open print needs to count for open brackets, so we can separate the code blocks.
          if (nextToken === tm.lex.tagClose && tm._openPrints.length > 0) {
            tm._countBrackets(matches[1]);
          }

          // We matched, so let's keep going.
          // from where we left with our new next token.
          source    = source.substr(lastIndex);
          nextToken = tm._nextToken(nextToken);

        //No match:
        } else {
          // We couldn't match anything else. So: Either it's a syntax error,
          // or it's just text. If we were awaiting another token we fail, otherwise
          // assume it's just text.
          if (nextToken == tm.lex.tagOpen) {
            tm._tokens.push({
              type  : "text",
              value : source
            });

            source = "";

          } else {
            // If it wasn't optional. GO!
            if (nextToken.optional) {
              // So continue from where we left with our new next token.
              source    = source.substr(lastIndex);
              nextToken = tm._nextToken(nextToken);

            // so it was not a printing tag, so something went wrong.
            } else {
              throw "Syntax Error: Expecting " + nextToken.name + " ... Nowhere to be found. So I was like, what?";
            }
          }
        }
      }
    }
1254100
def get_matched_token(self, match):
        """"""
        match_groupdict = match.groupdict()
        for group in self.groups:
            if match_groupdict[group] is not None:
                token, match_type = self.groups[group]
                return (token, match_type, group)
1130660
function markBrackets(tokens) {
  let tokensLength = tokens.length;
  let ps = []; // Parentheses
  let sbs = []; // Square brackets
  let cbs = []; // Curly brackets
  let t; // Current token

  // For every token in the token list, if we meet an opening (left)
  // bracket, push its index number to a corresponding array.
  // If we then meet a closing (right) bracket, look at the corresponding
  // array. If there are any elements (records about previously met
  // left brackets), take a token of the last left bracket (take
  // the last index number from the array and find a token with
  // this index number) and save right bracket's index as a reference:
  for (var i = 0; i < tokensLength; i++) {
    t = tokens[i];
    var type = t.type;

    if (type === TokenType.LeftParenthesis) {
      ps.push(i);
    } else if (type === TokenType.RightParenthesis) {
      if (ps.length) {
        t.left = ps.pop();
        tokens[t.left].right = i;
      }
    } else if (type === TokenType.LeftSquareBracket) {
      sbs.push(i);
    } else if (type === TokenType.RightSquareBracket) {
      if (sbs.length) {
        t.left = sbs.pop();
        tokens[t.left].right = i;
      }
    } else if (type === TokenType.LeftCurlyBracket) {
      cbs.push(i);
    } else if (type === TokenType.RightCurlyBracket) {
      if (cbs.length) {
        t.left = cbs.pop();
        tokens[t.left].right = i;
      }
    }
  }
}
359017
private void getImChannels(String slackToken, int limit, String nextCursor) {
        try {
            Event event = restTemplate.getForEntity(slackApiEndpoints.getImListApi(), Event.class,
                    slackToken, limit, nextCursor).getBody();
            imChannelIds.addAll(Arrays.stream(event.getIms()).map(Channel::getId).collect(Collectors.toList()));
            if (event.getResponseMetadata() != null &&
                    !StringUtils.isEmpty(event.getResponseMetadata().getNextCursor())) {
                Thread.sleep(5000L); // sleep because its a tier 2 api which allows only 20 calls per minute
                getImChannels(slackToken, limit, event.getResponseMetadata().getNextCursor());
            }
        } catch (Exception e) {
            logger.error("Error fetching im channels for the bot: ", e);
        }
    }
918183
function step1b$1(token) {
                var r1 = getR1(token);

                if (!r1) {
                    return token;
                }

                if (token.match(/(b|c|d|f|g|h|j|l|m|n|o|p|r|t|v|y|z)s$/)) {
                    return token.slice(0, -1);
                }

                if (token.match(/([^aeiouyæåø]k)s$/)) {
                    return token.slice(0, -1);
                }

                return token;
            }
437414
def process_tokens(self, tokens):
        
        """
        for (tok_type, token, (start_row, _), _, _) in tokens:
            if tok_type == tokenize.STRING:
                self._check_string(token, start_row)
776046
def ibatch(iterable, size):
    """"""
    source = iter(iterable)
    while True:
        batch = itertools.islice(source, size)
        yield itertools.chain([next(batch)], batch)
1157437
private void getMultiWordSpans(final String[] tokens, final List<WF> wfs,
      final List<ixa.kaflib.Span<WF>> tokenSpans) {
    final Span[] multiWordSpans = this.multiWordMatcher
        .multiWordsToSpans(tokens);
    int counter = 0;
    for (final Span mwSpan : multiWordSpans) {
      final Integer fromIndex = mwSpan.getStart() - counter;
      final Integer toIndex = mwSpan.getEnd() - counter;
      // add to the counter the length of the span removed
      counter = counter + tokenSpans.subList(fromIndex, toIndex).size() - 1;
      // create multiword targets and Span
      final List<WF> wfTargets = wfs
          .subList(mwSpan.getStart(), mwSpan.getEnd());
      final ixa.kaflib.Span<WF> multiWordSpan = KAFDocument
          .newWFSpan(wfTargets);
      // remove the token Spans to be replaced by the multiword span
      tokenSpans.subList(fromIndex, toIndex).clear();
      // add the new Span containing several WFs (multiWordSpan)
      // the counter is used to allow matching the spans to the
      // tokenSpans list indexes
      tokenSpans.add(fromIndex, multiWordSpan);
    }
  }
1254772
def match_string(self, tokens, item):
        """"""
        prefix, name = tokens
        return self.match_mstring((prefix, name, None), item, use_bytes=prefix.startswith("b"))