Take in a sequence of indices and transform it back into a tokenized utterance
498831
def devectorize_utterance(self, utterance):
        
        utterance = self.swap_pad_and_zero(utterance)
        return self.ie.inverse_transform(utterance).tolist()
1048970
def vectorize_dialogues(self, dialogues):
        
        return np.array([self.vectorize_dialogue(d) for d in dialogues])
1261521
def next_tokens_in_sequence(observed, current):
    
    idx = 0
    for word in current:
        if observed[idx:].count(word) != 0:
            found_pos = observed.index(word, idx)
            idx = max(idx + 1, found_pos)
        # otherwise, don't increment idx
    if idx < len(observed):
        return observed[idx:]
    else:
        return []
625682
def get_strings_from_utterance(tokenized_utterance: List[Token]) -> Dict[str, List[int]]:
    
    string_linking_scores: Dict[str, List[int]] = defaultdict(list)

    for index, token in enumerate(tokenized_utterance):
        for string in ATIS_TRIGGER_DICT.get(token.text.lower(), []):
            string_linking_scores[string].append(index)

    token_bigrams = bigrams([token.text for token in tokenized_utterance])
    for index, token_bigram in enumerate(token_bigrams):
        for string in ATIS_TRIGGER_DICT.get(' '.join(token_bigram).lower(), []):
            string_linking_scores[string].extend([index,
                                                  index + 1])

    trigrams = ngrams([token.text for token in tokenized_utterance], 3)
    for index, trigram in enumerate(trigrams):
        if trigram[0] == 'st':
            natural_language_key = f'st. {trigram[2]}'.lower()
        else:
            natural_language_key = ' '.join(trigram).lower()
        for string in ATIS_TRIGGER_DICT.get(natural_language_key, []):
            string_linking_scores[string].extend([index,
                                                  index + 1,
                                                  index + 2])
    return string_linking_scores
424823
def forward(self, X):
        """"""
        # Identify the first non-zero integer from the right (i.e., the length
        # of the sequence before padding starts).
        batch_size, max_seq = X.shape[0], X.shape[1]
        seq_lengths = torch.zeros(batch_size, dtype=torch.long)
        for i in range(batch_size):
            for j in range(max_seq - 1, -1, -1):
                if not torch.all(X[i, j] == 0):
                    seq_lengths[i] = j + 1
                    break

        # Sort by length because pack_padded_sequence requires it
        # Save original order to restore before returning
        seq_lengths, perm_idx = seq_lengths.sort(0, descending=True)
        X = X[perm_idx]
        inv_perm_idx = torch.tensor(
            [i for i, _ in sorted(enumerate(perm_idx), key=lambda idx: idx[1])],
            dtype=torch.long,
        )

        # Encode and pack input sequence
        X_packed = rnn_utils.pack_padded_sequence(
            self.encoder.encode(X), seq_lengths, batch_first=True
        )

        # Run LSTM
        outputs, (h_t, c_t) = self.lstm(X_packed)

        # Unpack and reduce outputs
        outputs_unpacked, _ = rnn_utils.pad_packed_sequence(outputs, batch_first=True)
        reduced = self._reduce_output(outputs_unpacked, seq_lengths)
        return reduced[inv_perm_idx, :]
579313
def convert_tokens_to_ids(self, tokens):
        """"""
        ids = []
        for token in tokens:
            ids.append(self.vocab[token])
        if len(ids) > self.max_len:
            logger.warning(
                "Token indices sequence length is longer than the specified maximum "
                " sequence length for this BERT model ({} > {}). Running this"
                " sequence through BERT will result in indexing errors".format(len(ids), self.max_len)
            )
        return ids
441680
def tokenize(self):

        

        self.tokens = []
        self.terms = OrderedDict()

        # Generate tokens.
        for token in utils.tokenize(self.text):

            # Ignore stopwords.
            if token['unstemmed'] in self.stopwords:
                self.tokens.append(None)

            else:

                # Token:
                self.tokens.append(token)

                # Term:
                offsets = self.terms.setdefault(token['stemmed'], [])
                offsets.append(token['offset'])
719705
def dense_to_human_readable(dense_repr: Sequence[Sequence[int]], index_to_label: Dict[int, str]) -> List[List[str]]:
    

    transcripts = []
    for dense_r in dense_repr:
        non_empty_phonemes = [phn_i for phn_i in dense_r if phn_i != 0]
        transcript = [index_to_label[index] for index in non_empty_phonemes]
        transcripts.append(transcript)

    return transcripts
898965
def get_times_from_utterance(utterance: str,
                             char_offset_to_token_index: Dict[int, int],
                             indices_of_approximate_words: Set[int]) -> Dict[str, List[int]]:
    

    pm_linking_dict = _time_regex_match(r'\d+pm',
                                        utterance,
                                        char_offset_to_token_index,
                                        pm_map_match_to_query_value,
                                        indices_of_approximate_words)

    am_linking_dict = _time_regex_match(r'\d+am',
                                        utterance,
                                        char_offset_to_token_index,
                                        am_map_match_to_query_value,
                                        indices_of_approximate_words)

    oclock_linking_dict = _time_regex_match(r"\d+ o'clock",
                                            utterance,
                                            char_offset_to_token_index,
                                            lambda match: digit_to_query_time(match.rstrip(" o'clock")),
                                            indices_of_approximate_words)

    hours_linking_dict = _time_regex_match(r"\d+ hours",
                                           utterance,
                                           char_offset_to_token_index,
                                           lambda match: [int(match.rstrip(" hours"))],
                                           indices_of_approximate_words)


    times_linking_dict: Dict[str, List[int]] = defaultdict(list)
    linking_dicts = [pm_linking_dict, am_linking_dict, oclock_linking_dict, hours_linking_dict]

    for linking_dict in linking_dicts:
        for key, value in linking_dict.items():
            times_linking_dict[key].extend(value)

    return times_linking_dict
48568
def _lex_fortran(self, match, ctx=None):
        """"""
        lexer = FortranLexer()
        text = match.group(0) + "\n"
        for index, token, value in lexer.get_tokens_unprocessed(text):
            value = value.replace('\n', '')
            if value != '':
                yield index, token, value
1020651
def transform_source(text):
    '''
    '''
    toks = tokenize.generate_tokens(StringIO(text).readline)
    result = []
    replacing_keyword = False
    for toktype, tokvalue, _, _, _ in toks:
        if toktype == tokenize.NAME and tokvalue == 'switch':
            result.extend([
                (tokenize.NAME, 'for'),
                (tokenize.NAME, '__case'),
                (tokenize.NAME, 'in'),
                (tokenize.NAME, '_Switch'),
                (tokenize.OP, '(')
            ])
            replacing_keyword = True
        elif toktype == tokenize.NAME and (tokvalue == 'case' or tokvalue == 'default'):
            result.extend([
                (tokenize.NAME, 'if'),
                (tokenize.NAME, '__case'),
                (tokenize.OP, '(')
            ])
            replacing_keyword = True
        elif replacing_keyword and tokvalue == ':':
            result.extend([
                (tokenize.OP, ')'),
                (tokenize.OP, ':')
            ])
            replacing_keyword = False
        else:
            result.append((toktype, tokvalue))
    return tokenize.untokenize(result)
574854
def create_masked_lm_predictions(tokens, masked_lm_prob,
                                 max_predictions_per_seq, vocab_words, rng):
    """"""

    cand_indexes = []
    for (i, token) in enumerate(tokens):
        if token in ['[CLS]', '[SEP]']:
            continue
        cand_indexes.append(i)

    rng.shuffle(cand_indexes)

    output_tokens = list(tokens)

    num_to_predict = min(max_predictions_per_seq,
                         max(1, int(round(len(tokens) * masked_lm_prob))))

    masked_lms = []
    covered_indexes = set()
    for index in cand_indexes:
        if len(masked_lms) >= num_to_predict:
            break
        if index in covered_indexes:
            continue
        covered_indexes.add(index)

        masked_token = None
        # 80% of the time, replace with [MASK]
        if rng.random() < 0.8:
            masked_token = '[MASK]'
        else:
            # 10% of the time, keep original
            if rng.random() < 0.5:
                masked_token = tokens[index]
            # 10% of the time, replace with random word
            else:
                masked_token = vocab_words[rng.randint(0,
                                                       len(vocab_words) - 1)]

        output_tokens[index] = masked_token

        masked_lms.append(MaskedLmInstance(index=index, label=tokens[index]))

    masked_lms = sorted(masked_lms, key=lambda x: x.index)

    masked_lm_positions = []
    masked_lm_labels = []
    for p in masked_lms:
        masked_lm_positions.append(p.index)
        masked_lm_labels.append(p.label)

    return (output_tokens, masked_lm_positions, masked_lm_labels)
1363502
def nltk_tokenzier_factory(nltk_tokenizer):
	'''
	
	'''

	def tokenize(text):
		toks = []
		for tok in nltk_tokenizer.tokenize(text):
			if len(tok) > 0:
				toks.append(Tok(_get_pos_tag(tok),
				                tok.lower(),
				                tok.lower(),
				                ent_type='',
				                tag=''))
		return Doc([toks], text)

	return tokenize
180178
def _peek(tokens, n=0):
    """"""
    return tokens.peek(n=n, skip=_is_comment, drop=True)
1237049
def crude_tokenizer(line):
    """"""
    tokens = []
    buffer = ''
    for c in line.strip():
        if c == ' ' or c in string.punctuation:
            if buffer:
                tokens.append(buffer)
                buffer = ''
        else:
            buffer += c
    if buffer: tokens.append(buffer)
    return tokens
501607
def _tokenize(sentence):
    ''''''

    _tokens = nltk.word_tokenize(sentence)
    tokens = [stemmer.stem(tk) for tk in _tokens]
    return tokens
774248
def _getPhrase( self, i, sentence, NPlabels ):
        ''' 
        ''' 
        phrase  = []
        indices = []
        if 0 <= i and i < len(sentence) and NPlabels[i] == 'B':
            phrase  = [ sentence[i] ]
            indices = [ i ]
            j = i + 1
            while ( j < len(sentence) ):
                if NPlabels[j] in ['B', '']:
                    break
                else:
                    phrase.append( sentence[j] )
                    indices.append( j )
                j += 1
        return phrase, indices