Yields a list of all the token tags as they appeared when the word was parsed.
            For example: ["was", "VBD", "B-VP", "O", "VP-1", "A1", "be"]
322330
def tags(self):
        
        # See also. Sentence.__repr__().
        ch, I,O,B = self.chunk, INSIDE+"-", OUTSIDE, BEGIN+"-"
        tags = [OUTSIDE for i in range(len(self.sentence.token))]
        for i, tag in enumerate(self.sentence.token): # Default: [WORD, POS, CHUNK, PNP, RELATION, ANCHOR, LEMMA]
            if tag == WORD:
                tags[i] = encode_entities(self.string)
            elif tag == POS and self.type:
                tags[i] = self.type
            elif tag == CHUNK and ch and ch.type:
                tags[i] = (self == ch[0] and B or I) + ch.type
            elif tag == PNP and self.pnp:
                tags[i] = (self == self.pnp[0] and B or I) + "PNP"
            elif tag == REL and ch and len(ch.relations) > 0:
                tags[i] = ["-".join([str(x) for x in [ch.type]+list(reversed(r)) if x]) for r in ch.relations]
                tags[i] = "*".join(tags[i])
            elif tag == ANCHOR and ch:
                tags[i] = ch.anchor_id or OUTSIDE
            elif tag == LEMMA:
                tags[i] = encode_entities(self.lemma or "")
            elif tag in self.custom_tags:
                tags[i] = self.custom_tags.get(tag) or OUTSIDE
        return tags
752756
def get_identifiers(self):
        
        for token in self.tokens:
            if not (token.is_whitespace or token.match(T.Punctuation, ',')):
                yield token
54699
public final List<Morpheme> getMorphemesFromStrings(
      final List<String> posTags, final String[] tokens) {
    final List<Morpheme> morphemes = new ArrayList<Morpheme>();
    for (int i = 0; i < posTags.size(); i++) {
      final String word = tokens[i];
      final String tag = posTags.get(i);
      final Morpheme morpheme = this.morphoFactory.createMorpheme(word, tag);
      morphemes.add(morpheme);
    }
    return morphemes;
  }
641505
def tags(self):
        
        tag_list = self.spec.get('tag', [])
        if isinstance(tag_list, (list, set, tuple)):
            return list(tag_list)
        return [tag_list]
420304
def patterns(self):
        
        all_patterns = []
        for label, patterns in self.token_patterns.items():
            for pattern in patterns:
                all_patterns.append({"label": label, "pattern": pattern})
        for label, patterns in self.phrase_patterns.items():
            for pattern in patterns:
                all_patterns.append({"label": label, "pattern": pattern.text})
        return all_patterns
1064166
def encr_tags(self) -> dict:
        

        return {t: self._tags[t] for t in self.tags or {} if not t.startswith('~')} or None
1141492
def detect(self, tokens):
        """"""
        results = []
        for abbr_span, long_span in self.detect_spans(tokens):
            results.append((tokens[abbr_span[0]:abbr_span[1]], tokens[long_span[0]:long_span[1]]))
        return results
113265
def view_all_tags(ctx, param, value):
    
    # To make sure this function executed only when the flag was called
    if not value or ctx.resilient_parsing:
        return
    avail_tags = dict()
    print("\nMultiQC Available module tag groups:\n")
    for mod_dict in filter(lambda mod:isinstance(mod, dict), config.module_order):
        mod_key, mod_val = list(mod_dict.items())[0]
        tags = list(mod_val.get('module_tag', []))
        for t in tags:
            if t not in avail_tags:
                avail_tags[t] = []
            avail_tags[t].append(mod_key)
    for t in sorted(avail_tags.keys(), key=lambda s: s.lower()):
        print (" - {}:".format(t))
        for ttgs in avail_tags[t]:
            print ("   - {}".format(ttgs))
    ctx.exit()
451782
def tags(self):
        
        if 'tags' in self._values:
            return self._values['tags']
        self._values['tags'] = copy.deepcopy(self._defaults['tags'])
        return self._values['tags']
247551
def listified_tokenizer(source):
    """"""
    io_obj = io.StringIO(source)
    return [list(a) for a in tokenize.generate_tokens(io_obj.readline)]
4202
def translate_keyword_tokens( token )
      keywords = ["__LINE__", "__ENCODING__", "__FILE__", "BEGIN",   
                  "END", "alias", "and", "begin", "break", "case",
                  "class", "def", "defined?", "do", "else", "elsif",
                  "end", "ensure", "false", "for", "if", "in",
                  "module", "next", "nil", "not", "or", "redo",
                  "rescue", "retry", "return", "self", "super",
                  "then", "true", "undef", "unless", "until", 
                  "when", "while", "yield"]
      if keywords.include?( token.content )
        token.type = token.content.downcase.to_sym
        # Change the state if we match a keyword
        @expr_state = :beg
      end
      
      # A couple of exceptions      
      if token.content == "BEGIN"
        token.type = :begin_global 
        @expr_state = :beg
      elsif token.content == "END"
        token.type = :end_global 
        @expr_state = :beg
      end
      
      token
    end
277159
def pos_tags(self):
    
    tagged_words = []
    for word,t in self.pos_tagger.annotate(self.words):
      word.pos_tag = t
      tagged_words.append((word, t))
    return tagged_words
1340724
def tokens(self):
        
        for content in self.contents:
            if isinstance(content, TokenWithPosition):
                for word in content.split():
                    yield word
            else:
                yield content
767516
def _get_tags(self):
        # type: () -> List[str]
        
        tags = self.data.get('tags', None)
        if not tags:
            return list()
        return [x['name'] for x in tags]
1281768
@Override
  public List<AnalyzedTokenReadings> tag(List<String> sentenceTokens)
  throws IOException {
    List<AnalyzedToken> taggerTokens;
    List<AnalyzedToken> lowerTaggerTokens;
    List<AnalyzedToken> upperTaggerTokens;
    List<AnalyzedTokenReadings> tokenReadings = new ArrayList<>();
    int pos = 0;

    Matcher matcher;
    for (String word : sentenceTokens) {
      String probeWord = word;

      // This loop happens when we need to retry probing the dictionary
      // which happens rarely when trying to remove suffixes -mañ, -se, etc.
      for (;;) {
        List<AnalyzedToken> l = new ArrayList<>();
        String lowerWord = probeWord.toLowerCase(conversionLocale);
        taggerTokens = asAnalyzedTokenListForTaggedWords(word, getWordTagger().tag(probeWord));
        lowerTaggerTokens = asAnalyzedTokenListForTaggedWords(word, getWordTagger().tag(lowerWord));
        boolean isLowercase = probeWord.equals(lowerWord);

        // Normal case.
        addTokens(taggerTokens, l);

        if (!isLowercase) {
          // Lowercase.
          addTokens(lowerTaggerTokens, l);
        }

        // Uppercase.
        if (lowerTaggerTokens.isEmpty() && taggerTokens.isEmpty()) {
          if (isLowercase) {
            upperTaggerTokens = asAnalyzedTokenListForTaggedWords(word,
                getWordTagger().tag(StringTools.uppercaseFirstChar(probeWord)));
            if (!upperTaggerTokens.isEmpty()) {
              addTokens(upperTaggerTokens, l);
            }
          }
          if (l.isEmpty()) {
            if ((matcher = patternSuffix.matcher(probeWord)).find()) {
              // Remove the suffix and probe dictionary again.
              // So given a word such as "xxx-mañ", we're going to
              // try to probe the dictionary again with "xxx" this time.
              probeWord = matcher.group(1);
              continue;
            }
            l.add(new AnalyzedToken(word, null, null));
          }
        }
        tokenReadings.add(new AnalyzedTokenReadings(l, pos));
        pos += word.length();
        break;
      }
    }

    return tokenReadings;

  }
799130
def get_tags(self, tagtype):
        '''  '''
        return [t for t in self.__tags if t.tagtype == tagtype]
542991
public List<Token> scan() {
		while (peek() != EOF) {
			if (peek() == '#') {
				if (scanDire()) {
					continue ;
				}
				if (scanSingleLineComment()) {
					continue ;
				}
				if (scanMultiLineComment()) {
					continue ;
				}
				if (scanNoParse()) {
					continue ;
				}
			}
			
			scanText();
		}
		return tokens;
	}