Implementing a slightly modified feature detector.
        @param tokens: The tokens from the sentence to tag.
        @param index: The current token index to tag.
        @param history: The previous tagged tokens.
967257
def feature_detector(self, tokens, index, history):
        

        word = tokens[index]
        if index == 0: # At the beginning of the sentence
            prevword = prevprevword = None
            prevtag = prevprevtag = None
            #word = word.lower() # Lowercase at the beginning of sentence
        elif index == 1:
            prevword = tokens[index-1] # Note: no lowercase
            prevprevword = None
            prevtag = history[index-1]
            prevprevtag = None
        else:
            prevword = tokens[index-1]
            prevprevword = tokens[index-2]
            prevtag = history[index-1]
            prevprevtag = history[index-2]

        if re.match('[0-9]+([\.,][0-9]*)?|[0-9]*[\.,][0-9]+$', word):
            # Included "," as decimal point
            shape = 'number'
        elif re.compile('\W+$', re.UNICODE).match(word):
            # Included unicode flag
            shape = 'punct'
        elif re.match('([A-ZÄÖÜ]+[a-zäöüß]*-?)+$', word):
            # Included dash for dashed words and umlauts
            shape = 'upcase'
        elif re.match('[a-zäöüß]+', word):
            # Included umlauts
            shape = 'downcase'
        elif re.compile("\w+", re.UNICODE).match(word):
            # Included unicode flag
            shape = 'mixedcase'
        else:
            shape = 'other'

        features = {
            'prevtag': prevtag,
            'prevprevtag': prevprevtag,
            'word': word,
            'word.lower': word.lower(),
            'suffix3': word.lower()[-3:],
            #'suffix2': word.lower()[-2:],
            #'suffix1': word.lower()[-1:],
            'preffix1': word[:1], # included
            'prevprevword': prevprevword,
            'prevword': prevword,
            'prevtag+word': '%s+%s' % (prevtag, word),
            'prevprevtag+word': '%s+%s' % (prevprevtag, word),
            'prevword+word': '%s+%s' % (prevword, word),
            'shape': shape
            }
        return features
1068705
def from_node(index, value):
        
        try:
            lines = json.loads(value)
        except (TypeError, ValueError):
            lines = None
        if not isinstance(lines, list):
            lines = []
        return TimelineHistory(index, value, lines)
353856
private String getTokenOrBlank(final List<String> tokens, final int reverseIndex) {

		final int length = tokens.size();
		final int index  = length - reverseIndex - 1;

		if (index >= 0 && length > 0 && index < length) {
			return tokens.get(index);
		}

		return "";
	}
327735
def apply(self, token, previous=(None, None), next=(None, None)):
        
        return [token[0], self._classifier.classify(self._v(token[0], previous, next))]
707620
public final List<String> posAnnotate(final String[] tokens) {
    final String[] annotatedText = this.posTagger.tag(tokens);
    final List<String> posTags = new ArrayList<String>(
        Arrays.asList(annotatedText));
    return posTags;
  }
930528
protected int getHistorySubtitleSpacing (int index)
    {
        ChatMessage message = _history.get(index);
        return _logic.getSubtitleSpacing(getType(message, true));
    }
348973
public static void updateComponentTokenCounts(final List<Token> tokens)
    {
        final Map<String, Deque<Integer>> map = new HashMap<>();

        for (int i = 0, size = tokens.size(); i < size; i++)
        {
            final Token token = tokens.get(i);
            final Signal signal = token.signal();

            if (signal.name().startsWith("BEGIN_"))
            {
                final String componentType = signal.name().substring(6);
                map.computeIfAbsent(componentType, (key) -> new LinkedList<>()).push(i);
            }
            else if (signal.name().startsWith("END_"))
            {
                final String componentType = signal.name().substring(4);
                final int beginIndex = map.get(componentType).pop();

                final int componentTokenCount = (i - beginIndex) + 1;
                tokens.get(beginIndex).componentTokenCount(componentTokenCount);
                token.componentTokenCount(componentTokenCount);
            }
        }
    }
366837
def detect_spans(self, tokens):
        
        candidates = self._get_candidates(tokens)
        results = self._filter_candidates(tokens, candidates)
        return results
1141295
private JsDocToken recordDescription(JsDocToken token) {
    // Find marker's description (if applicable).
    if (jsdocBuilder.shouldParseDocumentation()) {
      ExtractionInfo descriptionInfo = extractMultilineTextualBlock(token);
      token = descriptionInfo.token;
    } else {
      token = eatTokensUntilEOL(token);
    }
    return token;
  }
330183
def apply(self, tokens):
        
        # Note: we could also scan for patterns, e.g.,
        # "my|his|her name is|was *" => NNP-PERS.
        i = 0
        while i < len(tokens):
            w = tokens[i][0].lower()
            if RE_ENTITY1.match(w) \
            or RE_ENTITY2.match(w) \
            or RE_ENTITY3.match(w):
                tokens[i][1] = self.tag
            if w in self:
                for e in self[w]:
                    # Look ahead to see if successive words match the named entity.
                    e, tag = (e[:-1], "-"+e[-1].upper()) if e[-1] in self._cmd else (e, "")
                    b = True
                    for j, e in enumerate(e):
                        if i + j >= len(tokens) or tokens[i+j][0].lower() != e:
                            b = False; break
                    if b:
                        for token in tokens[i:i+j+1]:
                            token[1] = token[1] if token[1].startswith(self.tag) else self.tag
                            token[1] += tag
                        i += j
                        break
            i += 1
        return tokens
654181
private void tokenize(JCas jcas) {
		// read tokenized text to add tokens to the jcas
		Logger.printDetail(component, "TreeTagger (tokenization) with: " + ttprops.abbFileName);
		
		EnumSet<Flag> flags = Flag.getSet(ttprops.languageSwitch);
		TreeTaggerTokenizer ttt; ttprops.abbFileName = "english-abbreviations";
		if(ttprops.abbFileName != null) {
			ttt = new TreeTaggerTokenizer(ttprops.rootPath + ttprops.fileSeparator + "lib" + ttprops.fileSeparator + ttprops.abbFileName, flags);
		} else {
			ttt = new TreeTaggerTokenizer(null, flags);
		}
		
		String docText = jcas.getDocumentText().replaceAll("\n\n", "\nEMPTYLINE\n");
		List<String> tokenized = ttt.tokenize(docText);
		
		int tokenOffset = 0;
		// loop through all the lines in the treetagger output
		for(String s : tokenized) {
			// charset missmatch fallback: signal (invalid) s
			if ((!(s.equals("EMPTYLINE"))) && (jcas.getDocumentText().indexOf(s, tokenOffset) < 0)) {
				Logger.printError(component, "Tokenization was interrupted because the token \"" + s 
						+ "\" could not be found in the original text. The reason for this might be "
						+ "that the encoding of the document is not UTF-8. This token was skipped and "
						+ "if it was part of a temporal expression, will not be extracted.");
				continue;
			}
			// create tokens and add them to the jcas's indexes.
			Token newToken = new Token(jcas);
			if (s.equals("EMPTYLINE")){
				newToken.setBegin(tokenOffset);
				newToken.setEnd(tokenOffset);
				newToken.setPos("EMPTYLINE");
				if (annotate_partofspeech){
					newToken.addToIndexes();
				}
			}
			else{
				newToken.setBegin(jcas.getDocumentText().indexOf(s, tokenOffset));
				newToken.setEnd(newToken.getBegin() + s.length());
				newToken.addToIndexes();
				tokenOffset = newToken.getEnd();
			}
		}
	}
264174
def convert_examples_to_features(examples, tokenizer, max_seq_length,
                                 is_training):
    """"""

    # Swag is a multiple choice task. To perform this task using Bert,
    # we will use the formatting proposed in "Improving Language
    # Understanding by Generative Pre-Training" and suggested by
    # @jacobdevlin-google in this issue
    # https://github.com/google-research/bert/issues/38.
    #
    # Each choice will correspond to a sample on which we run the
    # inference. For a given Swag example, we will create the 4
    # following inputs:
    # - [CLS] context [SEP] choice_1 [SEP]
    # - [CLS] context [SEP] choice_2 [SEP]
    # - [CLS] context [SEP] choice_3 [SEP]
    # - [CLS] context [SEP] choice_4 [SEP]
    # The model will output a single value for each input. To get the
    # final decision of the model, we will run a softmax over these 4
    # outputs.
    features = []
    for example_index, example in enumerate(examples):
        context_tokens = tokenizer.tokenize(example.context_sentence)
        start_ending_tokens = tokenizer.tokenize(example.start_ending)

        choices_features = []
        for ending_index, ending in enumerate(example.endings):
            # We create a copy of the context tokens in order to be
            # able to shrink it according to ending_tokens
            context_tokens_choice = context_tokens[:]
            ending_tokens = start_ending_tokens + tokenizer.tokenize(ending)
            # Modifies `context_tokens_choice` and `ending_tokens` in
            # place so that the total length is less than the
            # specified length.  Account for [CLS], [SEP], [SEP] with
            # "- 3"
            _truncate_seq_pair(context_tokens_choice, ending_tokens, max_seq_length - 3)

            tokens = ["[CLS]"] + context_tokens_choice + ["[SEP]"] + ending_tokens + ["[SEP]"]
            segment_ids = [0] * (len(context_tokens_choice) + 2) + [1] * (len(ending_tokens) + 1)

            input_ids = tokenizer.convert_tokens_to_ids(tokens)
            input_mask = [1] * len(input_ids)

            # Zero-pad up to the sequence length.
            padding = [0] * (max_seq_length - len(input_ids))
            input_ids += padding
            input_mask += padding
            segment_ids += padding

            assert len(input_ids) == max_seq_length
            assert len(input_mask) == max_seq_length
            assert len(segment_ids) == max_seq_length

            choices_features.append((tokens, input_ids, input_mask, segment_ids))

        label = example.label
        if example_index < 5:
            logger.info("*** Example ***")
            logger.info("swag_id: {}".format(example.swag_id))
            for choice_idx, (tokens, input_ids, input_mask, segment_ids) in enumerate(choices_features):
                logger.info("choice: {}".format(choice_idx))
                logger.info("tokens: {}".format(' '.join(tokens)))
                logger.info("input_ids: {}".format(' '.join(map(str, input_ids))))
                logger.info("input_mask: {}".format(' '.join(map(str, input_mask))))
                logger.info("segment_ids: {}".format(' '.join(map(str, segment_ids))))
            if is_training:
                logger.info("label: {}".format(label))

        features.append(
            InputFeatures(
                example_id = example.swag_id,
                choices_features = choices_features,
                label = label
            )
        )

    return features
260684
private static void processText(NameFinder[] finders, String[] tags, BufferedReader input) throws IOException {
    String[][] finderTags = new String[finders.length][];
    Map[] prevTokenMaps = createPrevTokenMaps(finders); 
    for (String line = input.readLine(); null != line; line = input.readLine()) {
      if (line.equals("")) {
        clearPrevTokenMaps(prevTokenMaps);
        System.out.println();
        continue;
      }
      Span[] spans = tokenizeToSpans(line);
      String[] tokens = spansToStrings(spans,line);
      for (int fi = 0, fl = finders.length; fi < fl; fi++) {
        finderTags[fi] = finders[fi].find(tokens, prevTokenMaps[fi]);
        //System.err.println("EnglighNameFinder.processText: "+tags[fi] + " " + java.util.Arrays.asList(finderTags[fi]));
      }
      updatePrevTokenMaps(prevTokenMaps,tokens,finderTags);
      for (int ti = 0, tl = tokens.length; ti < tl; ti++) {
        for (int fi = 0, fl = finders.length; fi < fl; fi++) {
          //check for end tags
          if (ti != 0) {
            if ((finderTags[fi][ti].equals(NameFinderME.START) || finderTags[fi][ti].equals(NameFinderME.OTHER)) && (finderTags[fi][ti - 1].equals(NameFinderME.START) || finderTags[fi][ti - 1].equals(NameFinderME.CONTINUE))) {
              System.out.print("</" + tags[fi] + ">");
            }
          }
        }
        if (ti > 0 && spans[ti - 1].getEnd() < spans[ti].getStart()) {
          System.out.print(line.substring(spans[ti - 1].getEnd(), spans[ti].getStart()));
        }
        //check for start tags
        for (int fi = 0, fl = finders.length; fi < fl; fi++) {
          if (finderTags[fi][ti].equals(NameFinderME.START)) {
            System.out.print("<" + tags[fi] + ">");
          }
        }
        System.out.print(tokens[ti]);
      }
      //final end tags
      if (tokens.length != 0) {
        for (int fi = 0, fl = finders.length; fi < fl; fi++) {
          if (finderTags[fi][tokens.length - 1].equals(NameFinderME.START) || finderTags[fi][tokens.length - 1].equals(NameFinderME.CONTINUE)) {
            System.out.print("</" + tags[fi] + ">");
          }
        }
      }
      if (tokens.length != 0) {
        if (spans[tokens.length - 1].getEnd() < line.length()) {
          System.out.print(line.substring(spans[tokens.length - 1].getEnd()));
        }
      }
      System.out.println();
    }
  }
1071211
public void visitToken(Token token) {
    for (Trivia trivia : token.getTrivia()) {
      if (!trivia.isComment()) {
        continue;
      }
      Token triviaToken = trivia.getToken();
      String comment = triviaToken.getOriginalValue();
      int line = triviaToken.getLine();
      if (indexOfIgnoreCase(comment) != -1) {
        String[] lines = EOL_PATTERN.split(comment);

        for (int i = 0; i < lines.length; i++) {
          int start = indexOfIgnoreCase(lines[i]);
          if (start != -1 && !isLetterAround(lines[i], start)) {
            check.getContext().createLineViolation(check, message, line + i);
          }
        }
      }
    }
  }
118350
def fetch(index, tokens):
    
    if len(tokens) == 0:
        return set()
    return set.intersection(*[set(index.get(token, [])) for token in tokens])
23977
public void setTokens(StringArray v) {
    if (FeatureTokens_Type.featOkTst && ((FeatureTokens_Type)jcasType).casFeat_tokens == null)
      jcasType.jcas.throwFeatMissing("tokens", "ch.epfl.bbp.uima.types.FeatureTokens");
    jcasType.ll_cas.ll_setRefValue(addr, ((FeatureTokens_Type)jcasType).casFeatCode_tokens, jcasType.ll_cas.ll_getFSRef(v));}
959485
def consume_token(self, tokens, index, tokens_len):
        
        finished = False

        if tokens[index].line > self.line:
            finished = True
            end = index
        elif index == tokens_len - 1:
            finished = True
            end = index + 1

        if finished:
            pasted_together_contents = ""
            for i in range(self.begin, end):
                pasted_together_contents += tokens[i].content

            replacement = [Token(type=TokenType.Comment,
                                 content=pasted_together_contents,
                                 line=tokens[self.begin].line,
                                 col=tokens[self.begin].col)]

            tokens = _replace_token_range(tokens,
                                          self.begin,
                                          end,
                                          replacement)

            return (self.begin, len(tokens), tokens)