When one or more tokens are pushed to a stream, those tokens
must be inserted, in given order, after the last token in the
stream.

@param {(number|!Array.<number>)} token The tokens(s) to push to the
stream.
494798
function(token) {
      if (Array.isArray(token)) {
        var tokens = /**@type {!Array.<number>}*/(token);
        while (tokens.length)
          this.tokens.unshift(tokens.shift());
      } else {
        this.tokens.unshift(token);
      }
    }
1334068
private void addTokens(List<AnalyzedToken> taggedTokens, List<AnalyzedToken> l) {
    if (taggedTokens != null) {
      for (AnalyzedToken at : taggedTokens) {
        l.add(at);
      }
    }
  }
1272807
@Override
    public boolean hasMoreTokens() {
        log.info("Tokens size: [" + tokens.size() + "], position: [" + position.get() + "]");
        if (!tokens.isEmpty())
            return position.get() < tokens.size();
        else
            return streamHasMoreTokens();
    }
455113
private void addTokens(final Token curToken, final List<Token> tokens) {
    if (curToken.tokenLength() != 0) {
      if (this.unTokenizable) {
        tokens.add(curToken);
      } else if (!this.unTokenizable) {
        if (!replacement.matcher(curToken.getTokenValue()).matches()) {
          tokens.add(curToken);
        }
      }
    }
  }
177068
private final void shift(ParserAction action) {
	stateStack.push(action.getParameter());
	actionStack.add(action);
	streamPosition++;
	shiftIgnoredTokens();
    }
381619
public void saveResumeToken(String token) {
        SharedPreferences settings = mContext.getSharedPreferences(PREF_DDPINFO, Context.MODE_PRIVATE);
        SharedPreferences.Editor editor = settings.edit();
        //REVIEW: should token be encrypted?
        editor.putString(PREF_RESUMETOKEN, token);
        editor.apply();
    }
916195
public void put(String key, T token) {
        purge();
        if (map.containsKey(key)) {
            map.remove(key);
        }
        final long time = System.currentTimeMillis();
        map.put(key, new Value(token, time + expireMillis));
        latestWriteTime = time;
    }
96962
function isLastTokenInCurrentLine(token) {
            const tokenAfter = sourceCode.getTokenAfter(token);

            return !(tokenAfter && astUtils.isTokenOnSameLine(token, tokenAfter));
        }
288056
function ast (tokens) {
  let token = tokens[0]

  const state = {
    content: []
  }

  let current = state
  let active = current
  const stack = []
  let lineStackSize = 0 // when inline entities are used this gets incremented so that when you
  // go onto a new line we can go back to the correct location in the stack

  const l = tokens.length
  let i = 0
  let absoluteIndent = 0
  let currentIndent = 0
  let extraIndent = 0
  let prevWasType = false
  let nextParamsAreEscaped = false
  let prevParamsWereEscaped = false
  const emptyLines = []
  let handlingInlineContent = false // true when between an START_INLINE_CONTENT and END_INLINE_CONTENT
  /*
    notCreatedInlineContent
    This is true when the inline content entry has not yet been created in the parsed
    content (which is done for the first content encountered between START_INLINE_CONTENT
    and END_INLINE_CONTENT)
  */
  let notCreatedInlineContent = true

  while (i < l) {
    token = tokens[i]
    if (token.type === 'TYPE') {
      if (token.value === '') {
        nextParamsAreEscaped = true
      } else {
        prevWasType = true

        active = {
          type: token.value,
          params: [],
          content: []
        }

        while (emptyLines.length) {
          current.content.push(emptyLines.pop())
        }

        current.content.push(active)
      }
    } else if (token.type === 'PARAMS') {
      if (nextParamsAreEscaped) {
        if (active.content.length > 0) {
          active.content[active.content.length - 1] += token.value
        } else {
          active.content.push(token.value)
        }
        nextParamsAreEscaped = false
        prevParamsWereEscaped = true
      } else {
        active.params = splitParams(token.value)
      }
    } else if (token.type === 'CONTENT') {
      if (prevParamsWereEscaped) {
        active.content[active.content.length - 1] += token.value
        prevParamsWereEscaped = false
      } else if (token.value === '') {
        emptyLines.push(token.value)
      } else {
        while (emptyLines.length) {
          current.content.push(emptyLines.pop())
        }

        prevWasType = false

        if (handlingInlineContent) {
          if (notCreatedInlineContent) {
            current.content.push(token.value)
            notCreatedInlineContent = false
          } else {
            current.content[current.content.length - 1] += ' ' + token.value
          }
        } else {
          let ex = ''
          for (let k = 0; k < extraIndent; k++) {
            ex += ' '
          }
          current.content.push(ex + token.value)
        }
      }
    } else if (token.type === 'EMPTY_CONTENT') {
      emptyLines.push(token.value)
    } else if (token.type === 'START_SAME_LINE_CONTENT') {
      lineStackSize++
      stack.push(current)
      current = active
    } else if (token.type === 'END_SAME_LINE_CONTENT') {
      // keep the active as it is, even though the current is being droped back a couple of levels
      // this is so that you can do @one: @two: @three\n @four, and have @four be a child of @three (not @one)
      active = current

      // drop back down the stack until you reach where we were at the start of the line
      while (lineStackSize) {
        current = stack.pop()
        lineStackSize--
      }
      prevWasType = true
    } else if (token.type === 'START_INLINE_CONTENT') {
      handlingInlineContent = true
      notCreatedInlineContent = true
      lineStackSize++
      stack.push(current)
      current = active
    } else if (token.type === 'END_INLINE_CONTENT') {
      handlingInlineContent = false
      active = current
      current = stack.pop()
      lineStackSize--
      prevWasType = true
    } else if (token.type === 'INDENT') {
      stack.push(current)
      current = active
      absoluteIndent += token.value

      if (prevWasType) {
        currentIndent = absoluteIndent
      }

      extraIndent = absoluteIndent - currentIndent
    } else if (token.type === 'DEDENT') {
      current = stack.pop()
      active = current
      absoluteIndent -= token.value

      if (extraIndent > 0) {
        extraIndent -= token.value
      } else {
        currentIndent = absoluteIndent
      }
    }
    i++
  }

  return state.content
}
222183
private static TokenRange tokenRange(Token token) {
        JavaToken javaToken = token.javaToken;
        return new TokenRange(javaToken, javaToken);
    }
1044920
public void insertOrUpdate(Token token) {
        EntityManager entityManager = getEntityManager();
        EntityTransaction entityTransaction = null;

        try {
            entityTransaction = entityManager.getTransaction();
            entityTransaction.begin();
			if (token.getId() == null) {
				entityManager.persist(token);
			} else {
				entityManager.merge(token);
			}
            entityTransaction.commit();
        } catch (Exception e) {
            if (entityTransaction != null && entityTransaction.isActive()) {
                entityTransaction.rollback();
            }
            throw e;
        } finally {
			entityManager.close();
		}
	}
731933
@Requires("lookahead >= 0")
  public Token peek(int lookahead) {
    ArrayDeque<Token> tmp = new ArrayDeque<Token>();
    for (int i = 0; i <= lookahead; ++i) {
      if (hasNext()) {
        tmp.push(next());
      }
    }
    Token token = tmp.peek();
    while (!tmp.isEmpty()) {
      pushback(tmp.pop());
    }
    return token;
  }
1267799
def forced_insert(self):
        
        # If we have any tokens we are waiting for
        if self.insert_till:
            # Determine where to append this token
            append_at = -1
            if self.inserted_line:
                append_at = -self.inserted_line+1

            # Reset insert_till if we found it
            if self.current.tokenum == self.insert_till[0] and self.current.value == self.insert_till[1]:
                self.insert_till = None
            else:
                # Adjust self.adjust_indent_at to take into account the new token
                for index, value in enumerate(self.adjust_indent_at):
                    if value < len(self.result) - append_at:
                        self.adjust_indent_at[index] = value + 1

                # Insert the new token
                self.result.insert(append_at, (self.current.tokenum, self.current.value))

            # We appended the token
            return True
686235
def insert_in_next_line(tokens, index, string):
    
    tokenized_string = token_utils.listified_tokenizer(string)
    for i, tok in list(enumerate(tokens[index:])):
        token_type = tok[0]
        if token_type in [tokenize.NL, tokenize.NEWLINE]:
            for count, item in enumerate(tokenized_string):
                tokens.insert(index+count+i+1, item)
            break
436716
private final void prevToken()
  {

    if (m_queueMark > 0)
    {
      m_queueMark--;

      m_token = (String) m_ops.m_tokenQueue.elementAt(m_queueMark);
      m_tokenChar = m_token.charAt(0);
    }
    else
    {
      m_token = null;
      m_tokenChar = 0;
    }
  }
254111
@SuppressWarnings("unchecked")
	public void setAccessToken(String token) {
		if (!StringUtils.isBlank(token)) {
			try {
				String payload = Utils.base64dec(StringUtils.substringBetween(token, ".", "."));
				Map<String, Object> decoded = ParaObjectUtils.getJsonMapper().readValue(payload, Map.class);
				if (decoded != null && decoded.containsKey("exp")) {
					this.tokenKeyExpires = (Long) decoded.get("exp");
					this.tokenKeyNextRefresh = (Long) decoded.get("refresh");
				}
			} catch (Exception ex) {
				this.tokenKeyExpires = null;
				this.tokenKeyNextRefresh = null;
			}
		}
		this.tokenKey = token;
	}
1178533
public void accept(TokenKind tk) {
        if (token.kind == tk) {
            nextToken();
        } else {
            setErrorEndPos(token.pos);
            reportSyntaxError(S.prevToken().endPos, "expected", tk);
        }
    }