Consumes a string token that ends at the given character, and returns the
 token.

 4.3.5. http://dev.w3.org/csswg/css-syntax/#consume-a-string-token
967306
def consume_string(ending = nil)
      ending = @s.current if ending.nil?
      value  = String.new

      until @s.eos?
        case char = @s.consume
        when ending
          break

        when "\n"
          # Parse error.
          @s.reconsume
          return create_token(:bad_string,
            :error => true,
            :value => value)

        when '\\'
          case @s.peek
          when ''
            # End of the input, so do nothing.
            next

          when "\n"
            @s.consume

          else
            value << consume_escaped
          end

        else
          value << char
        end
      end

      create_token(:string, :value => value)
    end
1367260
protected byte[] parseUnicodeString(Token token) {
		String v = token.text;
		/*
		 * Parsing a string requires several steps to be taken. First, we need
		 * to strip quotes from the ends of the string.
		 */
		v = v.substring(1, v.length() - 1);

		StringBuffer result = new StringBuffer();
		// Second, step through the string and replace escaped characters
		for (int i = 0; i < v.length(); i++) {
			if (v.charAt(i) == '\\') {
				if (v.length() <= i + 1) {
					throw new RuntimeException("unexpected end-of-string");
				} else {
					char replace = 0;
					int len = 2;
					switch (v.charAt(i + 1)) {
					case 'b':
						replace = '\b';
						break;
					case 't':
						replace = '\t';
						break;
					case 'n':
						replace = '\n';
						break;
					case 'f':
						replace = '\f';
						break;
					case 'r':
						replace = '\r';
						break;
					case '"':
						replace = '\"';
						break;
					case '\'':
						replace = '\'';
						break;
					case '\\':
						replace = '\\';
						break;
					case 'u':
						len = 6; // unicode escapes are six digits long,
						// including "slash u"
						String unicode = v.substring(i + 2, i + 6);
						replace = (char) Integer.parseInt(unicode, 16); // unicode
						i = i + 5;
						break;
					default:
						throw new RuntimeException("unknown escape character");
					}
					result.append(replace);
					i = i + 1;
				}
			} else {
				result.append(v.charAt(i));
			}
		}
		try {
			// Now, convert string into a sequence of UTF8 bytes.
			return result.toString().getBytes("UTF8");
		} catch (UnsupportedEncodingException e) {
			// This really should be deadcode
			syntaxError("invalid unicode string", token);
			return null; // deadcode
		}
	}
1340387
def string_token
      if %w(' ").include?(@chunk[0])
        result = process_string @chunk[0..-1], @chunk[0]
        if @looking_for_args
          token :Argument, result.str, attrs: { quoted_by: @chunk[0] }
        else
          token :Command, result.str
        end
        result.consumed_length
      else
        did_not_match
      end
    end
1094032
protected String consumeWord(ImapRequestLineReader request,
                                 CharacterValidator validator)
            throws ProtocolException {
        StringBuilder atom = new StringBuilder();

        char next = request.nextWordChar();
        while (!isWhitespace(next)) {
            if (validator.isValid(next)) {
                atom.append(next);
                request.consume();
            } else {
                throw new ProtocolException("Invalid character: '" + next + '\'');
            }
            next = request.nextChar();
        }
        return atom.toString();
    }
965944
@Override
    protected String consumeIdentifier( DdlTokenStream tokens ) throws ParsingException {
        Position startPosition = tokens.nextPosition();
        String id = super.consumeIdentifier(tokens);
        if (!tokens.hasNext()) return id;
        Position nextPosition = tokens.nextPosition();

        while (((nextPosition.getIndexInContent() - startPosition.getIndexInContent() - id.length()) == 0)) {
            // allowed symbols in an identifier: underscore, dollar sign, pound sign, at sign
            if (tokens.matches(DdlTokenizer.SYMBOL)) {
                if (tokens.matches('$') || tokens.matches('#') || tokens.matches('@')) {
                    id += tokens.consume(); // consume symbol
                } else {
                    break; // not a valid ID symbol
                }
            } else {
                id += tokens.consume(); // consume text
            }

            nextPosition = tokens.nextPosition();
        }

        return id;
    }
91927
function tokenize (str) {
  let pos = 0
  let row = 1
  let col = 1
  let newline = true
  let state = CONTENT
  let start = 0
  let consumingSameLineContent = false
  let escapedInlineCounter = 0
  let escapedParamCounter = 0
  let consumingEscapedParams = false
  let consumingUnparsed = false // true when in an @@ block
  let lastInlineContentEndPos = void (0)
  let unparsedIndentStart = void (0)

  const indent = [0]
  const tokens = []

  function emit (type, value) {
    if (arguments.length > 1) {
      tokens.push({type: type, value: value})
    } else {
      tokens.push({type: type})
    }
  }

  function consume (type, next) {
    emit(type, str.substring(start, pos))
    state = next
    start = pos + 1
  }

  function consumeIfNonEmpty (type, next, escaped) {
    let v = str.substring(start, pos)
    if (v.length > 0) {
      if (escaped) {
        const l = escaped.length
        for (let i = 0; i < l; i++) {
          const es = escaped[i]
          v = v.replace(es, es[1])
        }
      }
      emit(type, v)
    }
    state = next
    start = pos + 1
  }

  function shuffleIfNext (character) {
    if (str[pos + 1] === character) {
      pos++
      start++
      return true
    } else {
      return false
    }
  }

  function err (msg) {
    let start = str.lastIndexOf('\n', pos - 1)
    start = str.lastIndexOf('\n', start - 1)
    start = str.lastIndexOf('\n', start - 1)

    const nextNewline = str.indexOf('\n', pos)
    const errorLineEnd = nextNewline === -1 ? pos : nextNewline

    let end = str.indexOf('\n', errorLineEnd + 1)
    if (end === -1) {
      end = str.length - 1
    } else {
      end = str.indexOf('\n', end + 1)
      if (end === -1) {
        end = str.length - 1
      } else {
        end = str.indexOf('\n', end + 1)
        if (end === -1) {
          end = str.length - 1
        }
      }
    }

    const indentSpaces = (new Array(col)).join(' ')

    const message = 'Syntax error at line ' + row + ', col ' + col + ': ' + msg
    const context = str.substring(start, errorLineEnd) + '\n' + indentSpaces + '^^^^^^^^\n' + str.substring(errorLineEnd + 1, end)

    throw new ParseError(message + '\n' + context, context, str, row, col, msg, pos)
  }

  while (pos < str.length) {
    // indentation and comment handling.
    if (newline) {
      while (newline && pos < str.length) {
        let ind = 0
        while (str[pos + ind] === ' ' && pos + ind < str.length) {
          ind++
        }

        // if it was an empty content line, then add it to the list of empty
        // content to emit on the next non empty line
        if (str[pos + ind] === '\n') {
          pos += ind + 1
          row++
          emit('EMPTY_CONTENT', str.substring(start, pos - 1))
        } else if (str[pos + ind] !== '#' || consumingUnparsed) {
          if (ind > indent[indent.length - 1]) {
            emit('INDENT', ind - indent[indent.length - 1])
            indent.push(ind)
          } else if (ind < indent[indent.length - 1]) {
            while (indent[indent.length - 1] > ind) {
              const prev = indent.pop()
              emit('DEDENT', prev - indent[indent.length - 1])

              if (consumingUnparsed && unparsedIndentStart >= indent[indent.length - 1]) {
                consumingUnparsed = false
              }
            }

            if (indent.length > 0 && (indent[indent.length - 1] !== ind)) {
              err('indentation mismatch: this line dedents with an indentation of ' + ind + ' spaces, but an indentation of ' + indent[indent.length - 1] + ' spaces was expected')
            }
          }
          pos += ind

          if (str[pos] === '\\' && str[pos + 1] === '#') {
            pos++
          }
          newline = false
        } else {
          while (str[pos] !== '\n' && pos < str.length) {
            pos++
          }
          emit('COMMENT', str.substring(start + ind + 1, pos))
          pos++
          row++
        }

        start = pos
      }
    }

    if (pos >= str.length) {
      return tokens
    }

    const s = str[pos]

    if (state === TYPE) {
      if (s === ' ') {
        consume('TYPE', PARAMS)
        while (shuffleIfNext(' ') && pos < str.length) {}
      } else if (s === ':') {
        consume('TYPE', CONTENT)
        shuffleIfNext(' ')
        consumingSameLineContent = true
        emit('START_SAME_LINE_CONTENT')
      } else if (s === '[') {
        consume('TYPE', INLINE_CONTENT)
        emit('START_INLINE_CONTENT')
      } else if (s === '(') {
        consume('TYPE', INLINE_PARAMS)
      } else if (s === '\n') {
        consume('TYPE', CONTENT)
        if (consumingSameLineContent) {
          emit('END_SAME_LINE_CONTENT')
          consumingSameLineContent = false
        }
      }
    } else if (state === PARAMS) {
      if (s === '[') {
        escapedParamCounter++
        consumingEscapedParams = true
      } else if (consumingEscapedParams && (s === ']')) {
        escapedParamCounter--
        if (escapedParamCounter === 0) {
          consumingEscapedParams = false
        }
      } else if (s === ':' && !consumingEscapedParams) {
        consume('PARAMS', CONTENT)
        shuffleIfNext(' ')
        shuffleIfNext(']')
        consumingSameLineContent = true
        emit('START_SAME_LINE_CONTENT')
      } else if (s === '\n') {
        escapedParamCounter = 0
        consumingEscapedParams = false
        consume('PARAMS', CONTENT)
        if (consumingSameLineContent) {
          emit('END_SAME_LINE_CONTENT')
          consumingSameLineContent = false
        }
      }
    } else if (state === CONTENT) {
      if (s === '@' && !consumingUnparsed) {
        consumeIfNonEmpty('CONTENT', TYPE)
        if (str[pos + 1] === '@') {
          consumingUnparsed = true
          unparsedIndentStart = indent[indent.length - 1]
          pos++
          start++
        }
      } else if (s === '\n') {
        if (consumingSameLineContent) {
          if (lastInlineContentEndPos !== pos - 1) {
            consumeIfNonEmpty('CONTENT', CONTENT)
          }
          emit('END_SAME_LINE_CONTENT')
          consumingSameLineContent = false
          consumingUnparsed = false
        } else {
          if (lastInlineContentEndPos !== pos - 1) {
            consumeIfNonEmpty('CONTENT', CONTENT)
          }
        }
      }
    } else if (state === INLINE_PARAMS) {
      if (s === ')') {
        if (str[pos + 1] === '[') {
          consume('PARAMS', INLINE_CONTENT)
          emit('START_INLINE_CONTENT')
          start++
          pos++
        } else {
          consume('PARAMS', CONTENT)
        }
      }
    } else { // if (state === INLINE_CONTENT) {
      if (s === '\\' && (str[pos + 1] === '[' || str[pos + 1] === ']')) {
        pos++
      } else if (s === '[') {
        escapedInlineCounter++
      } else if (s === ']') {
        if (escapedInlineCounter === 0) {
          consumeIfNonEmpty('CONTENT', CONTENT, ['\\]', '\\['])
          emit('END_INLINE_CONTENT')
          lastInlineContentEndPos = pos + 1
        } else {
          escapedInlineCounter--
        }
      } else if (s === '\n') {
        consume('CONTENT', INLINE_CONTENT)
      }
    }

    if (s === '\n') {
      col = 0
      row++
      newline = true
    } else {
      col++
    }

    pos++
  }

  if (state === TYPE) {
    emit('TYPE', str.substring(start, pos))
  } else if (state === PARAMS) {
    emit('PARAMS', str.substring(start, pos))
  } else if (state === INLINE_PARAMS) {
    err('missing closing ) bracket?')
  } else if (state === INLINE_CONTENT) {
    err('missing closing ] bracket?')
  } else { // if (state === CONTENT) {
    if (start < pos) {
      emit('CONTENT', str.substring(start, pos))
    }
  }

  return tokens
}
1058906
def _get_token_type(self, char):
        
        if char in '()':
            return self.SPLIT, 0
        elif char == ',':
            return self.SPLIT, 1
        elif char in '<>':
            return self.IGNORE, 2
        elif char == '.':
            return self.JOIN, 3
        elif char.isdigit():
            return self.JOIN, 4
        elif char.isalpha():
            return self.JOIN, 5
        elif char == '^':
            return self.JOIN, 6
        elif char.isspace():
            return self.IGNORE, 7
        else:
            raise ValueError('TokeniserException: "{}" can not be tokenised'.format(char))
921143
protected void consumeUntilGreedy(Parser recognizer, IntervalSet set, CSSLexerState.RecoveryMode mode) {
        CSSToken t;
        do {
            Token next = recognizer.getInputStream().LT(1);
            if (next instanceof CSSToken) {
                t = (CSSToken) recognizer.getInputStream().LT(1);
                if (t.getType() == Token.EOF) {
                    logger.trace("token eof ");
                    break;
                }
            } else
                break; /* not a CSSToken, probably EOF */
            logger.trace("Skipped greedy: {}", t.getText());
            // consume token even if it will match
            recognizer.consume();
        }
        while (!(t.getLexerState().isBalanced(mode, null, t) && set.contains(t.getType())));
    }
520755
public static function ensure_end_with($string, $trailing_string)
    {
        if (StringTool :: end_with($string, $trailing_string))
        {
            return $string;
        }
        else
        {
            return $string . $trailing_string;
        }
    }
400246
function consumeParenthesised(parser, string) {
    var nesting = 0;
    for (var n = 0; n < string.length; n++) {
      if (/\s|,/.test(string[n]) && nesting == 0) {
        break;
      } else if (string[n] == '(') {
        nesting++;
      } else if (string[n] == ')') {
        nesting--;
        if (nesting == 0)
          n++;
        if (nesting <= 0)
          break;
      }
    }
    var parsed = parser(string.substr(0, n));
    return parsed == undefined ? undefined : [parsed, string.substr(n)];
  }
212741
function consumeWord (css, start) {
    let next = start;
    let code;
    do {
        code = css.charCodeAt(next);
        if (wordDelimiters[code]) {
            return next - 1;
        } else if (code === t.backslash) {
            next = consumeEscape(css, next) + 1;
        } else {
            // All other characters are part of the word
            next++;
        }
    } while (next < css.length);
    return next - 1;
}
398055
public void consumeUntil(Parser recognizer, IntervalSet follow, CSSLexerState.RecoveryMode mode, CSSLexerState ls) {
        CSSToken t;
        boolean finish;
        TokenStream input = recognizer.getInputStream();
        do {
            Token next = input.LT(1);
            if (next instanceof CSSToken) {
                t = (CSSToken) input.LT(1);
                if (t.getType() == Token.EOF) {
                    logger.trace("token eof ");
                    break;
                }
            } else
                break; /* not a CSSToken, probably EOF */
            // consume token if does not match
            finish = (t.getLexerState().isBalanced(mode, ls, t) && follow.contains(t.getType()));
            if (!finish) {
                logger.trace("Skipped: {}", t);
                input.consume();
            }
        } while (!finish);
    }
970875
private Token nextToken() {
        skipSpaces();
        if (hasNextChar()) {
            char ch = nextChar();
            if (Character.isJavaIdentifierStart(ch)) {
                StringBuffer buf = new StringBuffer();
                buf.append(ch);
                while ((ch = peekChar()) != -1
                        && Character.isJavaIdentifierPart(ch)) {
                    buf.append(ch);
                    nextChar();
                }
                return new Id(buf.toString());
            }

            if (ch == '\'' || ch == '"') {
                return parseQuotedChars(ch);
            } else {
                // For now...
                return new Char(ch);
            }
        }
        return null;
    }
913252
public String readUntil(final char... end)
    {
        final StringBuilder sb = new StringBuilder();
        int pos = this.pos;
        while (pos < this.value.length())
        {
            final char ch = this.value.charAt(pos);
            if (ch == '\\' && pos + 1 < this.value.length())
            {
                final char c;
                switch (c = this.value.charAt(pos + 1))
                {
                case '\\':
                case '[':
                case ']':
                case '(':
                case ')':
                case '{':
                case '}':
                case '#':
                case '"':
                case '\'':
                case '.':
                case '>':
                case '*':
                case '+':
                case '-':
                case '_':
                case '!':
                case '`':
                case '~':
                    sb.append(c);
                    pos++;
                    break;
                default:
                    sb.append(ch);
                    break;
                }
            }
            else
            {
                boolean endReached = false;
                for (int n = 0; n < end.length; n++)
                {
                    if (ch == end[n])
                    {
                        endReached = true;
                        break;
                    }
                }
                if (endReached)
                {
                    break;
                }
                sb.append(ch);
            }
            pos++;
        }

        final char ch = pos < this.value.length() ? this.value.charAt(pos) : '\n';
        for (int n = 0; n < end.length; n++)
        {
            if (ch == end[n])
            {
                this.pos = pos;
                return sb.toString();
            }
        }
        return null;
    }
964454
public boolean getBracketedToken(char start, char end, TextProvider textProvider)
    {
        clearLastToken(textProvider);
        clearLeadingSpaces(textProvider);
        mark(textProvider);
        if (m_debug)
			debug("testing " + start + " " + end,textProvider);
        StringBuilder sb = new StringBuilder();
        char c = getNextChar(textProvider);

        if (c != start)
        {
            reset(textProvider);
            return false;
        }
        int brackets = 0;

        while (true)
        {
            if (c == start) brackets++;
            if (c == end) brackets--;
            sb.append(c);
            if (brackets < 1) break;
            c = getNextChar(textProvider);
            if (c == 0)
            {
            	reset(textProvider);
				return false;
            }
        }
        unmark(textProvider);
        String s = sb.toString().trim();
        if (s.length() == 0) return false;
        textProvider.setLastToken(s);
        debug(textProvider);
        return true;
    }
1343377
function consumeWord(forward) {
            log.debug("consumeWord called, forward is " + forward);
            var pos, textChar;
            var newChars = [], it = forward ? forwardIterator : backwardIterator;

            var passedWordBoundary = false, insideWord = false;

            while ( (pos = it.next()) ) {
                textChar = pos.character;

                log.debug("Testing char '" + textChar + "'");

                if (allWhiteSpaceRegex.test(textChar)) {
                    if (insideWord) {
                        insideWord = false;
                        passedWordBoundary = true;
                    }
                } else {
                    log.debug("Got non-whitespace, passedWordBoundary is " + passedWordBoundary);
                    if (passedWordBoundary) {
                        it.rewind();
                        break;
                    } else {
                        insideWord = true;
                    }
                }
                newChars.push(pos);
            }

            log.debug("consumeWord done, pos is " + (pos ? pos.inspect() : "non-existent"));

            log.debug("consumeWord got new chars " + newChars.join(""));
            return newChars;
        }
1301784
def consume_token(self, tokens, index, tokens_len):
        
        del tokens_len

        consumption_ended = False

        q_type = self.quote_type

        begin_literal_type = getattr(TokenType,
                                     "Begin{0}QuotedLiteral".format(q_type))
        end_literal_type = getattr(TokenType,
                                   "End{0}QuotedLiteral".format(q_type))

        if (index != self.begin and
                tokens[index].type == begin_literal_type):
            # This is an edge case where a quote begins a line and matched
            # as a quoted region beginning and a quoted region ending.
            # Split the token before and after the quote, mark the
            # quote character itself as an ending and insert both
            # tokens back in, handling the ending afterwards.
            assert _RE_QUOTE_TYPE.match(tokens[index].content[0])

            # Mini-tokenize everything after the first token
            line_tokens = _scan_for_tokens(tokens[index].content[1:])
            end_type = getattr(TokenType,
                               "End{0}QuotedLiteral".format(q_type))
            replacement = [Token(type=end_type,
                                 content=tokens[index].content[0],
                                 line=tokens[index].line,
                                 col=tokens[index].col)]

            for after in line_tokens:
                replacement.append(Token(type=after.type,
                                         content=after.content,
                                         line=(tokens[index].line +
                                               after.line - 1),
                                         col=(tokens[index].col +
                                              after.col - 1)))

            tokens = _replace_token_range(tokens,
                                          index,
                                          index + 1,
                                          replacement)
            consumption_ended = True

        if tokens[index].type == end_literal_type:
            consumption_ended = True

        if consumption_ended:
            end = index + 1
            pasted = ""
            for i in range(self.begin, end):
                pasted += tokens[i].content

            tokens = _replace_token_range(tokens, self.begin, end,
                                          [Token(type=TokenType.QuotedLiteral,
                                                 content=pasted,
                                                 line=tokens[self.begin].line,
                                                 col=tokens[self.begin].col)])

            return (self.begin, len(tokens), tokens)