Skips over a field value.

  Args:
    tokenizer: A tokenizer to parse the field name and values.

  Raises:
    ParseError: In case an invalid field value is found.
559611
def _SkipFieldValue(tokenizer):
  
  # String/bytes tokens can come in multiple adjacent string literals.
  # If we can consume one, consume as many as we can.
  if tokenizer.TryConsumeByteString():
    while tokenizer.TryConsumeByteString():
      pass
    return

  if (not tokenizer.TryConsumeIdentifier() and
      not _TryConsumeInt64(tokenizer) and not _TryConsumeUint64(tokenizer) and
      not tokenizer.TryConsumeFloat()):
    raise ParseError('Invalid field value: ' + tokenizer.token)
286259
def analyze(self, text, index=None, analyzer=None, tokenizer=None, filters=None, field=None):
        
        if filters is None:
            filters = []
        argsets = 0
        args = {}

        if analyzer:
            args['analyzer'] = analyzer
            argsets += 1
        if tokenizer or filters:
            if tokenizer:
                args['tokenizer'] = tokenizer
            if filters:
                args['filters'] = ','.join(filters)
            argsets += 1
        if field:
            args['field'] = field
            argsets += 1

        if argsets > 1:
            raise ValueError('Argument conflict: Specify either analyzer, tokenizer/filters or field')

        if field and index is None:
            raise ValueError('field can only be specified with an index')

        path = make_path(index, '_analyze')
        return self.conn._send_request('POST', path, text, args)
4134
def expect_optional_token(lexer: Lexer, kind: TokenKind) -> Optional[Token]:
    
    token = lexer.token
    if token.kind == kind:
        lexer.advance()
        return token

    return None
1240907
public void skipNext() throws IOException
    {
        nameDeserializer.skipNext();
        nextFlags = nextFlags == Integer.MIN_VALUE ? in.readUnsignedByte() : nextFlags;
        if ((nextFlags & ColumnSerializer.RANGE_TOMBSTONE_MASK) != 0)
            type.rangeTombstoneSerializer().skipBody(in, version);
        else
            type.columnSerializer().skipColumnBody(in, nextFlags);
        nextFlags = Integer.MIN_VALUE;
    }
361855
function processLogicalAnd(tokenizer, scope) {
  let node = processLogicalEquality(tokenizer, scope);

  while (tokenizer.token.match(types.LOGICAL_AND)) {
    node = createOperatorNode(tokenizer.token.value, [node, processLogicalEquality(tokenizer.skipToken(), scope)]);
  }

  return node;
}
729913
def skip(parser, kind):
    # type: (Parser, int) -> bool
    """"""
    match = parser.token.kind == kind
    if match:
        advance(parser)

    return match
197048
def tokenize(self, s):
        
        javabridge.call(self.jobject, "tokenize", "(Ljava/lang/String;)V", s)
        return TokenIterator(self)
681857
def tokenize(stream, separator):
        
        for value in stream:
            for token in value.split(separator):
                if token:
                    yield token.strip()
272333
def expecting_but_got(expecting, got, line, position)
      @tokenizer.expecting_but_got(expecting, got, line, position)
    end
376598
public static function parse(Tokenizer $tokenizer, StreamReader $streamReader)
    {
        $v = new self;
        if (\strspn($streamReader->getByte(), "\x00\x09\x0A\x0C\x0D\x20()<>[]{}/%") === 0) {
            $v->value = (string) $tokenizer->getNextToken();
            return $v;
        }

        $v->value = '';
        return $v;
    }
238946
private static boolean expect(StringTokenizer tokenizer, String token) {
        if (!tokenizer.hasMoreTokens()) {
            return false;
        }
        String s = tokenizer.nextToken();
        if (DEBUG) {
            System.out.println("token=" + s);
        }
        return s.equals(token);
    }
187592
public void skip() {
        int we = wEnd;
        if (wOffset == we) { //requires special handling
            //if no variants at 'wOutside',advance pointer and clear
            if (top == null) {
                wOffset++;
                flush();
            }
            //otherwise, if there exist a variant,
            //don't clear(), i.e. allow it to match
            return;
        } else {
            if (we < 0) wOffset = 0;
            else wOffset = we;
        }
        //rflush(); //rflush() works faster on simple regexes (with a small group/branch number)
        flush();
    }
753616
def tokenize(self, string):
        ''''''
        pos = 0
        while pos < len(string):
            m = SKIP_RE.match(string, pos=pos)
            if m:
                skip_lines = m.group(0).split('\n')
                if len(skip_lines) > 1:
                    self.row += len(skip_lines) - 1
                    self.column = 1 + len(skip_lines[-1])
                else:
                    self.column += len(skip_lines[0])

                pos = m.end()
                continue

            for cls, type, regex in self.token_map:
                m = regex.match(string, pos=pos)
                if m:
                    yield cls(type, m.group(0),
                              self.filename, self.row, self.column)
                    self.column += len(m.group(0))
                    pos = m.end()
                    break
            else:
                raise ConfigParseError(
                    "Couldn't load config in %r row %d, column %d: %r" %
                    (self.filename, self.row, self.column,
                     string[pos:pos+20]))
1321558
def span_tokenize(self, s):
        
        if self._tokenizer is None:
            self._tokenizer = load_model(self.model)
        # for debug in tokenizer.debug_decisions(s):
        #     log.debug(format_debug_decision(debug))
        return self._tokenizer.span_tokenize(s)
944113
private void skipField(JsonParser parser) throws IOException
    {
        int nestedObjects = 1; // we already parsed first '{' or '['
        while (nestedObjects > 0 && parser.nextToken() != null)
        {
            JsonToken token = parser.getCurrentToken();
            if (token == START_OBJECT || token == START_ARRAY)
            {
                nestedObjects++;
            }
            else if (token == END_OBJECT || token == END_ARRAY)
            {
                nestedObjects--;
            }
        }
    }
837066
private void handleToken(String token) throws ParseException
    {
        currentToken = token;

        if (skipParsing)
        {
            cmd.addArg(token);
        }
        else if ("--".equals(token))
        {
            skipParsing = true;
        }
        else if (currentOption != null && currentOption.acceptsArg() && isArgument(token))
        {
            currentOption.addValueForProcessing(Util.stripLeadingAndTrailingQuotes(token));
        }
        else if (token.startsWith("--"))
        {
            handleLongOption(token);
        }
        else if (token.startsWith("-") && !"-".equals(token))
        {
            handleShortAndLongOption(token);
        }
        else
        {
            handleUnknownToken(token);
        }

        if (currentOption != null && !currentOption.acceptsArg())
        {
            currentOption = null;
        }
    }
294479
def _value_encode(cls, member, value):
        
        try:
            field_validator = cls.fields[member]
        except KeyError:
            return cls.valueparse.encode(value)

        return field_validator.encode(value)