A helper method to process the parsed token. This method removes leading and trailing blanks as well as enclosing
quotation marks, when necessary.

@param quoted <tt>true</tt> if quotation marks are expected, <tt>false</tt> otherwise.
@return the token
426217
private String getToken(boolean quoted) {
		// Trim leading white spaces
		while ((i1 < i2) && (Character.isWhitespace(chars[i1]))) {
			i1++;
		}
		// Trim trailing white spaces
		while ((i2 > i1) && (Character.isWhitespace(chars[i2 - 1]))) {
			i2--;
		}
		// Strip away quotation marks if necessary
		if (quoted) {
			if (((i2 - i1) >= 2) && (chars[i1] == '"') && (chars[i2 - 1] == '"')) {
				i1++;
				i2--;
			}
		}
		String result = null;
		if (i2 > i1) {
			result = new String(chars, i1, i2 - i1);
		}
		return result;
	}
904627
static String stringLiteralWithDoubleQuotes(String value, String indent) {
    StringBuilder result = new StringBuilder(value.length() + 2);
    result.append('"');
    for (int i = 0; i < value.length(); i++) {
      char c = value.charAt(i);
      // trivial case: single quote must not be escaped
      if (c == '\'') {
        result.append("'");
        continue;
      }
      // trivial case: double quotes must be escaped
      if (c == '\"') {
        result.append("\\\"");
        continue;
      }
      // default case: just let character literal do its work
      result.append(characterLiteralWithoutSingleQuotes(c));
      // need to append indent after linefeed?
      if (c == '\n' && i + 1 < value.length()) {
        result.append("\"\n").append(indent).append(indent).append("+ \"");
      }
    }
    result.append('"');
    return result.toString();
  }
14569
public char[] quoteAsString(String input) {
        TextBuffer textBuffer = _textBuffer;
        if (textBuffer == null) {
            // no allocator; can add if we must, shouldn't need to
            _textBuffer = textBuffer = new TextBuffer(null);
        }
        char[] outputBuffer = textBuffer.emptyAndGetCurrentSegment();
        final int[] escCodes = sOutputEscapes128;
        final int escCodeCount = escCodes.length;
        int inPtr = 0;
        final int inputLen = input.length();
        int outPtr = 0;

        outer_loop: while (inPtr < inputLen) {
            tight_loop: while (true) {
                char c = input.charAt(inPtr);
                if (c < escCodeCount && escCodes[c] != 0) {
                    break tight_loop;
                }
                if (outPtr >= outputBuffer.length) {
                    outputBuffer = textBuffer.finishCurrentSegment();
                    outPtr = 0;
                }
                outputBuffer[outPtr++] = c;
                if (++inPtr >= inputLen) {
                    break outer_loop;
                }
            }
        // something to escape; 2 or 6-char variant?
        int escCode = escCodes[input.charAt(inPtr++)];
        int length = _appendSingleEscape(escCode, _quoteBuffer);
        if ((outPtr + length) > outputBuffer.length) {
            int first = outputBuffer.length - outPtr;
            if (first > 0) {
                System.arraycopy(_quoteBuffer, 0, outputBuffer, outPtr, first);
            }
            outputBuffer = textBuffer.finishCurrentSegment();
            int second = length - first;
            System.arraycopy(_quoteBuffer, first, outputBuffer, outPtr, second);
            outPtr += second;
        }
        else {
            System.arraycopy(_quoteBuffer, 0, outputBuffer, outPtr, length);
            outPtr += length;
        }

        }
        textBuffer.setCurrentLength(outPtr);
        return textBuffer.contentsAsArray();
    }
271223
private int matchingQuote(int i){
        if(string.charAt(i)!='"')
            throw new UnsupportedOperationException();

        ++i;
        boolean escape = false;
        while(i<string.length()){
            char ch = string.charAt(i);
            if(escape)
                escape = false;
            else if(ch=='\\')
                escape = true;
            else if(ch=='"')
                return i;
            ++i;
        }
        throw new IllegalArgumentException("unterminated quoted-string");
    }
442975
private String quotedAV() {
    pos++;
    beg = pos;
    end = beg;
    while (true) {

      if (pos == length) {
        throw new IllegalStateException("Unexpected end of DN: " + dn);
      }

      if (chars[pos] == '"') {
        // enclosing quotation was found
        pos++;
        break;
      } else if (chars[pos] == '\\') {
        chars[end] = getEscaped();
      } else {
        // shift char: required for string with escaped chars
        chars[end] = chars[pos];
      }
      pos++;
      end++;
    }

    // skip trailing space chars before comma or semicolon.
    // (compatibility with RFC 1779)
    for (; pos < length && chars[pos] == ' '; pos++) {
    }

    return new String(chars, beg, end - beg);
  }
517610
private void checkForTokens( String content, int startOffset, int endOffset )
  {
    while( startOffset <= endOffset )
    {
      // skip the delimiters to find the start of a new token
      while( isDelimiter( content.substring( startOffset, startOffset + 1 ) ) )
      {
        if( isError( startOffset ) )
        {
          setCharacterAttributes( startOffset, 1, _error, true );
        }
        else if( isWarning( startOffset ) )
        {
          setCharacterAttributes( startOffset, 1, _warning, true );
        }

        if( startOffset < endOffset )
        {
          startOffset++;
        }
        else
        {
          return;
        }
      }
      // Extract and process the entire token
      if( isQuoteDelimiter( content.substring( startOffset, startOffset + 1 ) ) )
      {
        startOffset = getQuoteToken( content, startOffset, endOffset );
      }
      else if( isCharQuoteDelimiter( content.substring( startOffset, startOffset + 1 ) ) )
      {
        startOffset = getQuoteToken( content, startOffset, endOffset );
      }
      else
      {
        startOffset = getOtherToken( content, startOffset, endOffset );
      }
    }
  }
14242
public static boolean isValid(String token)
    {
        if (token == null || token.length() == 0)
        {
            return false;
        }

        int len = token.length();

        for (int i = 0; i < len; ++i)
        {
            if (isSeparator(token.charAt(i)))
            {
                return false;
            }
        }

        return true;
    }
830861
private static String convertToQuotedString(String s) {
    if (s == null || s.isEmpty()) {
      return null;
    }
    // If already quoted, return as-is
    if (s.charAt(0) == '"' && s.charAt(s.length() - 1) == '"') {
      return s;
    }
    return '\"' + s + '\"';
  }
662312
public static boolean needsDoubleQuotes(String s) {
    // this method should only be called for C*-provided identifiers,
    // so we expect it to be non-null and non-empty.
    assert s != null && !s.isEmpty();
    char c = s.charAt(0);
    if (!(c >= 97 && c <= 122)) // a-z
    return true;
    for (int i = 1; i < s.length(); i++) {
      c = s.charAt(i);
      if (!((c >= 48 && c <= 57) // 0-9
          || (c == 95) // _
          || (c >= 97 && c <= 122) // a-z
      )) {
        return true;
      }
    }
    return isReservedCqlKeyword(s);
  }
99051
static boolean isEvenQuotes(String string) {
        // In principle, we could use the regex given by:
        // Pattern pEvenQuotes = Pattern.compile("([^\"]*\\\"[^\"]*\\\")*[^\"]*");
        // We assume just counting the instances of double quotes is more efficient
        // but we haven't really tested that assumption.

        boolean even = true;
        for (int i = 0; i < string.length(); i++) {
            if (string.charAt(i) == '\"') {
                even = !even;
            }
        }
        return even;
    }
391028
public static List<String> parse(String line)
	{
		
		if(line == null || line.length() ==0)
			return null;
		
		final short START = 2;
		final short TERM = 3;
		final short QUOTED_TERM = 4;
		final short start_ESCAPEDDOUBLEQUOTE =5;
		
		short state = START;
		
		int length = line.length();
		ArrayList<String> tokens = new ArrayList<>(10);
		
		StringBuilder buffer = new StringBuilder();
		
		char currentChar;
		for (int i = 0; i < length; i++)
		{
			currentChar = line.charAt(i);
			
			switch(state)
			{
				case START:
					if(currentChar == '"')
					{
						state = QUOTED_TERM;
					}
					else if(currentChar != ',')
					{
						  state = TERM;
						  buffer.append(currentChar);
					}

				break; 
				case QUOTED_TERM:
					if(currentChar != '"')
					{
						buffer.append(currentChar);
					}
					else
					{
						if(line.length() > (i+ 1) && line.charAt(i+1) != '"')
						{
							state = START;
							tokens.add(buffer.toString());
							buffer.setLength(0);							
						}
						else if(line.length() > (i+ 1) && line.charAt(i+1) == '"')
						{
							state = start_ESCAPEDDOUBLEQUOTE;
						}
					}
				break;
				case start_ESCAPEDDOUBLEQUOTE:
					
					buffer.append(currentChar);

					state = QUOTED_TERM;
				break;
				case TERM: 
					if(currentChar == ',')
					{
						state = START;
						
						if(buffer.length() > 0)
						{
							tokens.add(buffer.toString());
							buffer.setLength(0);
						}
						
					}
					else if(currentChar == '"' && line.charAt(i+1) == '"' )
					{
						state = start_ESCAPEDDOUBLEQUOTE;
					}
					else
							buffer.append(line.charAt(i));
				break;
				default: throw new RuntimeException("Unknown parse state:"+state);
			}
		}
		
		if(buffer.length() > 0)
			tokens.add(buffer.toString());
			
		tokens.trimToSize();
		
		if(tokens.isEmpty())
			return null;
		
		return tokens;
	
	}
957178
private Token consumeString(char quote) {
    StringBuilder sb = new StringBuilder();
    int l = line;
    int c = column;
    Token T;
    nextChar();
    while(moreChars() && ch != quote) {
      if(ch == '\\') {
        nextChar();
        switch(ch) {
          case '"':
          case '\\':
          case '/':
            sb.append(ch);
            nextChar();
            break;
          case 'b':
            sb.append('\b');
            nextChar();
            break;
          case 'f':
            sb.append('\f');
            nextChar();
            break;
          case 'n':
            sb.append('\n');
            nextChar();
            break;
          case 'r':
            sb.append('\r');
            nextChar();
            break;
          case 't':
            sb.append('\t');
            nextChar();
            break;
          case 'u':
            nextChar();
            int u = 0;
            for(int i = 0; i < 4; i++) {
              if(isHexDigit(ch)) {
                u = u * 16 + ch - '0';
                if(ch >= 'A') { // handle hex numbers: 'A' = 65, '0' = 48. 'A'-'0' = 17, 17 - 7 = 10
                  u = u - 7;
                }
              } else {
                T = new Token(TokenType.ERROR, sb.toString(), line, column);
                nextChar();
                return T;
              }
              nextChar();
            }
            sb.append((char) u);
            break;
          default:
            T = new Token(TokenType.ERROR, sb.toString(), line, column);
            nextChar();
            return T;
        }
      } else {
        sb.append(ch);
        nextChar();
      }
    }
    if(ch == quote) {
      T = new Token(TokenType.STRING, sb.toString(), l, c);
    } else {
      T = new Token(TokenType.ERROR, sb.toString(), line, column);
    }
    nextChar();
    return T;
  }
743629
def _process_string(self, char):
        

        if char in self.QUOTES:
            # end of quoted string:
            #   1) quote must match original quote
            #   2) not escaped quote (e.g. "hey there" vs "hey there\")
            #   3) actual escape char prior (e.g. "hey there\\")

            if (char == self._last_quote and
                    not self._escaped or self._double_escaped):

                # store token
                self._new_token()
                self._state = self.ST_TOKEN
                return  # skip adding token char

        elif char == self.ESCAPE:
            # escape character:
            #   double escaped if prior char was escape (e.g. "hey \\ there")

            if not self._double_escaped:
                self._double_escaped = self._escaped

        else:
            self._double_escaped = False

        self._token_chars.append(char)
1311335
private static boolean needsQuote(String value) {
        if (null == value)
            return true;
        int len = value.length();
        if (0 == len) {
            return true;
        }
        if ('"' == value.charAt(0)) {
            if ('"' == value.charAt(len - 1)) {
                // already wrapped with quotes
                return false;
            }
            // we now know this has a quote (special char) in it, so return now
            return true;
        }

        for (int i = 0; i < len; i++) {
            char c = value.charAt(i);
            if (c < 0x20 || c >= 0x7f || TSPECIALS.indexOf(c) != -1) {
                return true;
            }
        }
        return false;
    }
650202
public static int getBytesToken(ParsingContext ctx) {
        String input = ctx.getInput().substring(ctx.getLocation());
        int tokenOffset = 0;
        int i = 0;
        char[] inputChars = input.toCharArray();
        for (; i < input.length(); i += 1) {
            char c = inputChars[i];
            if (c == ' ') {
                continue;
            }
            if (c != BYTES_TOKEN_CHARS[tokenOffset]) {
                return -1;
            } else {
                tokenOffset += 1;
                if (tokenOffset == BYTES_TOKEN_CHARS.length) {
                    // Found the token.
                    return i;
                }
            }
        }
        return -1;
    }
139457
public static StringBuilder maybeQuoteIdentifier(String s) {
        final EscapeStringResult escapeString = escapeString(s, '\'');
        final StringBuilder result = escapeString.buffer;

        final boolean need_quotes = s.isEmpty() ||
                escapeString.needQuotes ||
                KEYWORDS.contains(s) ||
                !IDENTIFIER_MATCHER.test(s);

        if (need_quotes) {
            result.insert(0, '\'');
            result.append('\'');
        }
        return result;
    }
1367260
protected byte[] parseUnicodeString(Token token) {
		String v = token.text;
		/*
		 * Parsing a string requires several steps to be taken. First, we need
		 * to strip quotes from the ends of the string.
		 */
		v = v.substring(1, v.length() - 1);

		StringBuffer result = new StringBuffer();
		// Second, step through the string and replace escaped characters
		for (int i = 0; i < v.length(); i++) {
			if (v.charAt(i) == '\\') {
				if (v.length() <= i + 1) {
					throw new RuntimeException("unexpected end-of-string");
				} else {
					char replace = 0;
					int len = 2;
					switch (v.charAt(i + 1)) {
					case 'b':
						replace = '\b';
						break;
					case 't':
						replace = '\t';
						break;
					case 'n':
						replace = '\n';
						break;
					case 'f':
						replace = '\f';
						break;
					case 'r':
						replace = '\r';
						break;
					case '"':
						replace = '\"';
						break;
					case '\'':
						replace = '\'';
						break;
					case '\\':
						replace = '\\';
						break;
					case 'u':
						len = 6; // unicode escapes are six digits long,
						// including "slash u"
						String unicode = v.substring(i + 2, i + 6);
						replace = (char) Integer.parseInt(unicode, 16); // unicode
						i = i + 5;
						break;
					default:
						throw new RuntimeException("unknown escape character");
					}
					result.append(replace);
					i = i + 1;
				}
			} else {
				result.append(v.charAt(i));
			}
		}
		try {
			// Now, convert string into a sequence of UTF8 bytes.
			return result.toString().getBytes("UTF8");
		} catch (UnsupportedEncodingException e) {
			// This really should be deadcode
			syntaxError("invalid unicode string", token);
			return null; // deadcode
		}
	}