Extract tokens from request.
@param req Request.
@return List of user agent tokens.
@throws IOException If some problems inside.
196706
private static List<String> tokens(final Request req) throws IOException {
        final List<String> tokens = new LinkedList<>();
        final Iterable<String> headers =
            new RqHeaders.Base(req).header("User-Agent");
        for (final String header : headers) {
            final Matcher matcher = PATTERN.matcher(header);
            if (matcher.matches()) {
                tokens.add(matcher.group());
            }
        }
        return tokens;
    }
1010139
private String[] getTokens(String line) {

    // these are fine because they do not affect offsets
    line = line.trim();
    line = doubleSpaces.matcher(line).replaceAll(" ");
    // remove non printable stuff
    line = asciiHex.matcher(line).replaceAll(" ");
    line = generalBlankPunctuation.matcher(line).replaceAll(" ");

    // separate question and exclamation marks
    line = qexc.matcher(line).replaceAll(" $1 ");
    // separate dash if before or after space
    line = spaceDashSpace.matcher(line).replaceAll(" $1 ");
    // tokenize everything but these characters [^\p{Alnum}s.'`,-?!/]
    line = specials.matcher(line).replaceAll(" $1 ");

    // do not separate multidots
    line = generateMultidots(line);

    // separate "," except if within numbers (1,200)
    line = noDigitComma.matcher(line).replaceAll("$1 $2");
    line = commaNoDigit.matcher(line).replaceAll("$1 $2");
    // separate pre and post digit
    line = digitCommaNoDigit.matcher(line).replaceAll("$1 $2 $3");
    line = noDigitCommaDigit.matcher(line).replaceAll("$1 $2 $3");

    // contractions it's, l'agila, c'est, don't
    line = treatContractions(line);
    // exceptions for period tokenization
    line = this.nonBreaker.TokenizerNonBreaker(line);

    // restore multidots
    line = restoreMultidots(line);
    // urls
    // TODO normalize URLs after tokenization for offsets
    line = detokenizeURLs(line);
    line = beginLink.matcher(line).replaceAll("$1://");
    line = endLink.matcher(line).replaceAll("$1$2");

    // these are fine because they do not affect offsets
    line = line.trim();
    line = doubleSpaces.matcher(line).replaceAll(" ");
    line = detokenParagraphs.matcher(line).replaceAll("$1$2");

    if (DEBUG) {
      System.out.println("->Tokens:" + line);
    }
    final String[] tokens = line.split(" ");
    return tokens;
  }
369839
private void processSamlToken(String[] rets, Socket socket) throws IOException, SFException
  {
    String targetLine = null;
    String userAgent = null;
    boolean isPost = false;
    for (String line : rets)
    {
      if (line.length() > PREFIX_GET.length() &&
          line.substring(0, PREFIX_GET.length()).equalsIgnoreCase(PREFIX_GET))
      {
        targetLine = line;
      }
      else if (line.length() > PREFIX_POST.length() &&
               line.substring(0, PREFIX_POST.length()).equalsIgnoreCase(PREFIX_POST))
      {
        targetLine = rets[rets.length - 1];
        isPost = true;
      }
      else if (line.length() > PREFIX_USER_AGENT.length() &&
               line.substring(0, PREFIX_USER_AGENT.length()).equalsIgnoreCase(PREFIX_USER_AGENT))
      {
        userAgent = line;
      }
    }
    if (targetLine == null)
    {
      throw new SFException(ErrorCode.NETWORK_ERROR,
                            "Invalid HTTP request. No token is given from the browser.");
    }
    if (userAgent != null)
    {
      logger.debug("{}", userAgent);
    }

    try
    {
      // attempt to get JSON response
      extractJsonTokenFromPostRequest(targetLine);
    }
    catch (IOException ex)
    {
      String parameters = isPost ?
                          extractTokenFromPostRequest(targetLine) :
                          extractTokenFromGetRequest(targetLine);
      try
      {
        URI inputParameter = new URI(parameters);
        for (NameValuePair urlParam : URLEncodedUtils.parse(
            inputParameter, UTF8_CHARSET))
        {
          if ("token".equals(urlParam.getName()))
          {
            this.token = urlParam.getValue();
            break;
          }
        }
      }
      catch (URISyntaxException ex0)
      {
        throw new SFException(ErrorCode.NETWORK_ERROR,
                              String.format(
                                  "Invalid HTTP request. No token is given from the browser. %s, err: %s",
                                  targetLine, ex0));
      }
    }
    if (this.token == null)
    {
      throw new SFException(ErrorCode.NETWORK_ERROR,
                            String.format(
                                "Invalid HTTP request. No token is given from the browser: %s",
                                targetLine));
    }

    returnToBrowser(socket);
  }
1339935
private String[] extractAndDecodeHeader(String header) throws IOException {

        byte[] base64Token = header.substring(6).getBytes(CREDENTIALS_CHARSET);
        byte[] decoded;
        try {
            decoded = Base64.decode(base64Token);
        } catch (IllegalArgumentException e) {
            throw new BadCredentialsException("Failed to decode basic authentication token");
        }

        String token = new String(decoded, CREDENTIALS_CHARSET);

        int delim = token.indexOf(":");

        if (delim == -1) {
            throw new BadCredentialsException("Invalid basic authentication token");
        }
        return new String[] {token.substring(0, delim), token.substring(delim + 1)};
    }
1057876
public UserRecoverRequest retrieveRecoverRequest(String token) {
		UserRecoverRequest req = this.dao.exists(token, UserRecoverRequest.class);
		if (req == null)
			return null;
		if (req.getExpiration().getTime() < System.currentTimeMillis())
			return null;
		if (req.isUsed())
			return null;
		return req;
	}
867684
private Map<String, List<Request>> requests(
        final Request req) throws IOException {
        final String header = new RqHeaders.Smart(req).single("Content-Type");
        if (!new EnglishLowerCase(header).string()
            .startsWith("multipart/form-data")) {
            throw new HttpException(
                HttpURLConnection.HTTP_BAD_REQUEST,
                String.format(
                    // @checkstyle LineLength (1 line)
                    "RqMtBase can only parse multipart/form-data, while Content-Type specifies a different type: \"%s\"",
                    header
                )
            );
        }
        final Matcher matcher = RqMtBase.BOUNDARY.matcher(header);
        if (!matcher.matches()) {
            throw new HttpException(
                HttpURLConnection.HTTP_BAD_REQUEST,
                String.format(
                    "boundary is not specified in Content-Type header: \"%s\"",
                    header
                )
            );
        }
        final ReadableByteChannel body = Channels.newChannel(this.stream);
        if (body.read(this.buffer) < 0) {
            throw new HttpException(
                HttpURLConnection.HTTP_BAD_REQUEST,
                "failed to read the request body"
            );
        }
        final byte[] boundary = String.format(
            "%s--%s", RqMtBase.CRLF, matcher.group(1)
        ).getBytes(RqMtBase.ENCODING);
        this.buffer.flip();
        this.buffer.position(boundary.length - 2);
        final Collection<Request> requests = new LinkedList<>();
        while (this.buffer.hasRemaining()) {
            final byte data = this.buffer.get();
            if (data == '-') {
                break;
            }
            this.buffer.position(this.buffer.position() + 1);
            requests.add(this.make(boundary, body));
        }
        return RqMtBase.asMap(requests);
    }
815943
public static Useragent getUserAgent(HttpServletRequest request) {
    String head = request.getHeader("USER-AGENT");
    Useragent agent = new Useragent(getIpAddr(request), Browser.parse(head), Os.parse(head));
    if (agent.getOs().equals(OsCategory.Unknown) || agent.getBrowser().equals(BrowserCategory.Unknown)) {
      logger.info("Cannot parser user agent:{}", request.getHeader("USER-AGENT"));
    }
    return agent;
  }
366965
public Token<DelegationTokenIdentifier> selectTokenFromFsOwner() throws IOException {
    return lookupToken(user.getCredentials(), service, tokenBinding.getKind());
  }
743135
public AbucoinsOrder[] getAbucoinsOrders(AbucoinsOrderRequest request) throws IOException {
    AbucoinsOrder.Status status = null;
    String productID = null;
    if (request != null) {
      status = request.getStatus();
      productID = request.getProductID();
    }

    if (status != null) {
      switch (status) {
        default:
        case open:
        case done:
          break;

        case pending:
        case rejected:
          throw new IllegalArgumentException(
              "/orders only accepts status of 'open' or 'done' not " + status);
      }
    }

    AbucoinsOrders retVal = null;
    if (status == null) {
      if (productID == null)
        retVal =
            abucoinsAuthenticated.getOrders(
                exchange.getExchangeSpecification().getApiKey(),
                signatureCreator,
                exchange.getExchangeSpecification().getPassword(),
                timestamp());
      else
        retVal =
            abucoinsAuthenticated.getOrdersByProductID(
                productID,
                exchange.getExchangeSpecification().getApiKey(),
                signatureCreator,
                exchange.getExchangeSpecification().getPassword(),
                timestamp());
    } else {
      if (productID == null)
        retVal =
            abucoinsAuthenticated.getOrdersByStatus(
                status.name(),
                exchange.getExchangeSpecification().getApiKey(),
                signatureCreator,
                exchange.getExchangeSpecification().getPassword(),
                timestamp());
      else
        retVal =
            abucoinsAuthenticated.getOrdersByStatusAndProductID(
                status.name(),
                productID,
                exchange.getExchangeSpecification().getApiKey(),
                signatureCreator,
                exchange.getExchangeSpecification().getPassword(),
                timestamp());
    }

    if (retVal.getOrders().length == 1 && retVal.getOrders()[0].getMessage() != null)
      throw new ExchangeException(retVal.getOrders()[0].getMessage());

    return retVal.getOrders();
  }
1274505
public List<TCert> getTCertBatch(GetTCertBatchRequest req) throws GetTCertBatchException {
        logger.debug(String.format("[MemberServicesImpl.getTCertBatch] [%s]", req));

        try {
            // create the proto
            TCertCreateSetReq.Builder tCertCreateSetReq = TCertCreateSetReq.newBuilder()
                    .setTs(Timestamp.newBuilder().setSeconds(new java.util.Date().getTime()))
                    .setId(Identity.newBuilder().setId(req.getName()))
                    .setNum(req.getNum());

            if (req.getAttrs() != null) {
                for (String attr : req.getAttrs()) {
                    tCertCreateSetReq.addAttributes(TCertAttribute.newBuilder().setAttributeName(attr).build());
                }
            }

            // serialize proto
            byte[] buf = tCertCreateSetReq.buildPartial().toByteArray();

            // sign the transaction using enrollment key
            java.security.PrivateKey signKey = cryptoPrimitives.ecdsaKeyFromPrivate(Hex.decode(req.getEnrollment().getKey()));
            BigInteger[] sig = cryptoPrimitives.ecdsaSign(signKey, buf);
            Signature protoSig = Signature.newBuilder().setType(CryptoType.ECDSA).setR(ByteString.copyFrom(sig[0].toString().getBytes())).setS(ByteString.copyFrom(sig[1].toString().getBytes())).build();
            tCertCreateSetReq.setSig(protoSig);

            // send the request
            TCertCreateSetResp tCertCreateSetResp = tcapClient.createCertificateSet(tCertCreateSetReq.build());
            logger.debug("[MemberServicesImpl.getTCertBatch] tCertCreateSetResp : [%s]" + tCertCreateSetResp.toByteString());

            return processTCertBatch(req, tCertCreateSetResp);
        } catch (Exception e) {
            throw new GetTCertBatchException("Failed to get tcerts", e);
        }
    }
1126925
public UserAgentInfo parse(String useragent) throws IOException {
		UserAgentInfo retObj = new UserAgentInfo();

		if (useragent == null) {
			return retObj;
		}
		useragent = useragent.trim();

		// check that the data maps are up-to-date
		checkDataMaps();

		// first check if it's a robot
		if (!processRobot(useragent, retObj)) {
			// search for a browser on the browser regex patterns
			boolean osFound = processBrowserRegex(useragent, retObj);

			if (!osFound) {
				// search the OS regex patterns for the used OS
				processOsRegex(useragent, retObj);
			}
		}
		return retObj;
	}
1062738
public static List<String> tokenize_path(String path) {
        String[] tokens = path.split(SEPARATOR);
        java.util.ArrayList<String> rtn = new ArrayList<>();
        for (String str : tokens) {
            if (!str.isEmpty()) {
                rtn.add(str);
            }
        }
        return rtn;
    }
24996
public static String getToken(HttpServletRequest request, ActionMapping mapping)
    {
        if (mapping instanceof PageFlowActionMapping && ((PageFlowActionMapping) mapping).isPreventDoubleSubmit()) {
            HttpSession session = request.getSession();
            String token = (String) session.getAttribute(Globals.TRANSACTION_TOKEN_KEY);
            if (token != null) return token;
            token = TokenProcessor.getInstance().generateToken(request);
            request.getSession().setAttribute(Globals.TRANSACTION_TOKEN_KEY, token);
            return token;
        }

        return null;
    }
1239712
public static String getHttpServletRequestUserAgent(final HttpServletRequest request) {
        if (request != null) {
            return request.getHeader(USER_AGENT_HEADER);
        }
        return null;
    }
915110
private static List<String> tokenize(final ShrinkWrapPath path) {
        final StringTokenizer tokenizer = new StringTokenizer(path.toString(), ArchivePath.SEPARATOR_STRING);
        final List<String> tokens = new ArrayList<>();
        while (tokenizer.hasMoreTokens()) {
            tokens.add(tokenizer.nextToken());
        }
        return tokens;
    }
76283
public static List<String> extractResponseCodes(JavadocComment jdoc){
		List<String> list = new ArrayList<>();
		list.addAll(extractDocAnnotation(DOC_RESPONSE_CODE,jdoc));

		return list;
	}
11805
public OutputStream extractData(
                    final ClientHttpResponse response
                    )
    throws IOException
    {
        IoUtil.copy( response.getBody(), _output );

        return _output;
    }