Constructs a new instance of {@link S3AUnderFileSystem}.

@param uri the {@link AlluxioURI} for this UFS
@param conf the configuration for this UFS
@param alluxioConf Alluxio configuration
@return the created {@link S3AUnderFileSystem} instance
557753
public static S3AUnderFileSystem createInstance(AlluxioURI uri, UnderFileSystemConfiguration conf,
      AlluxioConfiguration alluxioConf) {

    AWSCredentialsProvider credentials = createAwsCredentialsProvider(conf);
    String bucketName = UnderFileSystemUtils.getBucketName(uri);

    // Set the client configuration based on Alluxio configuration values.
    ClientConfiguration clientConf = new ClientConfiguration();

    // Socket timeout
    clientConf
        .setSocketTimeout((int) alluxioConf.getMs(PropertyKey.UNDERFS_S3A_SOCKET_TIMEOUT_MS));

    // HTTP protocol
    if (Boolean.parseBoolean(conf.get(PropertyKey.UNDERFS_S3A_SECURE_HTTP_ENABLED))) {
      clientConf.setProtocol(Protocol.HTTPS);
    } else {
      clientConf.setProtocol(Protocol.HTTP);
    }

    // Proxy host
    if (conf.isSet(PropertyKey.UNDERFS_S3_PROXY_HOST)) {
      clientConf.setProxyHost(conf.get(PropertyKey.UNDERFS_S3_PROXY_HOST));
    }

    // Proxy port
    if (conf.isSet(PropertyKey.UNDERFS_S3_PROXY_PORT)) {
      clientConf.setProxyPort(Integer.parseInt(conf.get(PropertyKey.UNDERFS_S3_PROXY_PORT)));
    }

    // Number of metadata and I/O threads to S3.
    int numAdminThreads = Integer.parseInt(conf.get(PropertyKey.UNDERFS_S3_ADMIN_THREADS_MAX));
    int numTransferThreads =
        Integer.parseInt(conf.get(PropertyKey.UNDERFS_S3_UPLOAD_THREADS_MAX));
    int numThreads = Integer.parseInt(conf.get(PropertyKey.UNDERFS_S3_THREADS_MAX));
    if (numThreads < numAdminThreads + numTransferThreads) {
      LOG.warn("Configured s3 max threads ({}) is less than # admin threads ({}) plus transfer "
          + "threads ({}). Using admin threads + transfer threads as max threads instead.",
              numThreads, numAdminThreads, numTransferThreads);
      numThreads = numAdminThreads + numTransferThreads;
    }
    clientConf.setMaxConnections(numThreads);

    // Set client request timeout for all requests since multipart copy is used,
    // and copy parts can only be set with the client configuration.
    clientConf
        .setRequestTimeout((int) alluxioConf.getMs(PropertyKey.UNDERFS_S3A_REQUEST_TIMEOUT));

    boolean streamingUploadEnabled =
        conf.getBoolean(PropertyKey.UNDERFS_S3A_STREAMING_UPLOAD_ENABLED);

    // Signer algorithm
    if (conf.isSet(PropertyKey.UNDERFS_S3A_SIGNER_ALGORITHM)) {
      clientConf.setSignerOverride(conf.get(PropertyKey.UNDERFS_S3A_SIGNER_ALGORITHM));
    }

    AmazonS3Client amazonS3Client = new AmazonS3Client(credentials, clientConf);

    // Set a custom endpoint.
    if (conf.isSet(PropertyKey.UNDERFS_S3_ENDPOINT)) {
      amazonS3Client.setEndpoint(conf.get(PropertyKey.UNDERFS_S3_ENDPOINT));
    }

    // Disable DNS style buckets, this enables path style requests.
    if (Boolean.parseBoolean(conf.get(PropertyKey.UNDERFS_S3_DISABLE_DNS_BUCKETS))) {
      S3ClientOptions clientOptions = S3ClientOptions.builder().setPathStyleAccess(true).build();
      amazonS3Client.setS3ClientOptions(clientOptions);
    }

    ExecutorService service = ExecutorServiceFactories
        .fixedThreadPool("alluxio-s3-transfer-manager-worker",
            numTransferThreads).create();

    TransferManager transferManager = TransferManagerBuilder.standard()
        .withS3Client(amazonS3Client).withExecutorFactory(() -> service)
        .withMultipartCopyThreshold(MULTIPART_COPY_THRESHOLD)
        .build();

    return new S3AUnderFileSystem(uri, amazonS3Client, bucketName,
        service, transferManager, conf, alluxioConf, streamingUploadEnabled);
  }
1094655
public static WasbUnderFileSystem createInstance(AlluxioURI uri,
      UnderFileSystemConfiguration conf, AlluxioConfiguration alluxioConf) {
    Configuration wasbConf = createConfiguration(conf);
    return new WasbUnderFileSystem(uri, conf, wasbConf, alluxioConf);
  }
883184
@VisibleForTesting
  static TieredIdentity create(AlluxioConfiguration conf) {
    TieredIdentity scriptIdentity = fromScript(conf);

    List<LocalityTier> tiers = new ArrayList<>();
    List<String> orderedTierNames = conf.getList(PropertyKey.LOCALITY_ORDER, ",");
    for (int i = 0; i < orderedTierNames.size(); i++) {
      String tierName = orderedTierNames.get(i);
      String value = null;
      if (scriptIdentity != null) {
        LocalityTier scriptTier = scriptIdentity.getTier(i);
        Preconditions.checkState(scriptTier.getTierName().equals(tierName));
        value = scriptTier.getValue();
      }
      // Explicit configuration overrides script output.
      if (conf.isSet(Template.LOCALITY_TIER.format(tierName))) {
        value = conf.get(Template.LOCALITY_TIER.format(tierName));
      }
      tiers.add(new LocalityTier(tierName, value));
    }
    // If the user doesn't specify the value of the "node" tier, we fill in a sensible default.
    if (tiers.size() > 0 && tiers.get(0).getTierName().equals(Constants.LOCALITY_NODE)
        && tiers.get(0).getValue() == null) {
      String name = NetworkAddressUtils.getLocalNodeName(conf);
      tiers.set(0, new LocalityTier(Constants.LOCALITY_NODE, name));
    }
    return new TieredIdentity(tiers);
  }
1092466
private URI createServerURI()
            throws URISyntaxException {
        // We store this in plain text for the purposes of simple demonstration,
        // you might want to use something more secure.
        SharedPreferences sharedPref = PreferenceManager.getDefaultSharedPreferences(this.mContext);
        String username = sharedPref.getString(TodoActivity.SETTINGS_CLOUDANT_USER, "");
        String dbName = sharedPref.getString(TodoActivity.SETTINGS_CLOUDANT_DB, "");
        String apiKey = sharedPref.getString(TodoActivity.SETTINGS_CLOUDANT_API_KEY, "");
        String apiSecret = sharedPref.getString(TodoActivity.SETTINGS_CLOUDANT_API_SECRET, "");
        String host = username + ".cloudant.com";

        // We recommend always using HTTPS to talk to Cloudant.
        return new URI("https", apiKey + ":" + apiSecret, host, 443, "/" + dbName, null, null);
    }
49003
public static void addCacheFile(URI uri, Configuration conf) {
    String files = conf.get("mapred.cache.files");
    conf.set("mapred.cache.files", files == null ? uri.toString() : files + ","
             + uri.toString());
  }
137651
public static UriComponentsBuilder fromUri(URI uri) {
		UriComponentsBuilder builder = new UriComponentsBuilder();
		builder.uri(uri);
		return builder;
	}
321311
private String createUri(String uri) {
        try {
            return FileUtil.URI_SEPARATOR_CHAR
                + FilenameUtils.getPath(uri)
                + URLEncoder.encode(FilenameUtils.getBaseName(uri), StandardCharsets.UTF_8.name())
                + config.getOutputExtension();
        } catch (UnsupportedEncodingException e) {
            throw new RuntimeException("Missing UTF-8 encoding??", e); // Won't happen unless JDK is broken.
        }
    }
342390
public function fromUri(Uri $uri): Configuration
    {
        $filename = (string) $uri;

        if (!file_exists($filename)) {
            throw new \InvalidArgumentException(sprintf('File %s could not be found', $filename));
        }

        $xml = new \SimpleXMLElement($filename, 0, true);
        foreach ($this->strategies as $strategy) {
            if ($strategy->supports($xml) !== true) {
                continue;
            }

            return new Configuration($this->applyMiddleware($strategy->convert($xml)));
        }

        throw new RuntimeException('No supported configuration files were found');
    }
676022
public LockedInodePath lockFullInodePath(AlluxioURI uri, LockPattern lockPattern)
      throws InvalidPathException, FileDoesNotExistException {
    LockedInodePath inodePath = lockInodePath(uri, lockPattern);
    if (!inodePath.fullPathExists()) {
      inodePath.close();
      throw new FileDoesNotExistException(ExceptionMessage.PATH_DOES_NOT_EXIST.getMessage(uri));
    }
    return inodePath;
  }
1356568
private void checkUri(AlluxioURI uri) {
    Preconditions.checkNotNull(uri, "uri");
    if (uri.hasScheme()) {
      String warnMsg = "The URI scheme \"{}\" is ignored and not required in URIs passed to"
          + " the Alluxio Filesystem client.";
      switch (uri.getScheme()) {
        case Constants.SCHEME:
          LOG.warn(warnMsg, Constants.SCHEME);
          break;
        case Constants.SCHEME_FT:
          LOG.warn(warnMsg, Constants.SCHEME_FT);
          break;
        default:
          throw new IllegalArgumentException(
              String.format("Scheme %s:// in AlluxioURI is invalid. Schemes in filesystem"
                  + " operations are ignored. \"alluxio://\" or no scheme at all is valid.",
                  uri.getScheme()));
      }
    }

    if (uri.hasAuthority()) {
      LOG.warn("The URI authority (hostname and port) is ignored and not required in URIs passed "
          + "to the Alluxio Filesystem client.");
      /* Even if we choose to log the warning, check if the Configuration host matches what the
       * user passes. If not, throw an exception letting the user know they don't match.
       */
      Authority configured =
          MasterInquireClient.Factory.create(mFsContext.getClusterConf())
              .getConnectDetails().toAuthority();
      if (!configured.equals(uri.getAuthority())) {
        throw new IllegalArgumentException(
            String.format("The URI authority %s does not match the configured " + "value of %s.",
                uri.getAuthority(), configured));
      }
    }
    return;
  }
665860
public static Observable<List<File>> createFileFromUri(final Context context,
      final ArrayList<Uri> uris) {
    return createFileFromUri(context, uris, MimeMap.UrlConnection);
  }
386829
public AlluxioConfiguration getPathConf(AlluxioURI path) {
    return new SpecificPathConfiguration(mClientContext.getConf(), mClientContext.getPathConf(),
        path);
  }
493471
public UriComponentsBuilder uri(URI uri) {
		Assert.notNull(uri, "'uri' must not be null");
		this.scheme = uri.getScheme();
		if (uri.isOpaque()) {
			this.ssp = uri.getRawSchemeSpecificPart();
			resetHierarchicalComponents();
		}
		else {
			if (uri.getRawUserInfo() != null) {
				this.userInfo = uri.getRawUserInfo();
			}
			if (uri.getHost() != null) {
				this.host = uri.getHost();
			}
			if (uri.getPort() != -1) {
				this.port = String.valueOf(uri.getPort());
			}
			if (StringUtils.hasLength(uri.getRawPath())) {
				this.pathBuilder = new CompositePathComponentBuilder(uri.getRawPath());
			}
			if (StringUtils.hasLength(uri.getRawQuery())) {
				this.queryParams.clear();
				query(uri.getRawQuery());
			}
			resetSchemeSpecificPart();
		}
		if (uri.getRawFragment() != null) {
			this.fragment = uri.getRawFragment();
		}
		return this;
	}
237294
List<AlluxioURI> checkConsistency(AlluxioURI path, CheckConsistencyPOptions options)
      throws IOException {
    FileSystemMasterClient client = mFsContext.acquireMasterClient();
    try {
      return client.checkConsistency(path, options);
    } finally {
      mFsContext.releaseMasterClient(client);
    }
  }
169689
private URI createFileSystemURI(URI configKey) throws URISyntaxException, IOException {
    // Validate the scheme
    String configKeyScheme = configKey.getScheme();
    if (!configKeyScheme.startsWith(getSchemePrefix())) {
      throw new IllegalArgumentException(
          String.format("Scheme for configKey \"%s\" must begin with \"%s\"!", configKey, getSchemePrefix()));
    }

    if (Strings.isNullOrEmpty(configKey.getAuthority())) {
      return new URI(getPhysicalScheme(), getDefaultStoreFsLazy().getUri().getAuthority(), "", "", "");
    }
    String uriPhysicalScheme = configKeyScheme.substring(getSchemePrefix().length(), configKeyScheme.length());
    return new URI(uriPhysicalScheme, configKey.getAuthority(), "", "", "");
  }
748500
public synchronized void initialize(URI uri, org.apache.hadoop.conf.Configuration conf,
      @Nullable AlluxioConfiguration alluxioConfiguration)
      throws IOException {
    Preconditions.checkArgument(uri.getScheme().equals(getScheme()),
        PreconditionMessage.URI_SCHEME_MISMATCH.toString(), uri.getScheme(), getScheme());

    super.initialize(uri, conf);
    LOG.debug("initialize({}, {}). Connecting to Alluxio", uri, conf);
    HadoopUtils.addSwiftCredentials(conf);
    setConf(conf);

    // HDFS doesn't allow the authority to be empty; it must be "/" instead.
    String authority = uri.getAuthority() == null ? "/" : uri.getAuthority();
    mAlluxioHeader = getScheme() + "://" + authority;
    // Set the statistics member. Use mStatistics instead of the parent class's variable.
    mStatistics = statistics;

    Authority auth = Authority.fromString(uri.getAuthority());
    if (auth instanceof UnknownAuthority) {
      throw new IOException(String.format("Authority \"%s\" is unknown. The client can not be "
              + "configured with the authority from %s", auth, uri));
    }

    mUri = URI.create(mAlluxioHeader);

    if (mFileSystem != null) {
      return;
    }

    Map<String, Object> uriConfProperties = getConfigurationFromUri(uri);

    AlluxioProperties alluxioProps =
        (alluxioConfiguration != null) ? alluxioConfiguration.copyProperties()
            : ConfigurationUtils.defaults();
    AlluxioConfiguration alluxioConf = mergeConfigurations(uriConfProperties, conf, alluxioProps);
    Subject subject = getHadoopSubject();
    if (subject != null) {
      LOG.debug("Using Hadoop subject: {}", subject);
    } else {
      LOG.debug("No Hadoop subject. Using context without subject.");
    }

    LOG.info("Initializing filesystem with connect details {}",
        Factory.getConnectDetails(alluxioConf));

    mFileSystem = FileSystem.Factory.create(ClientContext.create(subject, alluxioConf));
  }
950843
public UnderFileSystemConfiguration createMountSpecificConf(Map<String, String> mountConf) {
    UnderFileSystemConfiguration ufsConf = new UnderFileSystemConfiguration(mProperties.copy());
    ufsConf.mProperties.merge(mountConf, Source.MOUNT_OPTION);
    ufsConf.mReadOnly = mReadOnly;
    ufsConf.mShared = mShared;
    return ufsConf;
  }