We leave a file behind in job submission directory so that clr client can figure out
the applicationId and yarn rest endpoint.
@param driverFolder
@param applicationId
@throws IOException
199373
private void writeDriverHttpEndPoint(final File driverFolder,
                                       final String applicationId,
                                       final Path dfsPath) throws  IOException {
    final FileSystem fs = FileSystem.get(yarnConfiguration);
    final Path httpEndpointPath = new Path(dfsPath, fileNames.getDriverHttpEndpoint());

    String trackingUri = null;
    LOG.log(Level.INFO, "Attempt to reading " + httpEndpointPath.toString());
    for (int i = 0; i < 60; i++) {
      try {
        LOG.log(Level.FINE, "Attempt " + i + " reading " + httpEndpointPath.toString());
        if (fs.exists(httpEndpointPath)) {
          final FSDataInputStream input = fs.open(httpEndpointPath);
          final BufferedReader reader = new BufferedReader(new InputStreamReader(input, StandardCharsets.UTF_8));
          trackingUri = reader.readLine();
          reader.close();
          break;
        }
      } catch (Exception ignored) {
        // readLine might throw IOException although httpEndpointPath file exists.
        // the for-loop waits until the actual content of file is written completely
      }
      try{
        Thread.sleep(1000);
      } catch(InterruptedException ex2) {
        break;
      }
    }

    if (null == trackingUri) {
      trackingUri = "";
      LOG.log(Level.WARNING, "Failed reading " + httpEndpointPath.toString());
    } else {
      LOG.log(Level.INFO, "Completed reading trackingUri :" + trackingUri);
    }

    final File driverHttpEndpointFile = new File(driverFolder, fileNames.getDriverHttpEndpoint());
    BufferedWriter out = new BufferedWriter(new OutputStreamWriter(
                   new FileOutputStream(driverHttpEndpointFile), StandardCharsets.UTF_8));
    out.write(applicationId + "\n");
    out.write(trackingUri + "\n");
    String addr = yarnConfiguration.get("yarn.resourcemanager.webapp.address");
    if (null == addr || addr.startsWith("0.0.0.0")) {
      String str2 = yarnConfiguration.get("yarn.resourcemanager.ha.rm-ids");
      if (null != str2) {
        for (String rm : str2.split(",")) {
          out.write(yarnConfiguration.get("yarn.resourcemanager.webapp.address." + rm) +"\n");
        }
      }
    } else {
      out.write(addr +"\n");
    }
    out.close();
  }
235761
public void deployApplication(String applicationName, String... classpathLocations) throws IOException {

        final List<URL> classpathElements = Arrays.stream(classpathLocations)
            .map(classpath -> Thread.currentThread().getContextClassLoader().getResource(classpath))
            .collect(Collectors.toList());

        deployApplication(applicationName, classpathElements.toArray(new URL[classpathElements.size()]));
    }
145106
private void initJobDir (SourceState state) throws IOException {
    String tmpBase = state.getProp(MRCompactor.COMPACTION_TMP_DEST_DIR, MRCompactor.DEFAULT_COMPACTION_TMP_DEST_DIR);
    String jobId;

    if (state instanceof JobState) {
      jobId = ((JobState) state).getJobId();
    } else {
      jobId = UUID.randomUUID().toString();
    }

    this.tmpJobDir = new Path (tmpBase, jobId);
    this.fs.mkdirs(this.tmpJobDir);
    state.setProp (MRCompactor.COMPACTION_JOB_DIR, tmpJobDir.toString());
    log.info ("Job dir is created under {}", this.tmpJobDir);
  }
1095667
private ProjectFile handleDatabaseInDirectory(File directory) throws Exception
   {
      byte[] buffer = new byte[BUFFER_SIZE];
      File[] files = directory.listFiles();
      if (files != null)
      {
         for (File file : files)
         {
            if (file.isDirectory())
            {
               continue;
            }

            FileInputStream fis = new FileInputStream(file);
            int bytesRead = fis.read(buffer);
            fis.close();

            //
            // If the file is smaller than the buffer we are peeking into,
            // it's probably not a valid schedule file.
            //
            if (bytesRead != BUFFER_SIZE)
            {
               continue;
            }

            if (matchesFingerprint(buffer, BTRIEVE_FINGERPRINT))
            {
               return handleP3BtrieveDatabase(directory);
            }

            if (matchesFingerprint(buffer, STW_FINGERPRINT))
            {
               return handleSureTrakDatabase(directory);
            }
         }
      }
      return null;
   }
500725
public static void main(final String[] args) throws IOException, InjectionException {
    final ClientProtocol.DriverClientConfiguration driverClientConfigurationProto;
    if (args.length > 0) {
      final String content = new String(Files.readAllBytes(Paths.get(args[0])));
      final ClientProtocol.DriverClientConfiguration.Builder driverClientConfigurationProtoBuilder =
          ClientProtocol.DriverClientConfiguration.newBuilder();
      JsonFormat.parser()
          .usingTypeRegistry(JsonFormat.TypeRegistry.getEmptyTypeRegistry())
          .merge(content, driverClientConfigurationProtoBuilder);
      driverClientConfigurationProto = driverClientConfigurationProtoBuilder.build();
      final int port = (args.length == 2) ? Integer.parseInt(args[1]) : 0;
      final LauncherStatus status = submit(port, driverClientConfigurationProto);
      LOG.log(Level.INFO, "Status: {0}", status);
    } else {
      LOG.log(Level.SEVERE, ClientLauncher.class.getName() +
          " accepts single required argument referencing a file that contains a " +
          "client protocol buffer driver configuration");
      System.exit(1);
    }
  }
299606
public static DriverFiles fromJobSubmission(
      final JobSubmissionEvent jobSubmissionEvent,
      final REEFFileNames fileNames) throws IOException {

    final DriverFiles driverFiles = new DriverFiles(fileNames);

    for (final FileResource frp : jobSubmissionEvent.getGlobalFileSet()) {
      final File f = new File(frp.getPath());
      if (frp.getType() == FileType.LIB) {
        driverFiles.addGlobalLib(f);
      } else {
        driverFiles.addGlobalFile(f);
      }
    }

    for (final FileResource frp : jobSubmissionEvent.getLocalFileSet()) {
      final File f = new File(frp.getPath());
      if (frp.getType() == FileType.LIB) {
        driverFiles.addLocalLib(f);
      } else {
        driverFiles.addLocalFile(f);
      }
    }

    return driverFiles;
  }
828888
private void createLibFolder() throws IOException {
        if(!Files.exists(libLocation.toPath()))
            Files.createDirectories(libLocation.toPath());
    }
840641
public static List<File> createRecipeDirectories( File applicationDirectory ) throws IOException {

		List<File> result = new ArrayList<> ();
		ApplicationLoadResult alr = RuntimeModelIo.loadApplicationFlexibly( applicationDirectory );
		if( alr.getApplicationTemplate() != null ) {
			for( Component c : ComponentHelpers.findAllComponents( alr.getApplicationTemplate())) {
				File directory = ResourceUtils.findInstanceResourcesDirectory( applicationDirectory, c );
				if( ! directory.exists())
					result.add( directory );

				Utils.createDirectory( directory );
			}
		}

		return result;
	}
493826
public static void main(final String[] args) {
    LOG.log(Level.INFO, "Entering Launch at :::" + new Date());
    try {
      if (args == null || args.length == 0) {
        throw new IllegalArgumentException("No arguments provided, at least a clrFolder should be supplied.");
      }
      final File dotNetFolder = new File(args[0]).getAbsoluteFile();
      final String[] removedArgs = Arrays.copyOfRange(args, 1, args.length);

      final Configuration config = getClientConfiguration(removedArgs);
      final Injector commandLineInjector = Tang.Factory.getTang().newInjector(parseCommandLine(removedArgs));
      final int waitTime = commandLineInjector.getNamedInstance(WaitTimeForDriver.class);
      final int driverMemory = commandLineInjector.getNamedInstance(DriverMemoryInMb.class);
      final boolean isLocal = commandLineInjector.getNamedInstance(Local.class);
      final String driverIdentifier = commandLineInjector.getNamedInstance(DriverIdentifier.class);
      final String jobSubmissionDirectory = commandLineInjector.getNamedInstance(DriverJobSubmissionDirectory.class);
      final boolean submit = commandLineInjector.getNamedInstance(Submit.class);
      final Injector injector = Tang.Factory.getTang().newInjector(config);
      final JobClient client = injector.getInstance(JobClient.class);
      client.setDriverInfo(driverIdentifier, driverMemory, jobSubmissionDirectory);

      if (submit) {
        client.submit(dotNetFolder, true, isLocal, null);
        client.waitForCompletion(waitTime);
      } else {
        client.submit(dotNetFolder, false, isLocal, config);
        client.waitForCompletion(0);
      }


      LOG.info("Done!");
    } catch (final BindException | InjectionException | IOException ex) {
      LOG.log(Level.SEVERE, "Job configuration error", ex);
    }
  }
446477
public JenkinsServer createFolder(FolderJob folder, String jobName) throws IOException {
        return createFolder(folder, jobName, false);
    }
898112
private void writeDriverWebInformation(final HttpServletResponse response) throws IOException {

    LOG.log(Level.INFO, "HttpServerReefEventHandler writeDriverWebInformation invoked.");

    final PrintWriter writer = response.getWriter();

    writer.println("<h1>Driver Information:</h1>");

    writer.println(String.format("Driver Remote Identifier:[%s]",
        this.reefStateManager.getDriverEndpointIdentifier()));
    writer.write("<br/><br/>");

    writer.println(String.format("Services registered on Driver:"));
    writer.write("<br/><br/>");
    for (final AvroReefServiceInfo service : this.reefStateManager.getServicesInfo()) {
      writer.println(String.format("Service: [%s] , Information: [%s]", service.getServiceName(),
          service.getServiceInfo()));
      writer.write("<br/><br/>");
    }

    writer.println(String.format("Driver Start Time:[%s]", this.reefStateManager.getStartTime()));
  }
1022848
public void initDriverFromFile(File shpFile, String forceEncoding) throws IOException {             // Read columns from files metadata
        this.shpFile = shpFile;        
        
        if(!shpFile.exists()){
            throw new FileNotFoundException("The following file does not exists: " + shpFile.getPath());
        }        
        // Find appropriate file extension for shx and dbf, maybe SHX or Shx..
        //String shxFileName = shpFile.getName();
        
        String fileName = shpFile.getAbsolutePath();
        final int dotIndex = fileName.lastIndexOf('.');
        final String fileNamePrefix = fileName.substring(0, dotIndex).toLowerCase();
                     
        DirectoryStream.Filter<Path> filter = new DirectoryStream.Filter<Path>() {            
            @Override
            public boolean accept(Path entry) throws IOException {
                String path = entry.toAbsolutePath().toString().toLowerCase();
                if(path.equals(fileNamePrefix+".shx")){
                    shxFile = entry.toFile();
                    return true;
                }
                else if(path.equals(fileNamePrefix+".dbf")){
                    dbfFile = entry.toFile();
                    return true;
                }
                else if(path.equals(fileNamePrefix+".prj")){
                    prjFile = entry.toFile();
                    return true;
                }
                return false;
            }

        };
        
        try (DirectoryStream<Path> stream = Files.newDirectoryStream(shpFile.getParentFile().toPath(), filter)) {            
            for (Path pathDir : stream) {                
               //Do nothing   
            } 
        }
        if(dbfFile != null) {
            dbfDriver.initDriverFromFile(dbfFile, forceEncoding);
        } else {
            throw new IllegalArgumentException("DBF File not found");
        }
        if(shxFile==null){
            throw new IllegalArgumentException("SHX File not found");
        }
        FileInputStream shpFis = new FileInputStream(shpFile);
        shapefileReader = new ShapefileReader(shpFis.getChannel());
        FileInputStream shxFis = new FileInputStream(shxFile);
        shxFileReader = new IndexFile(shxFis.getChannel());
    }
1163891
public void api_application_applicationId_DELETE(Long applicationId) throws IOException {
		String qPath = "/me/api/application/{applicationId}";
		StringBuilder sb = path(qPath, applicationId);
		exec(qPath, "DELETE", sb.toString(), null);
	}
282358
public static void main(final String[] args) throws InjectionException, IOException, YarnException {
    final File jobSubmissionParametersFile = new File(args[0]);
    final File appSubmissionParametersFile = new File(args[1]);

    if (!(appSubmissionParametersFile.exists() && appSubmissionParametersFile.canRead())) {
      throw new IOException("Unable to open and read " + appSubmissionParametersFile.getAbsolutePath());
    }

    if (!(jobSubmissionParametersFile.exists() && jobSubmissionParametersFile.canRead())) {
      throw new IOException("Unable to open and read " + jobSubmissionParametersFile.getAbsolutePath());
    }

    final YarnClusterSubmissionFromCS yarnSubmission =
        YarnClusterSubmissionFromCS.fromJobSubmissionParametersFile(
            appSubmissionParametersFile, jobSubmissionParametersFile);

    LOG.log(Level.INFO, "YARN job submission received from C#: {0}", yarnSubmission);

    if (!yarnSubmission.getFileSystemUrl().equalsIgnoreCase(FileSystemUrl.DEFAULT_VALUE)) {
      LOG.log(Level.INFO, "getFileSystemUrl: {0}", yarnSubmission.getFileSystemUrl());
    } else {
      LOG.log(Level.INFO, "FileSystemUrl is not set, use default from the environment.");
    }

    final List<String> launchCommandPrefix = new ArrayList<String>() {{
        final REEFFileNames reefFileNames = TANG.newInjector().getInstance(REEFFileNames.class);
        add(reefFileNames.getDriverLauncherExeFile().getPath());
      }};

    final Configuration yarnJobSubmissionClientConfig = TANG.newConfigurationBuilder()
        .bindImplementation(RuntimeClasspathProvider.class, YarnClasspathProvider.class)
        .bindConstructor(org.apache.hadoop.yarn.conf.YarnConfiguration.class, YarnConfigurationConstructor.class)
        .bindNamedParameter(JobSubmissionDirectoryPrefix.class, yarnSubmission.getJobSubmissionDirectoryPrefix())
        .bindNamedParameter(FileSystemUrl.class, yarnSubmission.getFileSystemUrl())
        .bindList(DriverLaunchCommandPrefix.class, launchCommandPrefix)
        .build();

    final YarnJobSubmissionClient client =
        TANG.newInjector(yarnJobSubmissionClientConfig).getInstance(YarnJobSubmissionClient.class);

    client.launch(yarnSubmission);

    LOG.log(Level.INFO, "Returned from launch in Java YarnJobSubmissionClient");
    System.exit(0);
    LOG.log(Level.INFO, "End of main in Java YarnJobSubmissionClient");
  }
158654
private void cleanUpDriverResources() {
    ctrlc.setComplete();
    try {
      Runtime.getRuntime().removeShutdownHook(ctrlc);
    }
    catch (IllegalStateException ignore) {
      // If "Shutdown in progress" exception would be thrown, just ignore and don't bother to remove the hook.
    }
    ctrlc = null;

    try {
      setShutdownRequested();
      driverCallbackSocket.close();
      driverCallbackSocket = null;
    }
    catch (Exception e) {
      System.out.println("ERROR: " + (e.getMessage() != null ? e.getMessage() : "(null)"));
      e.printStackTrace();
    }

    // At this point, resources are released.
    // The hadoop job has completed (job.isComplete() is true),
    // so the cluster memory and cpus are freed.
    // The driverCallbackSocket has been closed so a new one can be made.

    // The callbackManager itself may or may not have finished, but it doesn't
    // matter since the server socket has been closed.
  }
610492
private void createPropertiesFolder() throws IOException {
        if(!Files.exists(propertiesLocation.toPath()))
            Files.createDirectories(propertiesLocation.toPath());
    }
325164
public void setApplicationId(String applicationId) {
		// Set the application id as a GeoPackage
		int applicationIdInt = ByteBuffer.wrap(applicationId.getBytes())
				.asIntBuffer().get();
		execSQL(String.format("PRAGMA application_id = %d;", applicationIdInt));
	}