抓取当前所有任务，会阻塞到爬取完成

@throws IOException 异常
332276
public int fetchAll(GeneratorFilter generatorFilter) throws Exception {
        if (executor == null) {
            LOG.info("Please Specify An Executor!");
            return 0;
        }

        dbManager.merge();

        try {
            dbManager.initSegmentWriter();
            LOG.info("init segmentWriter:" + dbManager.getClass().getName());
            running = true;
            lastRequestStart = new AtomicLong(System.currentTimeMillis());

            activeThreads = new AtomicInteger(0);
            startedThreads = new AtomicInteger(0);
            spinWaiting = new AtomicInteger(0);
            fetchQueue = new FetchQueue();
            feeder = new QueueFeeder(fetchQueue, dbManager, generatorFilter, 1000);
            feeder.start();

            FetcherThread[] fetcherThreads = new FetcherThread[threads];
            for (int i = 0; i < threads; i++) {
                fetcherThreads[i] = new FetcherThread();
                fetcherThreads[i].start();
            }

            do {
                try {
                    Thread.sleep(1000);
                } catch (InterruptedException ex) {
                }
                LOG.info("-activeThreads=" + activeThreads.get()
                        + ", spinWaiting=" + spinWaiting.get() + ", fetchQueue.size="
                        + fetchQueue.getSize());

                if (!feeder.isAlive() && fetchQueue.getSize() < 5) {
                    fetchQueue.dump();
                }

                if ((System.currentTimeMillis() - lastRequestStart.get()) > getConf().getThreadKiller()) {
                    LOG.info("Aborting with " + activeThreads + " hung threads.");
                    break;
                }

            } while (running && (startedThreads.get() != threads || activeThreads.get() > 0));
            running = false;
            long waitThreadEndStartTime = System.currentTimeMillis();
            if (activeThreads.get() > 0) {
                LOG.info("wait for activeThreads to end");
            }
            /*等待存活线程结束*/
            while (activeThreads.get() > 0) {
                LOG.info("-activeThreads=" + activeThreads.get());
                try {
                    Thread.sleep(500);
                } catch (Exception ex) {
                }
                if (System.currentTimeMillis() - waitThreadEndStartTime > getConf().getWaitThreadEndTime()) {
                    LOG.info("kill threads");
                    for (int i = 0; i < fetcherThreads.length; i++) {
                        if (fetcherThreads[i].isAlive()) {
                            try {
                                fetcherThreads[i].stop();
                                LOG.info("kill thread " + i);
                            } catch (Exception ex) {
                                LOG.info("Exception", ex);
                            }
                        }
                    }
                    break;
                }
            }
            LOG.info("clear all activeThread");
            feeder.stopFeeder();
            fetchQueue.clear();
        } finally {
            if(feeder!=null) {
                feeder.closeGenerator();
            }
            dbManager.closeSegmentWriter();
            LOG.info("close segmentWriter:" + dbManager.getClass().getName());
        }
        return feeder.generator.getTotalGenerate();
    }
46885
private void init() throws IOException {
    WR = new Path(conf.get("fs.snapshot.waitingroom", "/.WR"));
    WR = WR.makeQualified(dfs);
  }
1163651
public static void main(String[] args) throws IOException {
    if (args.length == 0){
      usage();
    }
    int ai=0;
    try {
      String encoding = null;
      String dict = null;
      while (args[ai].startsWith("-")) {
        if (args[ai].equals("-encoding")) {
          ai++;
          if (ai < args.length) {
            encoding = args[ai++];
          }
          else {
            usage();
          }
        }
        else if (args[ai].equals("-dict")) {
          ai++;
          if (ai < args.length) {
            dict = args[ai++];
          }
          else {
            usage();
          }
        }
        else {
          System.err.println("Unknown option "+args[ai]);
          usage();
        }
      }
      File inFile = new File(args[ai++]);
      File outFile = new File(args[ai++]);
      int cutoff = 5;
      int iterations = 100;
      if (args.length > ai) {
        cutoff = Integer.parseInt(args[ai++]);
        iterations = Integer.parseInt(args[ai++]);
      }
      GISModel mod;
      if (dict != null) {
        System.err.println("Building dictionary");
        MutableDictionary mdict = new MutableDictionary(cutoff);
        DataStream data = new ch.epfl.bbp.shaded.opennlp.maxent.PlainTextByLineDataStream(new java.io.FileReader(inFile));
        while(data.hasNext()) {
          String tagStr = (String) data.nextToken();
          String[] tt = tagStr.split(" ");
          String[] words = new String[tt.length];
          for (int wi=0;wi<words.length;wi++) {
            words[wi] = tt[wi].substring(0,tt[wi].lastIndexOf('_'));
          }
          mdict.add(words,1,true);
        }
        System.out.println("Saving the dictionary");
        mdict.persist(new File(dict));
      }
      EventStream es;
      if (encoding == null) {
        if (dict == null) {
          es = new POSEventStream(new PlainTextByLineDataStream(new InputStreamReader(new FileInputStream(inFile))));
        }
        else {
          es = new POSEventStream(new PlainTextByLineDataStream(new InputStreamReader(new FileInputStream(inFile))), new Dictionary(dict));
        }
      }
      else {
        if (dict == null) {
          es = new POSEventStream(new PlainTextByLineDataStream(new InputStreamReader(new FileInputStream(inFile),encoding)));
        }
        else {
          es = new POSEventStream(new PlainTextByLineDataStream(new InputStreamReader(new FileInputStream(inFile),encoding)), new Dictionary(dict));
        }
      }
      mod = train(es, iterations, cutoff);
      System.out.println("Saving the model as: " + outFile);
      new SuffixSensitiveGISModelWriter(mod, outFile).persist();

    }
    catch (Exception e) {
      e.printStackTrace();
    }
  }
1153896
public void printAllValues() throws Exception {
    err("List of all the available keys:");

    Object val = null;

    for (ObjectName oname: hadoopObjectNames) {
      err(">>>>>>>>jmx name: " + oname.getCanonicalKeyPropertyListString());
      MBeanInfo mbinfo = mbsc.getMBeanInfo(oname);
      MBeanAttributeInfo [] mbinfos = mbinfo.getAttributes();

      for (MBeanAttributeInfo mb: mbinfos) {
        val = mbsc.getAttribute(oname, mb.getName());
        System.out.format(format,mb.getName(),val.toString());
      }
    }
  }
747609
@SuppressWarnings("unchecked")
	private void createConnection() throws ThriftConnectionPoolException {
		try {
			transport = new TSocket(host, port, connectionTimeOut);
			transport.open();
			TProtocol protocol = createTProtocol(transport);
			// 反射实例化客户端对象
			Constructor<? extends TServiceClient> clientConstructor = clientClass.getConstructor(TProtocol.class);
			client = (T) clientConstructor.newInstance(protocol);
			if (logger.isDebugEnabled()) {
				logger.debug("创建新连接成功:" + host + " 端口：" + port);
			}
		} catch (Exception e) {
			throw new ThriftConnectionPoolException("无法连接服务器：" + host + " 端口：" + port);
		}
	}
216886
@Override
  public void print() throws IOException {
    results.sort(comparator());
    String report = assemble(results);
    String output = settings.report().output();
    if (output.equalsIgnoreCase("console")) {
      System.out.println(report);
    } else {
      Files.write(Paths.get(output), report.getBytes(UTF_8));
    }
  }
62225
public synchronized void finish() throws IOException
    {
        synchronized(out)
        {
            for(ByteArrayOutputStream baos : all_buffers)
            {
                baos.writeTo(out);
                baos.reset();
            }
            out.flush();
        }
        
    }
751098
private void run() throws IOException
	{
		PrintWriter pw = new PrintWriter(new FileWriter(outputFile));
		LineNumberReader lr = new LineNumberReader(new FileReader(inputFile));
		String line = null;
		int j  = 0;
		//String t = null;
		while ((line = lr.readLine()) != null)
		{
			logger.info(j + " " + line);
			pw.print("1");
			pw.print("\t");
			pw.print(++j);
			pw.print(".tease\t");
			String[] s = line.split(" ");
			for (int i=0;i<s.length - 1;i++)
			{

				String[] t = s[i].split("/");
				if (i != 0)
					pw.print(" ");

				pw.print(i);
				pw.print("&&");
				pw.print(t[0]); // form
				pw.print("&&");
				pw.print(t[1]); // lemma
				pw.print("&&");
				pw.print(t[2]); // pos
				pw.print("&&");
				pw.print(t[3]); // NE  | O
				pw.print("&&");
				pw.print(t[4]); // target | O

			}
			pw.print("\n");
		} // end while
		lr.close();
		pw.flush();
		pw.close();
	}
926579
public void make()
    throws Exception
  {
    for (int i = 0; i < _makeList.size(); i++) {
      Make make = _makeList.get(i);

      make.make();
    }
  }
282443
public boolean execute() throws IOException {
    if (stopOnError) {
      for (int i = 0; i < actions.length; i++) {
        if (!actions[i].execute()) {
          return false;
        }
      }

      return true;
    } else {
      boolean status = true;
      IOException exception = null;

      for (int i = 0; i < actions.length; i++) {
        try {
          status &= actions[i].execute();
        } catch (IOException ex) {
          status = false;

          if (exception == null) {
            exception = ex;
          }
        }
      }

      if (exception != null) {
        throw exception;
      }

      return status;
    }
  }
80994
public static SqlSession getSession() throws IOException {
		SqlSession session = (SqlSession) THREAD_SESSION.get();  
	    try {  
	        if (session == null) {  
	        	session = getSessionFactory().getSession();
	        	THREAD_SESSION.set(session);  
	        }  
	    } catch (Exception e) {  
	        throw new IOException(e);  
	    }  
	    return session;  
	}
719943
public void write()
        throws IOException
    {
        BufferedWriter writer = new BufferedWriter( new OutputStreamWriter( m_outputStream ) );
        for( String line : m_content )
        {
            writer.write( line );
            writer.newLine();
        }
        writer.flush();
        writer.close();
    }
1056196
@Override
  void write(ByteCodeWriter out)
    throws IOException
  {
    out.write(ConstantPool.CP_METHODHANDLE);
    out.write(_type.getCode());
    out.writeShort(_entry.getIndex());
  }
96221
@Override
  public void run() throws IOException {
    String outputDirectoryName = experiment.getReferenceFrontDirectory() ;
    createOutputDirectory(outputDirectoryName) ;

    List<String> referenceFrontFileNames = new LinkedList<>() ;

    for (ExperimentProblem<?> problem : experiment.getProblemList()) {
      List<DoubleSolution> nonDominatedSolutions = getNonDominatedSolutions(problem) ;

      referenceFrontFileNames.add(problem.getReferenceFront());

      writeReferenceFrontFile(outputDirectoryName, problem, nonDominatedSolutions) ;
      writeReferenceSetFile(outputDirectoryName, problem, nonDominatedSolutions) ;

      writeFilesWithTheSolutionsContributedByEachAlgorithm(outputDirectoryName, problem, nonDominatedSolutions) ;
    }

  }
223295
@Override
  public synchronized Map<String, LocalResource> call() throws IOException {
    final Map<String, LocalResource> globalResources = new HashMap<>(1);
    if (!this.isUploaded){
      this.pathToGlobalJar = this.uploader.uploadToJobFolder(makeGlobalJar());
      this.isUploaded = true;
    }

    final LocalResource updatedGlobalJarResource = this.uploader.makeLocalResourceForJarFile(this.pathToGlobalJar);

    if (this.globalJarResource != null
        && this.globalJarResource.getTimestamp() != updatedGlobalJarResource.getTimestamp()) {
      LOG.log(Level.WARNING,
              "The global JAR LocalResource timestamp has been changed from "
              + this.globalJarResource.getTimestamp() + " to " + updatedGlobalJarResource.getTimestamp());
    }

    this.globalJarResource = updatedGlobalJarResource;

    // For now, always rewrite the information due to REEF-348
    globalResources.put(this.fileNames.getGlobalFolderPath(), updatedGlobalJarResource);

    return globalResources;
  }
517849
public void println() throws IOException
    {
        if(this._listener!= null &&  !checkIfCalledFromWLonError()){
            
            _outHelper.write_NonBlocking(CRLF, 0, 2);
        }
        else {
            this.write(CRLF, 0, 2);
        }

    }
683230
public List<String> process() throws IOException {
		MissingRecordAwareHSSFListener listener = new MissingRecordAwareHSSFListener(this);
		formatListener = new FormatTrackingHSSFListener(listener);

		HSSFEventFactory factory = new HSSFEventFactory();
		HSSFRequest request = new HSSFRequest();

		if(outputFormulaValues) {
			request.addListenerForAllRecords(formatListener);
		} else {
			workbookBuildingListener = new SheetRecordCollectingListener(formatListener);
			request.addListenerForAllRecords(workbookBuildingListener);
		}

		factory.processWorkbookEvents(request, fs);
		
		return results;
	}