Sets up and starts streaming pipeline.

@throws IOException if there is a problem setting up resources
338358
public static void main(String[] args) throws IOException {
    StreamingWordExtractOptions options = PipelineOptionsFactory.fromArgs(args)
        .withValidation()
        .as(StreamingWordExtractOptions.class);
    options.setStreaming(true);

    options.setBigQuerySchema(StringToRowConverter.getSchema());
    ExampleUtils exampleUtils = new ExampleUtils(options);
    exampleUtils.setup();

    Pipeline pipeline = Pipeline.create(options);

    String tableSpec = new StringBuilder()
        .append(options.getProject()).append(":")
        .append(options.getBigQueryDataset()).append(".")
        .append(options.getBigQueryTable())
        .toString();
    pipeline
        .apply("ReadLines", TextIO.read().from(options.getInputFile()))
        .apply(ParDo.of(new ExtractWords()))
        .apply(ParDo.of(new Uppercase()))
        .apply(ParDo.of(new StringToRowConverter()))
        .apply(BigQueryIO.writeTableRows().to(tableSpec)
            .withSchema(StringToRowConverter.getSchema()));

    PipelineResult result = pipeline.run();

    // ExampleUtils will try to cancel the pipeline before the program exists.
    exampleUtils.waitToFinish(result);
  }
1096389
public void init()
	{
		pipeline = pipeline();
		
		pipeline.addLast("messageBufferEventDecoder", messageBufferEventDecoder);
		pipeline.addLast("upstream", upstream);
		
		// Downstream handlers - Filter for data which flows from server to
		// client. Note that the last handler added is actually the first
		// handler for outgoing data.
		pipeline.addLast("messageBufferEventEncoder",messageBufferEventEncoder);
	}
1146207
public void startRTS() throws IOException {
    List<HubDataSource> dataSources = hubDataStore.getHubDataSources();
    List<String> com = Lists.newArrayList();
    com.add(getHostPort(hubDataStore.getHubAddress()));
    com.add(hubDataStore.getInstanceName());
    for (HubDataSource source : dataSources) {
      com.add(source.getName());
    }
    ExternalProgramExecutor executorService = new ExternalProgramExecutor(
      "RTS", new File(binLocation, "rts"), com.toArray(new String[com.size()]));
    rtsExecutor.add(executorService);
    LOG.info("Starting RTS : {}", executorService);
    executorService.startAndWait();
  }
702388
public final void run() throws IOException, InterruptedException {
        InputStreamReader isr = new InputStreamReader(this.is, this.charset);
        BufferedReader br = new BufferedReader(isr);
        this.line = null;
        this.lineNumber = 0;
        try {
            do {
                if (this.exit) {
                    return;
                }
                if(Thread.currentThread().isInterrupted()){
                throw new InterruptedException();
            }
                this.nextLine = br.readLine();
                try {
                    if (this.line != null) {
                        processLine(this.line);
                        if (this.exit) {
                            return;
                        }
                    }
                } catch (Exception e) {
                    onExceptionFound(e);
                } finally {
                    this.lineNumber++;
                    this.line = this.nextLine;
                }
            } while (this.line != null);
        } finally {
            onFinish();
        }
    }
111888
public void start() throws IOException {
    lock.lock();
    try {
      if (process != null) {
        return;
      }
      process = new CommandLine(this.executable, args.toArray(new String[] {}));
      process.setEnvironmentVariables(environment);
      process.copyOutputTo(getOutputStream());
      process.executeAsync();

      waitUntilAvailable();
    } finally {
      lock.unlock();
    }
  }
1327112
private boolean setup() throws IOException {
    estimateSavings();

    final String randomId = getRandomId();
    JobClient jClient = new JobClient(jobconf);
    Path jobdir = new Path(jClient.getSystemDir(), NAME + "_" + randomId);

    LOG.info(JOB_DIR_LABEL + "=" + jobdir);
    jobconf.set(JOB_DIR_LABEL, jobdir.toString());
    Path log = new Path(jobdir, "_logs");

    // The control file should have small size blocks. This helps
    // in spreading out the load from mappers that will be spawned.
    jobconf.setInt("dfs.blocks.size",  OP_LIST_BLOCK_SIZE);

    FileOutputFormat.setOutputPath(jobconf, log);
    LOG.info("log=" + log);

    // create operation list
    FileSystem fs = jobdir.getFileSystem(jobconf);
    Path opList = new Path(jobdir, "_" + OP_LIST_LABEL);
    jobconf.set(OP_LIST_LABEL, opList.toString());
    int opCount = 0, synCount = 0;
    SequenceFile.Writer opWriter = null;

    try {
      opWriter = SequenceFile.createWriter(fs, jobconf, opList, Text.class,
          PolicyInfo.class, SequenceFile.CompressionType.NONE);
      for (RaidPolicyPathPair p : raidPolicyPathPairList) {
        // If a large set of files are Raided for the first time, files
        // in the same directory that tend to have the same size will end up
        // with the same map. This shuffle mixes things up, allowing a better
        // mix of files.
        java.util.Collections.shuffle(p.srcPaths);
        for (EncodingCandidate ec : p.srcPaths) {
          opWriter.append(new Text(ec.toString()), p.policy);
          opCount++;
          if (++synCount > SYNC_FILE_MAX) {
            opWriter.sync();
            synCount = 0;
          }
        }
      }

    } finally {
      if (opWriter != null) {
        opWriter.close();
      }
      fs.setReplication(opList, OP_LIST_REPLICATION); // increase replication for control file
    }
    raidPolicyPathPairList.clear();
    
    jobconf.setInt(OP_COUNT_LABEL, opCount);
    LOG.info("Number of files=" + opCount);
    jobconf.setNumMapTasks(getMapCount(opCount));
    LOG.info("jobName= " + jobName + " numMapTasks=" + jobconf.getNumMapTasks());
    return opCount != 0;

  }
1117235
public void startHFTA() throws IOException {
    int hftaCount = MetaInformationParser.getHFTACount(new File(this.hubDataStore.getBinaryLocation().toURI()));
    for (int i = 0; i < hftaCount; i++) {
      List<String> com = Lists.newArrayList();
      com.add(getHostPort(hubDataStore.getHubAddress()));
      com.add(hubDataStore.getInstanceName());
      ExternalProgramExecutor executorService = new ExternalProgramExecutor(
        "HFTA-" + i, new File(binLocation, "hfta_" + i), com.toArray(new String[com.size()]));
      hftaExecutor.add(executorService);
      LOG.info("Starting HFTA : {}", executorService);
      executorService.startAndWait();
    }
  }
1184148
private void setProcessInputStream(OutputStream os) {
        if (input != null) {
            inputThread = createPump(input, os, true);
        } else {
            AntFileUtils.close(os);
        }
    }
58468
private void startRegular() throws IOException {
    String pwd = DeepwaterCaffeBackend.CAFFE_H2O_DIR;
    ProcessBuilder pb = new ProcessBuilder("python3 backend.py".split(" "));
    pb.environment().put("PYTHONPATH", DeepwaterCaffeBackend.CAFFE_DIR + "python");
    pb.redirectError(ProcessBuilder.Redirect.INHERIT);
    pb.directory(new File(pwd));
    _process = pb.start();
  }
657542
private void handleDrainedSource() throws IOException {
         if (!inCatalog() && m_committedBuffers.isEmpty()) {
             //Returning null indicates end of stream
             try {
                 if (m_pollTask != null) {
                     m_pollTask.setFuture(null);
                 }
             } catch (RejectedExecutionException reex) {
                 // Ignore, {@code GuestProcessor} was closed
             }
             m_pollTask = null;
             m_generation.onSourceDrained(m_partitionId, m_tableName);
             return;
         }
     }
30243
private void startTwitterStream() {
        logger.info("starting {} twitter stream", streamType);

        if (stream == null) {
            logger.debug("creating twitter stream");

            stream = new TwitterStreamFactory(buildTwitterConfiguration()).getInstance();
            if (streamType.equals("user")) {
                stream.addListener(new UserStreamHandler());
            } else {
                stream.addListener(new StatusHandler());
            }

            logger.debug("twitter stream created");
        }

        if (riverStatus != RiverStatus.STOPPED && riverStatus != RiverStatus.STOPPING) {
            if (streamType.equals("filter") || filterQuery != null) {
                stream.filter(filterQuery);
            } else if (streamType.equals("firehose")) {
                stream.firehose(0);
            } else if (streamType.equals("user")) {
                stream.user();
            } else {
                stream.sample();
            }
        }
        logger.debug("{} twitter stream started!", streamType);
    }
503773
void start() {
    for (int i = 0; i < threadCount; i++) {
      Runnable r = null;
      switch (config.getPayloadConfig().getPayloadCase()) {
        case SIMPLE_PARAMS: {
          if (config.getClientType() == Control.ClientType.ASYNC_CLIENT) {
            if (config.getRpcType() == Control.RpcType.UNARY) {
              r = new AsyncUnaryWorker(clients[i % clients.length]);
            } else if (config.getRpcType() == Control.RpcType.STREAMING) {
              r = new AsyncPingPongWorker(mat, clients[i % clients.length]);
            }
          }
          break;
        }
        default: {
          throw Status.UNIMPLEMENTED.withDescription(
              "Unknown payload case " + config.getPayloadConfig().getPayloadCase().name())
              .asRuntimeException();
        }
      }
      if (r == null) {
        throw new IllegalStateException(config.getRpcType().name()
            + " not supported for client type "
            + config.getClientType());
      }
      fixedThreadPool.execute(r);
    }
    if (osBean != null) {
      lastMarkCpuTime = osBean.getProcessCpuTime();
    }
  }
1343489
private void initStreams() throws IOException {
		if (audio != null) {
			audio.close();
		}
		
		OggInputStream audio;
		
		if (url != null) {
			audio = new OggInputStream(url.openStream());
		} else {
			audio = new OggInputStream(ResourceLoader.getResourceAsStream(ref));
		}
		
		this.audio = audio;
		positionOffset = 0;
	}
949072
@GwtIncompatible
    public static void start() {
        computation().start();
        io().start();
        newThread().start();
        single().start();
        trampoline().start();
        SchedulerPoolFactory.start();
    }
1017367
public final synchronized void start() throws IOException {
		if (this.acceptor != null) {
			if (this.acceptor.getHandler() == null) {
				this.acceptor.setHandler(new TcpIpHandlerAdapter());
			}
			
			LaunchingMessageKind.ITCPIP0001.format(this.host, this.port);
			this.acceptor.bind(new InetSocketAddress(this.host, this.port));
		}
	}
649348
private void refreshInputStream() throws IOException {
    // if stream is null, we probably switched to a new
    // segment.
    if (currentEditLogInputStream == null) {
      LOG.info("Segment - setup input stream for txid: " + currentSegmentTxId);
      setupIngestStreamWithRetries(currentSegmentTxId);
      if (currentEditLogInputStreamPosition == -1) {
        // we are opening a fresh segment
        currentEditLogInputStreamPosition = currentEditLogInputStream.getPosition();
      }
    }

    // if we are re-opening stream previously consumed
    // set correct position
    if (currentEditLogInputStreamPosition != -1) {
      currentEditLogInputStream.refresh(currentEditLogInputStreamPosition,
          mostRecentlyReadTransactionTxId);
    }
  }
308020
@Setup
  public void setup() throws IOException {
    buffer = new Buffer();
    while (buffer.size() < bufferSize) {
      buffer.write(new byte[8192]);
    }
  }