Create ASP dem_mosaic command 
    Useful for spawning many single-threaded mosaicing processes
325459
def get_dem_mosaic_cmd(fn_list, o, fn_list_txt=None, tr=None, t_srs=None, t_projwin=None, georef_tile_size=None, threads=None, tile=None, stat=None):
    
    cmd = ['dem_mosaic',]
    if o is None:
        o = 'mos'
    cmd.extend(['-o', o])
    if threads is None:
        from pygeotools.lib import iolib
        threads = iolib.cpu_count()
    cmd.extend(['--threads', threads])
    if tr is not None:
        cmd.extend(['--tr', tr])
    if t_srs is not None:
        #cmd.extend(['--t_srs', t_srs.ExportToProj4()])
        cmd.extend(['--t_srs', '"%s"' % t_srs.ExportToProj4()])
        #cmd.extend(['--t_srs', "%s" % t_srs.ExportToProj4()])
    if t_projwin is not None:
        cmd.append('--t_projwin')
        cmd.extend(t_projwin)
        cmd.append('--force-projwin')
    if tile is not None:
        #Not yet implemented
        #cmd.extend(tile_list)
        cmd.append('--tile-index')
        cmd.append(tile)
    if georef_tile_size is not None:
        cmd.extend(['--georef-tile-size', georef_tile_size])
    if stat is not None:
        if stat == 'wmean':
            stat = None
        else:
            cmd.append('--%s' % stat.replace('index',''))
            if stat in ['lastindex', 'firstindex', 'medianindex']:
                #This will write out the index map to -last.tif by default
                cmd.append('--save-index-map')
                #Make sure we don't have ndv that conflicts with 0-based DEM indices
                cmd.extend(['--output-nodata-value','-9999'])
    #else:
    #    cmd.extend(['--save-dem-weight', o+'_weight'])
    #If user provided a file containing list of DEMs to mosaic (useful to avoid long bash command issues)
    if fn_list_txt is not None:
        if os.path.exists(fn_list_txt):
            cmd.append('-l')
            cmd.append(fn_list_txt)
        else: 
            print("Could not find input text file containing list of inputs")
    else:
        cmd.extend(fn_list)
    cmd = [str(i) for i in cmd]
    #print(cmd)
    #return subprocess.call(cmd)
    return cmd
1313210
function spawnProcess(opts, cmdOptions, childProcs, watchedFiles) {
  var color = colorSet[Object.keys(childProcs).length % colorSet.length];
  var cmdTokens = opts.command.split(' ');
  var childProcess = spawn(_.head(cmdTokens),
			   _.tail(cmdTokens),
			   {
			     cwd: parser.cmdWorkingDir(cmdOptions),
			     env: opts.env
			   });

  childProcess.stdout.on('data', function(data) {
    console.log(('[' + opts.name + '] ' + data)[color]);
  });

  childProcess.stderr.on('data', function(data) {
    console.log(('[' + opts.name + '] ' + data)[color]);
  });

  childProcess.on('exit', function(code) {
    console.log(('[' + opts.name + '] exited with code ' + code)[color]);
    removePIDFile(opts.name);
  });

  childProcess.command = opts;
  writePIDFile(childProcess.pid, opts.name);

  return childProcess;
}
1072481
def every(colors, output_dir=CACHE_DIR):
    """"""
    colors = flatten_colors(colors)
    template_dir = os.path.join(MODULE_DIR, "templates")
    template_dir_user = os.path.join(CONF_DIR, "templates")
    util.create_dir(template_dir_user)

    join = os.path.join  # Minor optimization.
    for file in [*os.scandir(template_dir),
                 *os.scandir(template_dir_user)]:
        if file.name != ".DS_Store" and not file.name.endswith(".swp"):
            template(colors, file.path, join(output_dir, file.name))

    logging.info("Exported all files.")
    logging.info("Exported all user files.")
658702
def imagemagick(color_count, img, magick_command):
    """"""
    flags = ["-resize", "25%", "-colors", str(color_count),
             "-unique-colors", "txt:-"]
    img += "[0]"

    return subprocess.check_output([*magick_command, img, *flags]).splitlines()
671271
def paste_mashes(sketches, pasted_mash, force = False):
    
    if os.path.isfile(pasted_mash):
        if force:
            subprocess.Popen(['rm', pasted_mash]).wait()
        else:
            return
    pasted_mash = pasted_mash.rsplit('.msh')[0]
    mash_cmd = ['/opt/bin/bio/mash', 'paste', pasted_mash]
    mash_cmd.extend(sketches)
    process = subprocess.Popen(mash_cmd)
    process.wait()
    return
181426
def _run_mpi_cmd(self, cmd):
        
        log("Number of Processes: {0}".format(self.num_processors))
        time_start = datetime.utcnow()

        # Construct the taudem command line.
        cmd = [self.mpiexec_path, '-n', str(self.num_processors)] + cmd
        log("Command Line: {0}".format(" ".join(cmd)))
        process = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=False)
        out, err = process.communicate()
        if out:
            log("OUTPUT:")
            for line in out.split(b'\n'):
                log(line)
        if err:
            log(err, severity="WARNING")
        log("Time to complete: {0}".format(datetime.utcnow()-time_start))
1227286
public synchronized void batchProcessComplete() {
    if (UIMAFramework.getLogger().isLoggable(Level.FINEST)) {
      UIMAFramework.getLogger(this.getClass()).logrb(Level.FINEST, this.getClass().getName(),
              "process", CPMUtils.CPM_LOG_RESOURCE_BUNDLE, "UIMA_CPM_method_ping__FINEST",
              new Object[] { Thread.currentThread().getName() });
    }
  }
906535
def build(
        documentPath,
        outputUFOFormatVersion=3,
        roundGeometry=True,
        verbose=True,           # not supported
        logPath=None,           # not supported
        progressFunc=None,      # not supported
        processRules=True,
        logger=None,
        useVarlib=False,
        ):
    
    import os, glob
    if os.path.isdir(documentPath):
        # process all *.designspace documents in this folder
        todo = glob.glob(os.path.join(documentPath, "*.designspace"))
    else:
        # process the
        todo = [documentPath]
    results = []
    for path in todo:
        document = DesignSpaceProcessor(ufoVersion=outputUFOFormatVersion)
        document.useVarlib = useVarlib
        document.roundGeometry = roundGeometry
        document.read(path)
        try:
            r = document.generateUFO(processRules=processRules)
            results.append(r)
        except:
            if logger:
                logger.exception("ufoProcessor error")
        #results += document.generateUFO(processRules=processRules)
        reader = None
    return results
1037945
def _step_template(name, run_file, inputs, outputs, parallel, step_parallelism, scatter=None):
    
    scatter_inputs = []
    sinputs = []
    for inp in inputs:
        step_inp = {"id": workflow.get_base_id(inp["id"]), "source": inp["id"]}
        if inp.get("wf_duplicate"):
            step_inp["id"] += "_toolinput"
        for attr in ["source", "valueFrom"]:
            if attr in inp:
                step_inp[attr] = inp[attr]
        sinputs.append(step_inp)
        # An initial parallel scatter and multiple chained parallel sample scatters
        if (parallel == "multi-parallel" and
              (not step_parallelism or
               step_parallelism.get(workflow.get_step_prefix(inp["id"])) == "multi-parallel")):
            scatter_inputs.append(step_inp["id"])
        # scatter on inputs from previous processes that have been arrayed
        elif (_is_scatter_parallel(parallel) and (_do_scatter_var(inp, parallel)
                                                  or (scatter and inp["id"] in scatter))):
            scatter_inputs.append(step_inp["id"])
    out = {"run": run_file,
           "id": name,
           "in": sinputs,
           "out": [{"id": workflow.get_base_id(output["id"])} for output in outputs]}
    if _is_scatter_parallel(parallel):
        assert scatter_inputs, "Did not find items to scatter on: %s" % name
        out.update({"scatterMethod": "dotproduct",
                    "scatter": scatter_inputs})
    return out
1185846
def create(client, mosaic=False, **kw):
    '''
    '''
    if mosaic:
        return _MosaicDownloader(client, **kw)
    else:
        return _Downloader(client, **kw)
819161
def apply_postcompilers(root, src_list, dst, processors):
    
    dst_file = os.path.join(root, dst)

    matches = [(pattern, cmds) for pattern, cmds in processors.iteritems() if fnmatch(dst, pattern)]
    if not matches:
        ensure_dirs(dst_file)
        logger.info('Combining [%s] into [%s]', ' '.join(src_list), dst_file)
        # We should just concatenate the files
        with open(dst_file, 'w') as dst_fp:
            for src in src_list:
                with open(os.path.join(root, src)) as src_fp:
                    for chunk in src_fp:
                        dst_fp.write(chunk)
        return True

    params = get_format_params(dst)

    # TODO: probably doesnt play nice everywhere
    src_names = src_list
    for pattern, cmd_list in matches:
        for cmd in cmd_list:
            run_command(cmd, root=root, dst=dst, input=' '.join(src_names), params=params)
            src_names = [dst]

    return True
433940
def generate(env):
    """"""
    static_obj, shared_obj = SCons.Tool.createObjBuilders(env)

    # TODO(batch):  shouldn't reach in to cmdgen this way; necessary
    # for now to bypass the checks in Builder.DictCmdGenerator.__call__()
    # and allow .cc and .cpp to be compiled in the same command line.
    static_obj.cmdgen.source_ext_match = False
    shared_obj.cmdgen.source_ext_match = False

    for suffix in CSuffixes:
        static_obj.add_action(suffix, CAction)
        shared_obj.add_action(suffix, ShCAction)
        static_obj.add_emitter(suffix, static_object_emitter)
        shared_obj.add_emitter(suffix, shared_object_emitter)

    for suffix in CXXSuffixes:
        static_obj.add_action(suffix, CXXAction)
        shared_obj.add_action(suffix, ShCXXAction)
        static_obj.add_emitter(suffix, static_object_emitter)
        shared_obj.add_emitter(suffix, shared_object_emitter)

    env['CCPDBFLAGS'] = SCons.Util.CLVar(['${(PDB and "/Z7") or ""}'])
    env['CCPCHFLAGS'] = SCons.Util.CLVar(['${(PCH and "/Yu%s \\\"/Fp%s\\\""%(PCHSTOP or "",File(PCH))) or ""}'])
    env['_MSVC_OUTPUT_FLAG'] = msvc_output_flag
    env['_CCCOMCOM']  = '$CPPFLAGS $_CPPDEFFLAGS $_CPPINCFLAGS $CCPCHFLAGS $CCPDBFLAGS'
    env['CC']         = 'cl'
    env['CCFLAGS']    = SCons.Util.CLVar('/nologo')
    env['CFLAGS']     = SCons.Util.CLVar('')
    env['CCCOM']      = '${TEMPFILE("$CC $_MSVC_OUTPUT_FLAG /c $CHANGED_SOURCES $CFLAGS $CCFLAGS $_CCCOMCOM","$CCCOMSTR")}'
    env['SHCC']       = '$CC'
    env['SHCCFLAGS']  = SCons.Util.CLVar('$CCFLAGS')
    env['SHCFLAGS']   = SCons.Util.CLVar('$CFLAGS')
    env['SHCCCOM']    = '${TEMPFILE("$SHCC $_MSVC_OUTPUT_FLAG /c $CHANGED_SOURCES $SHCFLAGS $SHCCFLAGS $_CCCOMCOM","$SHCCCOMSTR")}'
    env['CXX']        = '$CC'
    env['CXXFLAGS']   = SCons.Util.CLVar('$( /TP $)')
    env['CXXCOM']     = '${TEMPFILE("$CXX $_MSVC_OUTPUT_FLAG /c $CHANGED_SOURCES $CXXFLAGS $CCFLAGS $_CCCOMCOM","$CXXCOMSTR")}'
    env['SHCXX']      = '$CXX'
    env['SHCXXFLAGS'] = SCons.Util.CLVar('$CXXFLAGS')
    env['SHCXXCOM']   = '${TEMPFILE("$SHCXX $_MSVC_OUTPUT_FLAG /c $CHANGED_SOURCES $SHCXXFLAGS $SHCCFLAGS $_CCCOMCOM","$SHCXXCOMSTR")}'
    env['CPPDEFPREFIX']  = '/D'
    env['CPPDEFSUFFIX']  = ''
    env['INCPREFIX']  = '/I'
    env['INCSUFFIX']  = ''
#    env.Append(OBJEMITTER = [static_object_emitter])
#    env.Append(SHOBJEMITTER = [shared_object_emitter])
    env['STATIC_AND_SHARED_OBJECTS_ARE_THE_SAME'] = 1

    env['RC'] = 'rc'
    env['RCFLAGS'] = SCons.Util.CLVar('')
    env['RCSUFFIXES']=['.rc','.rc2']
    env['RCCOM'] = '$RC $_CPPDEFFLAGS $_CPPINCFLAGS $RCFLAGS /fo$TARGET $SOURCES'
    env['BUILDERS']['RES'] = res_builder
    env['OBJPREFIX']      = ''
    env['OBJSUFFIX']      = '.obj'
    env['SHOBJPREFIX']    = '$OBJPREFIX'
    env['SHOBJSUFFIX']    = '$OBJSUFFIX'

    # Set-up ms tools paths
    msvc_setup_env_once(env)

    env['CFILESUFFIX'] = '.c'
    env['CXXFILESUFFIX'] = '.cc'

    msvc_set_PCHPDBFLAGS(env)


    env['PCHCOM'] = '$CXX /Fo${TARGETS[1]} $CXXFLAGS $CCFLAGS $CPPFLAGS $_CPPDEFFLAGS $_CPPINCFLAGS /c $SOURCES /Yc$PCHSTOP /Fp${TARGETS[0]} $CCPDBFLAGS $PCHPDBFLAGS'
    env['BUILDERS']['PCH'] = pch_builder

    if 'ENV' not in env:
        env['ENV'] = {}
    if 'SystemRoot' not in env['ENV']:    # required for dlls in the winsxs folders
        env['ENV']['SystemRoot'] = SCons.Platform.win32.get_system_root()
1014654
def display_workflows(prefix, **kwargs):
    
    from IPython.display import display, Markdown

    def create_svg(name, workflow):
        """Create an SVG file with rendered graph from a workflow."""
        name = name.replace('_', '-')
        filename = '{}-{}.svg'.format(prefix, name)
        dot = get_workflow_graph(workflow)
        dot.attr('graph', bgcolor='transparent')
        with open(filename, 'bw') as file:
            file.write(dot.pipe(format='svg'))
        return name, filename

    svg_files = {name: svg for name, svg in
                 (create_svg(name, workflow)
                  for name, workflow in kwargs.items())}

    markdown_table = \
        '| ' + ' | '.join(svg_files.keys()) + ' |\n' + \
        '| ' + ' | '.join('---' for _ in svg_files) + ' |\n' + \
        '| ' + ' | '.join('![workflow {}]({})'.format(name, svg)
                          for name, svg in svg_files.items()) + ' |'
    display(Markdown(markdown_table))
854189
def cli(patterns, times, json, csv, rst, md, ref, unit, precision, debug):
    ''''''
    if ref:
        ref = JSON.load(ref)

    filenames = []
    reporters = [CliReporter(ref=ref, debug=debug, unit=unit, precision=precision)]
    kwargs = {}
    for pattern in patterns or ['**/*.bench.py']:
        filenames.extend(resolve_pattern(pattern))
    if json:
        reporters.append(JsonReporter(json, precision=precision))
    if csv:
        reporters.append(CsvReporter(csv, precision=precision))
    if rst:
        reporters.append(RstReporter(rst, precision=precision))
    if md:
        reporters.append(MarkdownReporter(md, precision=precision))
    if times:
        kwargs['times'] = times
    runner = BenchmarkRunner(*filenames, reporters=reporters, debug=debug)
    runner.run(**kwargs)
1360596
def generate(env):
    """"""
    static_obj, shared_obj = SCons.Tool.createObjBuilders(env)

    for suffix in ASSuffixes:
        static_obj.add_action(suffix, SCons.Defaults.ASAction)
        shared_obj.add_action(suffix, SCons.Defaults.ASAction)
        static_obj.add_emitter(suffix, SCons.Defaults.StaticObjectEmitter)
        shared_obj.add_emitter(suffix, SCons.Defaults.SharedObjectEmitter)

    for suffix in ASPPSuffixes:
        static_obj.add_action(suffix, SCons.Defaults.ASPPAction)
        shared_obj.add_action(suffix, SCons.Defaults.ASPPAction)
        static_obj.add_emitter(suffix, SCons.Defaults.StaticObjectEmitter)
        shared_obj.add_emitter(suffix, SCons.Defaults.SharedObjectEmitter)

    env['AS']        = 'ml'
    env['ASFLAGS']   = SCons.Util.CLVar('/nologo')
    env['ASPPFLAGS'] = '$ASFLAGS'
    env['ASCOM']     = '$AS $ASFLAGS /c /Fo$TARGET $SOURCES'
    env['ASPPCOM']   = '$CC $ASPPFLAGS $CPPFLAGS $_CPPDEFFLAGS $_CPPINCFLAGS /c /Fo$TARGET $SOURCES'
    env['STATIC_AND_SHARED_OBJECTS_ARE_THE_SAME'] = 1
161492
def create_run_config(model_name,
                      master="",
                      model_dir=None,
                      iterations_per_loop=1000,
                      num_shards=8,
                      log_device_placement=False,
                      save_checkpoints_steps=1000,
                      save_checkpoints_secs=None,
                      keep_checkpoint_max=20,
                      keep_checkpoint_every_n_hours=10000,
                      num_gpus=1,
                      gpu_order="",
                      num_async_replicas=1,
                      enable_graph_rewriter=False,
                      gpu_mem_fraction=0.95,
                      no_data_parallelism=False,
                      optionally_use_dist_strat=False,
                      daisy_chain_variables=True,
                      schedule="continuous_train_and_eval",
                      worker_job="/job:localhost",
                      worker_id=0,
                      ps_replicas=0,
                      ps_job="/job:ps",
                      ps_gpu=0,
                      random_seed=None,
                      sync=False,
                      tpu_infeed_sleep_secs=None,
                      use_tpu=False,
                      use_tpu_estimator=False,
                      xla_jit_level=tf.OptimizerOptions.OFF,
                      inter_op_parallelism_threads=0,
                      log_step_count_steps=100,
                      intra_op_parallelism_threads=0,
                      tpu_config_extra_kwargs=None,
                      cloud_tpu_name=""):
  """"""
  session_config = create_session_config(
      log_device_placement=log_device_placement,
      enable_graph_rewriter=enable_graph_rewriter,
      gpu_mem_fraction=gpu_mem_fraction,
      use_tpu=use_tpu,
      xla_jit_level=xla_jit_level,
      inter_op_parallelism_threads=inter_op_parallelism_threads,
      intra_op_parallelism_threads=intra_op_parallelism_threads)
  run_config_args = {
      "master": master,
      "evaluation_master": master,
      "model_dir": model_dir,
      "session_config": session_config,
      "save_summary_steps": 100,
      "save_checkpoints_steps": save_checkpoints_steps,
      "save_checkpoints_secs": save_checkpoints_secs,
      "keep_checkpoint_max": keep_checkpoint_max,
      "keep_checkpoint_every_n_hours": keep_checkpoint_every_n_hours,
      "tf_random_seed": random_seed,
      "log_step_count_steps": log_step_count_steps
  }
  if save_checkpoints_secs:
    del run_config_args["save_checkpoints_steps"]
  run_config_cls = tf.contrib.learn.RunConfig

  if use_tpu or use_tpu_estimator:
    # If using TPUEstimator, use TPU RunConfig, add TPUConfig, and add
    # additional args.
    tpu_config_kwargs = {
        "iterations_per_loop": iterations_per_loop,
        "num_shards": num_shards,
        "per_host_input_for_training": True,
        "initial_infeed_sleep_secs": tpu_infeed_sleep_secs,
    }
    if tpu_config_extra_kwargs is not None:
      tpu_config_kwargs.update(tpu_config_extra_kwargs)
    run_config_cls = tf.contrib.tpu.RunConfig
    tpu_config = tf.contrib.tpu.TPUConfig(
        **tpu_config_kwargs)
    run_config_args["tpu_config"] = tpu_config
    if not master and "KUBE_GOOGLE_CLOUD_TPU_ENDPOINTS" in os.environ:
      # If running on TPU but no master is set and the KUBE env var is present
      # then we're running on ML Engine. Set the master.
      run_config_args["master"] = os.environ[
          "KUBE_GOOGLE_CLOUD_TPU_ENDPOINTS"]
      run_config_args["evaluation_master"] = run_config_args["master"]
    elif not master and cloud_tpu_name:
      # Update run_config to use cluster instead of master/evaluation_master
      # as we need the cluster spec to use Cloud Pods
      tpu_cluster_resolver = tf.contrib.cluster_resolver.TPUClusterResolver(
          cloud_tpu_name)
      run_config_args["cluster"] = tpu_cluster_resolver
      del run_config_args["master"]
      del run_config_args["evaluation_master"]
  elif is_cloud_async_distributed():
    run_config_cls = tf.estimator.RunConfig
    del run_config_args["master"]
    del run_config_args["evaluation_master"]

  config = run_config_cls(**run_config_args)

  # If not using TPU, add device info for data_parallelism
  config.use_tpu = use_tpu
  if not use_tpu:
    config.t2t_device_info = {
        "num_async_replicas": num_async_replicas,
    }
    use_distribution_strategy = (
        optionally_use_dist_strat and
        t2t_model.T2TModel.has_symmetric_shards(model_name) and
        not no_data_parallelism and ps_replicas == 0 and ps_gpu == 0 and
        num_async_replicas == 1)

    if use_distribution_strategy:
      tf.logging.info(
          "Configuring MirroredStrategy DistributionStrategy to replicate the "
          "model."
      )
      distribution = tf.contrib.distribute.MirroredStrategy()
      config = config.replace(train_distribute=distribution)
      config.data_parallelism = None
    else:
      tf.logging.info("Configuring DataParallelism to replicate the model.")
      config.data_parallelism = devices.data_parallelism(
          daisy_chain_variables=daisy_chain_variables,
          ps_replicas=ps_replicas,
          ps_job=ps_job,
          ps_gpu=ps_gpu,
          schedule=schedule,
          sync=sync,
          worker_gpu=num_gpus,
          worker_replicas=num_async_replicas,
          worker_id=worker_id,
          gpu_order=gpu_order,
          worker_job=worker_job,
          no_data_parallelism=no_data_parallelism)

  return config
1325237
static TimingInterval merge(Iterable<TimingInterval> intervals, int maxSamples, long start)
    {
        ThreadLocalRandom rnd = ThreadLocalRandom.current();
        long operationCount = 0, partitionCount = 0, rowCount = 0, errorCount = 0;
        long maxLatency = 0, totalLatency = 0;
        List<SampleOfLongs> latencies = new ArrayList<>();
        long end = 0;
        long pauseStart = 0, pauseEnd = Long.MAX_VALUE;
        for (TimingInterval interval : intervals)
        {
            if (interval != null)
            {
                end = Math.max(end, interval.end);
                operationCount += interval.operationCount;
                maxLatency = Math.max(interval.maxLatency, maxLatency);
                totalLatency += interval.totalLatency;
                partitionCount += interval.partitionCount;
                rowCount += interval.rowCount;
                errorCount += interval.errorCount;
                latencies.addAll(Arrays.asList(interval.sample));
                if (interval.pauseLength > 0)
                {
                    pauseStart = Math.max(pauseStart, interval.pauseStart);
                    pauseEnd = Math.min(pauseEnd, interval.pauseStart + interval.pauseLength);
                }
            }
        }

        if (pauseEnd < pauseStart || pauseStart <= 0)
        {
            pauseEnd = pauseStart = 0;
        }

        return new TimingInterval(start, end, maxLatency, pauseStart, pauseEnd - pauseStart, partitionCount, rowCount,
                                  totalLatency, operationCount, errorCount, SampleOfLongs.merge(rnd, latencies, maxSamples));

    }