Internal implementation of `pipeline_output`.
547821
def _pipeline_output(self, pipeline, chunks, name):
        
        today = normalize_date(self.get_datetime())
        try:
            data = self._pipeline_cache.get(name, today)
        except KeyError:
            # Calculate the next block.
            data, valid_until = self.run_pipeline(
                pipeline, today, next(chunks),
            )
            self._pipeline_cache.set(name, data, valid_until)

        # Now that we have a cached result, try to return the data for today.
        try:
            return data.loc[today]
        except KeyError:
            # This happens if no assets passed the pipeline screen on a given
            # day.
            return pd.DataFrame(index=[], columns=data.columns)
361037
def run_pipelines(pipeline_id_pattern,
                  root_dir,
                  use_cache=True,
                  dirty=False,
                  force=False,
                  concurrency=1,
                  verbose_logs=True,
                  progress_cb=None,
                  slave=False):
    """"""
    with concurrent.futures.ThreadPoolExecutor(max_workers=concurrency,
                                               thread_name_prefix='T') as executor:
        try:
            results = []
            pending_futures = set()
            done_futures = set()
            finished_futures = []
            progress_thread = None
            progress_queue = None
            status_manager = status_mgr(root_dir)

            if progress_cb is not None:
                progress_queue = Queue()
                progress_thread = threading.Thread(target=progress_report_handler, args=(progress_cb, progress_queue))
                progress_thread.start()

            all_specs = specs_to_execute(pipeline_id_pattern, root_dir, status_manager, force, dirty, results)

            while True:

                done = None
                if len(done_futures) > 0:
                    done = done_futures.pop()
                    finished_futures.append(done)
                    done = done.result()[0]

                try:
                    spec = all_specs.send(done)
                except StopIteration:
                    spec = None

                if spec is None:
                    # Wait for all runners to idle...
                    if len(done_futures) == 0:
                        if len(pending_futures) > 0:
                            done_futures, pending_futures = \
                                concurrent.futures.wait(pending_futures,
                                                        return_when=concurrent.futures.FIRST_COMPLETED)
                            continue
                        else:
                            break
                    else:
                        continue

                if len(spec.validation_errors) > 0:
                    results.append(
                        ExecutionResult(spec.pipeline_id,
                                        False,
                                        {},
                                        ['init'] + list(map(str, spec.validation_errors)))
                    )
                    continue

                if slave:
                    ps = status_manager.get(spec.pipeline_id)
                    ps.init(spec.pipeline_details,
                            spec.source_details,
                            spec.validation_errors,
                            spec.cache_hash)
                    eid = gen_execution_id()
                    if ps.queue_execution(eid, 'manual'):
                        success, stats, errors = \
                            execute_pipeline(spec, eid,
                                             use_cache=use_cache)
                        results.append(ExecutionResult(
                            spec.pipeline_id,
                            success,
                            stats,
                            errors
                        ))
                    else:
                        results.append(
                            ExecutionResult(spec.pipeline_id,
                                            False,
                                            None,
                                            ['Already Running'])
                        )

                else:
                    f = executor.submit(remote_execute_pipeline,
                                        spec,
                                        root_dir,
                                        use_cache,
                                        verbose_logs,
                                        progress_queue)
                    pending_futures.add(f)

            for f in finished_futures:
                ret = f.result()
                results.append(ExecutionResult(*ret))

        except KeyboardInterrupt:
            pass
        finally:
            if slave:
                finalize()

            if progress_thread is not None:
                progress_queue.put(None)
                progress_thread.join()

    return results
1250101
def sys_pipes(encoding=_default_encoding):
    
    return pipes(sys.stdout, sys.stderr, encoding=encoding)
935224
def output(data, **kwargs):  # pylint: disable=unused-argument
    '''
    
    '''
    out = ''
    for id_ in data['data']:
        out += '{0}\n'.format(id_)
        for vm_ in data['data'][id_]['vm_info']:
            out += '  {0}\n'.format(vm_)
            vm_data = data[id_]['vm_info'][vm_]
            if 'cpu' in vm_data:
                out += '    CPU: {0}\n'.format(vm_data['cpu'])
            if 'mem' in vm_data:
                out += '    Memory: {0}\n'.format(vm_data['mem'])
            if 'state' in vm_data:
                out += '    State: {0}\n'.format(vm_data['state'])
            if 'graphics' in vm_data:
                if vm_data['graphics'].get('type', '') == 'vnc':
                    out += '    Graphics: vnc - {0}:{1}\n'.format(
                            id_,
                            vm_data['graphics']['port'])
            if 'disks' in vm_data:
                for disk, d_data in six.iteritems(vm_data['disks']):
                    out += '    Disk - {0}:\n'.format(disk)
                    out += '      Size: {0}\n'.format(d_data['disk size'])
                    out += '      File: {0}\n'.format(d_data['file'])
                    out += '      File Format: {0}\n'.format(d_data['file format'])
            if 'nics' in vm_data:
                for mac in vm_data['nics']:
                    out += '    Nic - {0}:\n'.format(mac)
                    out += '      Source: {0}\n'.format(
                                vm_data['nics'][mac]['source'][next(six.iterkeys(vm_data['nics'][mac]['source']))])
                    out += '      Type: {0}\n'.format(vm_data['nics'][mac]['type'])
    return out
912191
def process_output(self, data, output_prompt,
                       input_lines, output, is_doctest, image_file):
        """"""
        if is_doctest:
            submitted = data.strip()
            found = output
            if found is not None:
                found = found.strip()

                # XXX - fperez: in 0.11, 'output' never comes with the prompt
                # in it, just the actual output text.  So I think all this code
                # can be nuked...

                # the above comment does not appear to be accurate... (minrk)

                ind = found.find(output_prompt)
                if ind<0:
                    e='output prompt="%s" does not match out line=%s' % \
                       (output_prompt, found)
                    raise RuntimeError(e)
                found = found[len(output_prompt):].strip()

                if found!=submitted:
                    e = ('doctest failure for input_lines="%s" with '
                         'found_output="%s" and submitted output="%s"' %
                         (input_lines, found, submitted) )
                    raise RuntimeError(e)
678876
def _report_command(self, cmd, procs=None):
        
        if isinstance(procs, list):
            procs = ",".join(map(str,procs))
        if procs:
            line = "\n> `{cmd}` ({procs})\n".format(cmd=str(cmd), procs=procs)
        else:
            line = "\n> `{cmd}`\n".format(cmd=str(cmd))
        print(line)

        with open(self.pipeline_commands_file, "a") as myfile:
            myfile.write(line + "\n\n")
1159851
def dump_all_outputs(self, job, target, abspaths=None):
        
        import os
        import shutil
        from pathlib import Path

        root = Path(native_str(target))
        true_outputs = job.get_output()

        if abspaths or len(true_outputs) < self.BULK_OUTPUT_FILE_THRESHOLD:
            return super().dump_all_outputs(job, root, abspaths)

        stagingdir = root / Path(native_str(job.workingdir)).name
        workdir = job.get_directory(job.workingdir)
        if not root.is_dir():
            root.mkdir(parents=False)
        if stagingdir.exists():
            if PY2:
                raise IOError('Path % exists' % stagingdir)
            else:
                raise FileExistsError(stagingdir)
        workdir.put(str(root))
        assert stagingdir.is_dir()
        assert root in stagingdir.parents

        for pathstr in true_outputs:
            if os.path.isabs(pathstr):
                continue
            destpath = root / pathstr
            currpath = stagingdir / pathstr
            if not destpath.parent.is_dir():
                destpath.parent.mkdir(parents=True)
            currpath.rename(destpath)

        shutil.rmtree(str(stagingdir))
786293
def _output_work(self, work, root):
        """"""
        output_filename = os.path.join(self._output_dir, work)
        tree = etree.ElementTree(root)
        tree.write(output_filename, encoding='utf-8', pretty_print=True)
1090152
def output(self):
        
        def decorator(outputMethod):
            return MethodicalOutput(machine=self, method=outputMethod)
        return decorator
702739
def dump_all_outputs(self, job, target, abspaths=None):
        
        from pathlib import Path
        root = Path(native_str(target))

        for outputpath, outputfile in job.get_output().items():
            path = Path(native_str(outputpath))

            # redirect absolute paths into the appropriate subdirectory
            if path.is_absolute():
                if abspaths:
                    path = Path(native_str(abspaths), *path.parts[1:])
                else:
                    continue

            dest = root / path
            if not dest.parent.is_dir():
                dest.parent.mkdir(parents=True)
            if dest.is_file():
                dest.unlink()
            try:
                outputfile.put(str(dest))
            except IsADirectoryError:
                if not dest.is_dir():
                    dest.mkdir(parents=True)
520262
def run(pipeline_id, verbose, use_cache, dirty, force, concurrency, slave):
    """"""
    exitcode = 0

    running = []
    progress = {}

    def progress_cb(report):
        pid, count, success, *_, stats = report

        print('\x1b[%sA' % (1+len(running)))
        if pid not in progress:
            running.append(pid)
        progress[pid] = count, success

        for pid in running:
            count, success = progress[pid]
            if success is None:
                if count == 0:
                    print('\x1b[2K%s: \x1b[31m%s\x1b[0m' % (pid, 'WAITING FOR OUTPUT'))
                else:
                    print('\x1b[2K%s: \x1b[33mRUNNING, processed %s rows\x1b[0m' % (pid, count))
            else:
                if success:
                    print('\x1b[2K%s: \x1b[32mSUCCESS, processed %s rows\x1b[0m' % (pid, count))
                else:
                    print('\x1b[2K%s: \x1b[31mFAILURE, processed %s rows\x1b[0m' % (pid, count))

    results = run_pipelines(pipeline_id, '.', use_cache,
                            dirty, force, concurrency,
                            verbose, progress_cb if not verbose else None,
                            slave)
    if not slave:
        logging.info('RESULTS:')
        errd = False
        for result in results:
            stats = user_facing_stats(result.stats)
            errd = errd or result.errors or not result.success
            logging.info('%s: %s %s%s',
                         'SUCCESS' if result.success else 'FAILURE',
                         result.pipeline_id,
                         repr(stats) if stats is not None else '',
                         (
                            '\nERROR log from processor %s:\n+--------\n| ' % result.errors[0] +
                            '\n| '.join(result.errors[1:]) +
                            '\n+--------'
                         ) if result.errors else '')
    else:
        result_obj = []
        errd = False
        for result in results:
            errd = errd or result.errors or not result.success
            stats = user_facing_stats(result.stats)
            result_obj.append(dict(
                success=result.success,
                pipeline_id=result.pipeline_id,
                stats=result.stats,
                errors=result.errors
            ))
            json.dump(result_obj, sys.stderr)

    if errd:
        exitcode = 1

    exit(exitcode)
511687
def output(stream):
    """"""
    while True:
        content = stream.read(1024)
        if len(content) == 0: break
        sys.stdout.write(content)
223845
def output_tree(self, t, seen):
        """"""
        if not t:
            return
        for c in t.children:
            self.output_tree(c, seen)
        if t.name in seen:
            return
        seen[t.name] = True
        if t.resource_name:
            output_class(t.name, t.props, self.override, t.resource_name)
        else:
            output_class(t.name, t.props, self.override)
570061
def output_file(self, _container):
        """"""
        p = local.path(_container)
        if p.exists():
            if not ui.ask("Path '{0}' already exists."
                          " Overwrite?".format(p)):
                sys.exit(0)
        CFG["container"]["output"] = str(p)
881464
def _parse_stdout(self, out_folder):
        """"""
        fname = self.node.load_process_class()._DEFAULT_OUTPUT_FILE  # pylint: disable=protected-access
        if fname not in out_folder._repository.list_object_names():  # pylint: disable=protected-access
            raise OutputParsingError("Cp2k output file not retrieved")

        result_dict = {'exceeded_walltime': False}
        abs_fn = os.path.join(out_folder._repository._get_base_folder().abspath, fname)  # pylint: disable=protected-access
        with io.open(abs_fn, mode="r", encoding="utf-8") as fobj:
            lines = fobj.readlines()
            for i_line, line in enumerate(lines):
                if line.startswith(' ENERGY| '):
                    result_dict['energy'] = float(line.split()[8])
                    result_dict['energy_units'] = "a.u."
                if 'The number of warnings for this run is' in line:
                    result_dict['nwarnings'] = int(line.split()[-1])
                if 'exceeded requested execution time' in line:
                    result_dict['exceeded_walltime'] = True
                if "KPOINTS| Band Structure Calculation" in line:
                    from aiida.orm import BandsData
                    bnds = BandsData()
                    kpoints, labels, bands = self._parse_bands(lines, i_line)
                    bnds.set_kpoints(kpoints)
                    bnds.labels = labels
                    bnds.set_bands(bands, units='eV')
                    self.out('output_bands', bnds)

        if 'nwarnings' not in result_dict:
            raise OutputParsingError("CP2K did not finish properly.")

        self.out('output_parameters', Dict(dict=result_dict))
727944
def getOutput(self):
        
        output = self.stdout
        if self.stdout:
            output += '\r\n'
        output += self.stderr
        return output
423784
function parseLdOutput(output) {
    const libraryList = output.split('\n').filter(ln => ln.includes('=>'));

    const result = {};
    libraryList.forEach(s => {
        const [name, _, libPath] = s.trim().split(' ');
        result[name] = libPath;
    });

    return result;
}