logstash install
        :return:
426033
def logstash_install(self):
        
        with cd('/tmp'):
            if not exists('logstash.deb'):
                sudo('wget {0} -O logstash.deb'.format(
                    bigdata_conf.logstash_download_url
                ))

            sudo('dpkg -i logstash.deb')
            sudo('apt-get install -y')
1250057
def docker_installed_rpms(broker):
        
        ctx = broker[DockerImageContext]
        root = ctx.root
        fmt = DefaultSpecs.rpm_format
        cmd = "/usr/bin/rpm -qa --root %s --qf '%s'" % (root, fmt)
        result = ctx.shell_out(cmd)
        return CommandOutputProvider(cmd, ctx, content=result)
1151174
def GLOGFILE(self):
        
        lines = []
        for package in sorted(self._packages.keys()):
            lines.append("%s %s" % (str(package), str(self._packages[package])))

        return "\n".join(lines)
562587
def setup_logdir(self):
    # todo: locking on logdir creation

    
    run_name = ncluster_globals.get_run_for_task(self)
    self.log("Creating logdir for run "+run_name)
    logdir_root = ncluster_globals.LOGDIR_ROOT
    assert logdir_root

    self.run(f'mkdir -p {logdir_root}')
    find_command = f'find {logdir_root} -maxdepth 1 -type d'

    stdout, stderr = self.run_with_output(find_command)
    logdir = f"{logdir_root}/{run_name}"

    counter = 0
    while logdir in stdout:
      counter += 1
      new_logdir = f'{logdir_root}/{run_name}.{counter:02d}'
      self.log(f'Warning, logdir {logdir} exists, deduping to {new_logdir}')
      logdir = new_logdir
    self.run(f'mkdir -p {logdir}')

    ncluster_globals.set_logdir(run_name, logdir)
    return logdir
1136171
def import_results(log, pathToYamlFile):
    
    ################ > IMPORTS ################
    ## STANDARD LIB ##
    ## THIRD PARTY ##
    import yaml
    ## LOCAL APPLICATION ##

    ################ >ACTION(S) ################
    fileName = pathToYamlFile
    stream = file(fileName, 'r')
    yamlContent = yaml.load(stream)
    snSurveyDiscoveryTimes = yamlContent['Discoveries Relative to Survey Year']
    lightCurveDiscoveryTimes = yamlContent[
        'Discoveries Relative to Peak Magnitudes']
    snTypes = yamlContent['SN Types']
    redshifts = yamlContent['Redshifts']
    cadenceDictionary = yamlContent['Cadence Dictionary']
    peakAppMagList = yamlContent['Peak Apparent Magnitudes']
    snCampaignLengthList = yamlContent['Campaign Length']

    # log.info('yamlContent %s' % (yamlContent,))
    stream.close()

    return snSurveyDiscoveryTimes, lightCurveDiscoveryTimes, snTypes, redshifts, cadenceDictionary, peakAppMagList, snCampaignLengthList
1074161
def setup(cls,
              opts=type('opts', (), {
                  'background': None,
                  'logdir': None,
                  'logging_conf_file': None,
                  'log_level': 'DEBUG'
              })):
        """"""
        logger = logging.getLogger('luigi')

        if cls._configured:
            logger.info('logging already configured')
            return False
        cls._configured = True

        if cls.config.getboolean('core', 'no_configure_logging', False):
            logger.info('logging disabled in settings')
            return False

        configured = cls._cli(opts)
        if configured:
            logger = logging.getLogger('luigi')
            logger.info('logging configured via special settings')
            return True

        configured = cls._conf(opts)
        if configured:
            logger = logging.getLogger('luigi')
            logger.info('logging configured via *.conf file')
            return True

        configured = cls._section(opts)
        if configured:
            logger = logging.getLogger('luigi')
            logger.info('logging configured via config section')
            return True

        configured = cls._default(opts)
        if configured:
            logger = logging.getLogger('luigi')
            logger.info('logging configured by default settings')
        return configured
113456
def logs(self, num=None, source=None, ps=None, tail=False):
        """"""

        # Bootstrap payload package.
        payload = {'logplex': 'true'}

        if num:
            payload['num'] = num

        if source:
            payload['source'] = source

        if ps:
            payload['ps'] = ps

        if tail:
            payload['tail'] = 1

        # Grab the URL of the logplex endpoint.
        r = self._h._http_resource(
            method='GET',
            resource=('apps', self.name, 'logs'),
            data=payload
        )

        # Grab the actual logs.
        r = requests.get(r.content.decode("utf-8"), verify=False, stream=True)

        if not tail:
            return r.content
        else:
            # Return line iterator for tail!
            return r.iter_lines()
936486
def log_path(scraper):
    
    return os.path.join(scraper.config.data_path,
                        '%s.jsonlog' % scraper.name)
805883
def crawl_result_path(result_path, include_log):
    """"""
    result = {
        'logs': [],
        'args': [],
        'commands': [],
        'snapshots': []
    }

    if os.path.isdir(result_path):
        if include_log:
            result['logs'] = load_result_json(result_path, 'log')
        result['args'] = load_result_json(result_path, 'args')
        result['commands'] = load_result_json(result_path, 'commands')

        snapshots = [
            x for x in os.listdir(result_path) if x.count('snapshot_iter_')
        ]
        snapshots.sort()
        result['snapshots'] = snapshots

    return result
1330828
def RunInstaller():
  

  try:
    os.makedirs(os.path.dirname(config.CONFIG["Installer.logfile"]))
  except OSError:
    pass

  # Always log to the installer logfile at debug level. This way if our
  # installer fails we can send detailed diagnostics.
  handler = logging.FileHandler(config.CONFIG["Installer.logfile"], mode="wb")

  handler.setLevel(logging.DEBUG)

  # Add this to the root logger.
  logging.getLogger().addHandler(handler)

  # Ordinarily when the client starts up, the local volatile
  # configuration is read. Howevwer, when running the installer, we
  # need to ensure that only the installer configuration is used so
  # nothing gets overridden by local settings. We there must reload
  # the configuration from the flag and ignore the Config.writeback
  # location.
  config.CONFIG.Initialize(filename=flags.FLAGS.config, reset=True)
  config.CONFIG.AddContext(contexts.INSTALLER_CONTEXT,
                           "Context applied when we run the client installer.")

  logging.warning("Starting installation procedure for GRR client.")
  try:
    Installer().Init()
  except Exception as e:  # pylint: disable=broad-except
    # Ouch! we failed to install... Not a lot we can do
    # here - just log the error and give up.
    logging.exception("Installation failed: %s", e)

    # Error return status.
    sys.exit(-1)

  # Exit successfully.
  sys.exit(0)
237852
def add_log_hooks_to_pytorch_module(self, module, name=None, prefix='', log_parameters=True, log_gradients=True, log_freq=0):
        
        if name is not None:
            prefix = prefix + name

        if log_parameters:
            def parameter_log_hook(module, input_, output, log_track):
                if not log_track_update(log_track):
                    return
                for name, parameter in module.named_parameters():
                    # for pytorch 0.3 Variables
                    if isinstance(parameter, torch.autograd.Variable):
                        data = parameter.data
                    else:
                        data = parameter
                    self.log_tensor_stats(
                        data.cpu(), 'parameters/' + prefix + name)
            log_track_params = log_track_init(log_freq)
            module.register_forward_hook(
                lambda mod, inp, outp: parameter_log_hook(mod, inp, outp, log_track_params))

        if log_gradients:
            for name, parameter in module.named_parameters():
                if parameter.requires_grad:
                    log_track_grad = log_track_init(log_freq)
                    self._hook_variable_gradient_stats(
                        parameter, 'gradients/' + prefix + name, log_track_grad)
1244135
def setup(self, settings):
        '''
        
        '''
        # set up the default sc logger
        my_level = settings.get('SC_LOG_LEVEL', 'INFO')
        my_name = settings.get('SClogger_NAME', 'sc-logger')
        my_output = settings.get('SC_LOG_STDOUT', True)
        my_json = settings.get('SC_LOG_JSON', False)
        my_dir = settings.get('SC_LOG_DIR', 'logs')
        my_bytes = settings.get('SC_LOG_MAX_BYTES', '10MB')
        my_file = settings.get('SC_LOG_FILE', 'main.log')
        my_backups = settings.get('SC_LOG_BACKUPS', 5)

        self.logger = LogFactory.get_instance(json=my_json,
                                         name=my_name,
                                         stdout=my_output,
                                         level=my_level,
                                         dir=my_dir,
                                         file=my_file,
                                         bytes=my_bytes,
                                         backups=my_backups)

        self.settings = settings
        self.stats_dict = {}

        # set up redis
        self.redis_conn = redis.Redis(host=settings.get('REDIS_HOST'),
            port=settings.get('REDIS_PORT'),
            db=settings.get('REDIS_DB'))

        try:
            self.redis_conn.info()
            self.logger.debug("Connected to Redis in ScraperHandler")
        except ConnectionError:
            self.logger.error("Failed to connect to Redis in Stats Middleware")
            # plugin is essential to functionality
            sys.exit(1)
836693
def _setup(self):
        
        self._populate_local()
        try:
            self._populate_latest()
        except Exception as e:
            self.log.exception('Unable to retrieve latest %s version information', self.meta_name)
        self._sort()
1095826
def setup_scrapy():
    

    def replace_configure_logging(install_root_handler=False, settings=None):
        fake_use(install_root_handler)
        fake_use(settings)

    def replace_log_scrapy_info(settings=None):
        fake_use(settings)

    # scrapy stuff
    import scrapy.utils.log  # for configure_logging
    import scrapy.crawler  # configure_logging, log_scrapy_info
    logging_settings = {
        'LOG_ENABLED': False,
        'LOG_LEVEL': logging.WARN,
    }
    scrapy.utils.log.configure_logging(install_root_handler=False, settings=logging_settings)
    # are you watching closely?!?
    scrapy.crawler.configure_logging = replace_configure_logging
    scrapy.crawler.log_scrapy_info = replace_log_scrapy_info
1066890
def setupLogFile(self):
		""""""
		self.logWrite("\n###############################################")
		self.logWrite("calcpkg.py log from " + str(datetime.datetime.now()))
		self.changeLogging(True)
207139
def setup_prefix_logging(logdir):
    
    if not os.path.exists(logdir):
        os.mkdir(logdir)

    file_handler = logging.FileHandler(
        filename=os.path.join(logdir, 'lago.log'),
    )
    file_formatter = get_default_log_formatter()
    file_handler.setFormatter(file_formatter)
    logging.root.addHandler(file_handler)
    hide_paramiko_logs()
    hide_stevedore_logs()
528372
def lint(session):
    

    session.install('flake8', *LOCAL_DEPS)
    session.install('-e', '.')
    session.run(
        'flake8', os.path.join('google', 'cloud', 'bigquery_storage_v1beta1'))
    session.run('flake8', 'tests')