Detect access log config (path and format) of nginx. Offer user to select if multiple access logs are detected.
    :return: path and format of detected / selected access log
427777
def detect_log_config(arguments):
    
    config = arguments['--config']
    if config is None:
        config = detect_config_path()
    if not os.path.exists(config):
        error_exit('Nginx config file not found: %s' % config)

    with open(config) as f:
        config_str = f.read()
    access_logs = dict(get_access_logs(config_str))
    if not access_logs:
        error_exit('Access log file is not provided and ngxtop cannot detect it from your config file (%s).' % config)

    log_formats = dict(get_log_formats(config_str))
    if len(access_logs) == 1:
        log_path, format_name = list(access_logs.items())[0]
        if format_name == 'combined':
            return log_path, LOG_FORMAT_COMBINED
        if format_name not in log_formats:
            error_exit('Incorrect format name set in config for access log file "%s"' % log_path)
        return log_path, log_formats[format_name]

    # multiple access logs configured, offer to select one
    print('Multiple access logs detected in configuration:')
    log_path = choose_one(list(access_logs.keys()), 'Select access log file to process: ')
    format_name = access_logs[log_path]
    if format_name not in log_formats:
        error_exit('Incorrect format name set in config for access log file "%s"' % log_path)
    return log_path, log_formats[format_name]
448682
def setup_logging():
    
    default_path_yaml = os.path.expanduser('~/.pylogconf.yaml')
    default_path_conf = os.path.expanduser('~/.pylogconf.conf')
    # this matches the default logging level of the logging
    # library and makes sense...
    default_level = logging.WARNING

    dbg = os.getenv("PYLOGCONF_DEBUG", False)

    """ try YAML config file first """
    value = os.getenv('PYLOGCONF_YAML', None)
    if value is None:
        path = default_path_yaml
    else:
        path = value
    if os.path.isfile(path):
        _debug('found logging configuration file [{0}]...'.format(path), dbg)
        with open(path) as f:
            config = yaml.load(f.read())
            logging.config.dictConfig(config)
            return

    """ Now try regular config file """
    value = os.getenv('PYLOGCONF_CONF', None)
    if value is None:
        path = default_path_conf
    else:
        path = value
    if os.path.isfile(path):
        _debug('found logging configuration file [{0}]...'.format(path), dbg)
        logging.config.fileConfig(path)
        return

    _debug('logging with level [{0}]...'.format(default_level), dbg)
    logging.basicConfig(level=default_level)
1289273
def log_analyzer2(path):
    
    with handle(MalformedLogEntryError,
                  lambda (c):
                      invoke_restart('reparse',
                                     'ERROR: ' + c.text)):
        for filename in find_all_logs(path):
            analyze_log(filename)
559445
def _read_from_paths():
    
    home = os.environ.get("HOME", "")
    home_path = os.path.join(home, _SETTINGS_PATH)
    etc_path = os.path.join("/etc", _SETTINGS_PATH)
    env_path = os.environ.get("SETTINGS_PATH", "")

    read_path = None
    if env_path and os.path.exists(env_path):
        read_path = env_path
    elif home and os.path.exists(home_path):
        read_path = home_path
    elif os.path.exists(etc_path):
        read_path = etc_path

    if not read_path:
        return "{}"

    with open(read_path) as f:
        return f.read()
1349787
def media_nginx_accel(request, path):
    
    response = HttpResponse(status=200)
    response['Content-Type'] = ''
    response['X-Accel-Redirect'] = '/protected' + request.path
    return response
1197287
def get_log_dir(self, conf_file):
        
        log_dir = None

        with open(conf_file, 'r') as conf_f:
            for line in conf_f:
                line = line.split('#')[0].strip()

                try:
                    (key, value) = line.split('=', 1)
                    if key.strip() == 'log-dir':
                        log_dir = value.strip()
                except ValueError:
                    pass

        return log_dir
871123
def default_logging(grab_log=None, # '/tmp/grab.log',
                    network_log=None, # '/tmp/grab.network.log',
                    level=logging.DEBUG, mode='a',
                    propagate_network_logger=False):
    

    logging.basicConfig(level=level)

    network_logger = logging.getLogger('grab.network')
    network_logger.propagate = propagate_network_logger
    if network_log:
        hdl = logging.FileHandler(network_log, mode)
        network_logger.addHandler(hdl)
        network_logger.setLevel(level)

    grab_logger = logging.getLogger('grab')
    if grab_log:
        hdl = logging.FileHandler(grab_log, mode)
        grab_logger.addHandler(hdl)
        grab_logger.setLevel(level)
356001
def nginx_web_ssl_config(self):
        

        dt = [self.nginx_web_dir, self.nginx_ssl_dir]
        return nginx_conf_string.simple_ssl_web_conf.format(dt=dt)
144214
def load_config(self, config):
        """"""

        # Read AMP confifuration.
        # For ex, the AMP foo should have the following section:
        #
        # [foo]
        # enable=true
        # regex=\/usr\/bin\/nginx
        # refresh=60
        #
        # and optionnaly:
        #
        # one_line=false
        # option1=opt1
        # ...
        #
        amp_section = 'amp_' + self.amp_name
        if (hasattr(config, 'has_section') and
                config.has_section(amp_section)):
            logger.debug("AMP - {}: Load configuration".format(self.NAME))
            for param, _ in config.items(amp_section):
                try:
                    self.configs[param] = config.get_float_value(amp_section, param)
                except ValueError:
                    self.configs[param] = config.get_value(amp_section, param).split(',')
                    if len(self.configs[param]) == 1:
                        self.configs[param] = self.configs[param][0]
                logger.debug("AMP - {}: Load parameter: {} = {}".format(self.NAME, param, self.configs[param]))
        else:
            logger.debug("AMP - {}: Can not find section {} in the configuration file".format(self.NAME, self.amp_name))
            return False

        # enable, regex and refresh are mandatories
        # if not configured then AMP is disabled
        if self.enable():
            for k in ['regex', 'refresh']:
                if k not in self.configs:
                    logger.warning("AMP - {}: Can not find configuration key {} in section {}".format(self.NAME, k, self.amp_name))
                    self.configs['enable'] = 'false'
        else:
            logger.debug("AMP - {} is disabled".format(self.NAME))

        # Init the count to 0
        self.configs['count'] = 0

        return self.enable()
1242
def init_access_log
      return if access_log_dest.nil?

      log = ::Logging.logger['access_log']
      pattern = ::Logging.layouts.pattern(:pattern => ACCESS_LOG_PATTERN)

      log.add_appenders(
        ::Logging.appenders.rolling_file(access_log_dest,
          :level => :debug,
          :age => 'daily',
          :layout => pattern)
        )

      log
    end
643035
def _load_config(self, path):
        '''
        
        '''
        config = {}
        log.debug('Reading configuration from %s', path)
        if not os.path.isdir(path):
            msg = (
                'Unable to read from {path}: '
                'the directory does not exist!'
            ).format(path=path)
            log.error(msg)
            raise IOError(msg)
        # The directory tree should look like the following:
        # .
        # ├── __init__.py
        # ├── eos
        # │   └── init.yml
        # ├── iosxr
        # │   └── __init__.py
        # ├── junos
        # │   └── init.yml
        # │   └── bgp_read_message.py
        # │   └── BGP_PREFIX_THRESH_EXCEEDED.py
        # └── nxos
        #     └── init.yml
        os_subdirs = [sdpath[0] for sdpath in os.walk(path)][1:]
        if not os_subdirs:
            log.error('%s does not contain any OS subdirectories', path)
        for os_dir in os_subdirs:
            os_name = os.path.split(os_dir)[1]  # the network OS name
            if os_name.startswith('__'):
                log.debug('Ignoring %s', os_name)
                continue
            if not self._whitelist_blacklist(os_name):
                log.debug('Not building config for %s (whitelist-blacklist logic)', os_name)
                # Ignore devices that are not in the whitelist (if defined),
                #   or those operating systems that are on the blacklist.
                # This way we can prevent starting unwanted sub-processes.
                continue
            log.debug('Building config for %s:', os_name)
            log.debug('='*40)
            if os_name not in config:
                config[os_name] = {}
            files = os.listdir(os_dir)
            # Read all files under the OS dir
            for file_ in files:
                log.debug('Inspecting %s', file_)
                file_name, file_extension = os.path.splitext(file_)
                file_extension = file_extension.replace('.', '')
                filepath = os.path.join(os_dir, file_)
                comment = ''
                if file_extension in ('yml', 'yaml'):
                    try:
                        log.debug('Loading %s as YAML', file_)
                        with open(filepath, 'r') as fstream:
                            cfg = yaml.load(fstream)
                            # Reposition at the top and read the comments.
                            if file_name not in CONFIG.OS_INIT_FILENAMES:
                                # If the file name is not a profile init.
                                fstream.seek(0)
                                comment = self._extract_yaml_docstring(fstream)
                                if 'messages' in cfg:
                                    for message in cfg['messages']:
                                        message['__doc__'] = comment
                            napalm_logs.utils.dictupdate(config[os_name], cfg)
                    except yaml.YAMLError as yamlexc:
                        log.error('Invalid YAML file: %s', filepath, exc_info=True)
                        if file_name in CONFIG.OS_INIT_FILENAMES:
                            # Raise exception and break only when the init file is borked
                            #   otherwise, it will try loading best efforts.
                            raise IOError(yamlexc)
                elif file_extension == 'py':
                    log.debug('Lazy loading Python module %s', file_)
                    mod_fp, mod_file, mod_data = imp.find_module(file_name, [os_dir])
                    mod = imp.load_module(file_name, mod_fp, mod_file, mod_data)
                    if file_name in CONFIG.OS_INIT_FILENAMES:
                        # Init file defined as Python module
                        log.debug('%s seems to be a Python profiler', filepath)
                        # Init files require to define the `extract` function.
                        # Sample init file:
                        # def extract(message):
                        #     return {'tag': 'A_TAG', 'host': 'hostname'}
                        if hasattr(mod, CONFIG.INIT_RUN_FUN) and\
                           hasattr(getattr(mod, CONFIG.INIT_RUN_FUN), '__call__'):
                            # if extract is defined and is callable
                            if 'prefixes' not in config[os_name]:
                                config[os_name]['prefixes'] = []
                            config[os_name]['prefixes'].append({
                                'values': {'tag': ''},
                                'line': '',
                                '__python_fun__': getattr(mod, CONFIG.INIT_RUN_FUN),
                                '__python_mod__': filepath  # Will be used for debugging
                            })
                            log.info('Adding the prefix function defined under %s to %s',
                                     filepath, os_name)
                        elif file_name != '__init__':
                            # If __init__.py does not have the extractor function, no problem.
                            log.warning('%s does not have the "%s" function defined. Ignoring.',
                                        filepath, CONFIG.INIT_RUN_FUN)
                    else:
                        # Other python files require the `emit` function.
                        if hasattr(mod, '__tag__'):
                            mod_tag = getattr(mod, '__tag__')
                        else:
                            log.info('%s does not have __tag__, defaulting the tag to %s', filepath, file_name)
                            mod_tag = file_name
                        if hasattr(mod, '__error__'):
                            mod_err = getattr(mod, '__error__')
                        else:
                            log.info('%s does not have __error__, defaulting the error to %s', filepath, file_name)
                            mod_err = file_name
                        if hasattr(mod, '__match_on__'):
                            err_match = getattr(mod, '__match_on__')
                        else:
                            err_match = 'tag'
                        model = CONFIG.OPEN_CONFIG_NO_MODEL
                        if hasattr(mod, '__yang_model__'):
                            model = getattr(mod, '__yang_model__')
                        log.debug('Mathing on %s', err_match)
                        if hasattr(mod, CONFIG.CONFIG_RUN_FUN) and\
                           hasattr(getattr(mod, CONFIG.CONFIG_RUN_FUN), '__call__'):
                            log.debug('Adding %s with tag:%s, error:%s, matching on:%s',
                                      file_, mod_tag, mod_err, err_match)
                            # the structure below must correspond to the VALID_CONFIG structure enforcement
                            if 'messages' not in config[os_name]:
                                config[os_name]['messages'] = []
                            config[os_name]['messages'].append({
                                'tag': mod_tag,
                                'error': mod_err,
                                'match_on': err_match,
                                '__doc__': mod.__doc__,
                                '__python_fun__': getattr(mod, CONFIG.CONFIG_RUN_FUN),
                                '__python_mod__': filepath,  # Will be used for debugging
                                'line': '',
                                'model': model,
                                'values': {},
                                'mapping': {'variables': {}, 'static': {}}
                            })
                        else:
                            log.warning('%s does not have the "%s" function defined. Ignoring.',
                                        filepath, CONFIG.CONFIG_RUN_FUN)
                else:
                    log.info('Ignoring %s (extension not allowed)', filepath)
            log.debug('-'*40)
        if not config:
            msg = 'Could not find proper configuration files under {path}'.format(path=path)
            log.error(msg)
            raise IOError(msg)
        log.debug('Complete config:')
        log.debug(config)
        log.debug('ConfigParserg size in bytes: %d', sys.getsizeof(config))
        return config
368893
def get_connection_logging(self, loadbalancer):
        
        uri = "/loadbalancers/%s/connectionlogging" % utils.get_id(loadbalancer)
        resp, body = self.api.method_get(uri)
        ret = body.get("connectionLogging", {}).get("enabled", False)
        return ret
734120
def init_log_config(handler=None):
    
    for applog in lognames.values():
        # propagate except for root app logger 'linkcheck'
        propagate = (applog != LOG_ROOT)
        configdict['loggers'][applog] = dict(level='INFO', propagate=propagate)

    logging.config.dictConfig(configdict)
    if handler is None:
        handler = ansicolor.ColoredStreamHandler(strm=sys.stderr)
    add_loghandler(handler)
752141
def _read_config(self):
        
        if not self._file_path:
            return None
        elif self._file_path.startswith('s3://'):
            return self._read_s3_config()
        elif self._file_path.startswith('http://') or \
                self._file_path.startswith('https://'):
            return self._read_remote_config()
        elif not path.exists(self._file_path):
            raise ValueError(
                'Configuration file not found: {}'.format(self._file_path))

        with open(self._file_path, 'r') as handle:
            return handle.read()
1079416
def file_logger(app, level=None):
    
    path = os.path.join(os.getcwd(), 'var', 'logs',  'app.log')

    max_bytes = 1024 * 1024 * 2
    file_handler = RotatingFileHandler(
        filename=path,
        mode='a',
        maxBytes=max_bytes,
        backupCount=10
    )

    if level is None: level = logging.INFO
    file_handler.setLevel(level)

    log_format  = '%(asctime)s %(levelname)s: %(message)s'
    log_format += ' [in %(pathname)s:%(lineno)d]'
    file_handler.setFormatter(logging.Formatter(log_format))

    return file_handler
1145952
def locate_config(self):
        '''  '''
        for f in self.__potential:
            f = FileHelper.abspath(f)
            if os.path.isfile(f):
                return f
        return None
837327
def read_settings(self, configfile):
        '''  '''
        if six.PY3:
            config = configparser.ConfigParser()
        else:
            config = configparser.SafeConfigParser()
        config.read(configfile)

        # is eucalyptus?
        self.eucalyptus_host = None
        self.eucalyptus = False
        if config.has_option('ec2', 'eucalyptus'):
            self.eucalyptus = config.getboolean('ec2', 'eucalyptus')
        if self.eucalyptus and config.has_option('ec2', 'eucalyptus_host'):
            self.eucalyptus_host = config.get('ec2', 'eucalyptus_host')

        # Regions
        self.regions = []
        configRegions = config.get('ec2', 'regions')
        configRegions_exclude = config.get('ec2', 'regions_exclude')
        if (configRegions == 'all'):
            if self.eucalyptus_host:
                self.regions.append(boto.connect_euca(host=self.eucalyptus_host).region.name)
            else:
                for regionInfo in ec2.regions():
                    if regionInfo.name not in configRegions_exclude:
                        self.regions.append(regionInfo.name)
        else:
            self.regions = configRegions.split(",")

        # Destination addresses
        self.destination_variable = config.get('ec2', 'destination_variable')
        self.vpc_destination_variable = config.get('ec2',
                                                   'vpc_destination_variable')

        # Route53
        self.route53_enabled = config.getboolean('ec2', 'route53')
        self.route53_excluded_zones = []
        if config.has_option('ec2', 'route53_excluded_zones'):
            self.route53_excluded_zones.extend(
                config.get('ec2', 'route53_excluded_zones', '').split(','))

        # Include RDS instances?
        self.rds_enabled = True
        if config.has_option('ec2', 'rds'):
            self.rds_enabled = config.getboolean('ec2', 'rds')

        # Include ElastiCache instances?
        self.elasticache_enabled = True
        if config.has_option('ec2', 'elasticache'):
            self.elasticache_enabled = config.getboolean('ec2', 'elasticache')

        # Return all EC2 instances?
        if config.has_option('ec2', 'all_instances'):
            self.all_instances = config.getboolean('ec2', 'all_instances')
        else:
            self.all_instances = False

        # Instance states to be gathered in inventory. Default is 'running'.
        # Setting 'all_instances' to 'yes' overrides this option.
        ec2_valid_instance_states = [
            'pending',
            'running',
            'shutting-down',
            'terminated',
            'stopping',
            'stopped'
        ]
        self.ec2_instance_states = []
        if self.all_instances:
            self.ec2_instance_states = ec2_valid_instance_states
        elif config.has_option('ec2', 'instance_states'):
            for instance_state in config.get('ec2',
                                             'instance_states').split(','):
                instance_state = instance_state.strip()
                if instance_state not in ec2_valid_instance_states:
                    continue
                self.ec2_instance_states.append(instance_state)
        else:
            self.ec2_instance_states = ['running']

        # Return all RDS instances? (if RDS is enabled)
        if config.has_option('ec2', 'all_rds_instances') and self.rds_enabled:
            self.all_rds_instances = config.getboolean('ec2',
                                                       'all_rds_instances')
        else:
            self.all_rds_instances = False

        # Return all ElastiCache replication groups?
        # (if ElastiCache is enabled)
        if (config.has_option('ec2', 'all_elasticache_replication_groups')
                and self.elasticache_enabled):
            self.all_elasticache_replication_groups = config.getboolean(
                'ec2', 'all_elasticache_replication_groups')
        else:
            self.all_elasticache_replication_groups = False

        # Return all ElastiCache clusters? (if ElastiCache is enabled)
        if (config.has_option('ec2', 'all_elasticache_clusters')
                and self.elasticache_enabled):
            self.all_elasticache_clusters = config.getboolean(
                'ec2', 'all_elasticache_clusters')
        else:
            self.all_elasticache_clusters = False

        # Return all ElastiCache nodes? (if ElastiCache is enabled)
        if config.has_option('ec2', 'all_elasticache_nodes') and self.elasticache_enabled:
            self.all_elasticache_nodes = config.getboolean('ec2', 'all_elasticache_nodes')
        else:
            self.all_elasticache_nodes = False

        # boto configuration profile (prefer CLI argument)
        if config.has_option('ec2', 'boto_profile') and not self.boto_profile:
            self.boto_profile = config.get('ec2', 'boto_profile')

        # Cache related
        cache_dir = os.path.expanduser(config.get('ec2', 'cache_path'))
        if self.boto_profile:
            cache_dir = os.path.join(cache_dir, 'profile_' + self.boto_profile)
        if not os.path.exists(cache_dir):
            os.makedirs(cache_dir)

        self.cache_path_cache = cache_dir + "/ansible-ec2.cache"
        self.cache_path_index = cache_dir + "/ansible-ec2.index"
        self.cache_max_age = config.getint('ec2', 'cache_max_age')

        if config.has_option('ec2', 'expand_csv_tags'):
            self.expand_csv_tags = config.getboolean('ec2', 'expand_csv_tags')
        else:
            self.expand_csv_tags = False

        # Configure nested groups instead of flat namespace.
        if config.has_option('ec2', 'nested_groups'):
            self.nested_groups = config.getboolean('ec2', 'nested_groups')
        else:
            self.nested_groups = False

        # Replace dash or not in group names
        if config.has_option('ec2', 'replace_dash_in_groups'):
            self.replace_dash_in_groups = config.getboolean('ec2', 'replace_dash_in_groups')
        else:
            self.replace_dash_in_groups = True

        # Configure which groups should be created.
        group_by_options = [
            'group_by_instance_id',
            'group_by_region',
            'group_by_availability_zone',
            'group_by_ami_id',
            'group_by_instance_type',
            'group_by_key_pair',
            'group_by_vpc_id',
            'group_by_security_group',
            'group_by_tag_keys',
            'group_by_tag_none',
            'group_by_route53_names',
            'group_by_rds_engine',
            'group_by_rds_parameter_group',
            'group_by_elasticache_engine',
            'group_by_elasticache_cluster',
            'group_by_elasticache_parameter_group',
            'group_by_elasticache_replication_group',
        ]
        for option in group_by_options:
            if config.has_option('ec2', option):
                setattr(self, option, config.getboolean('ec2', option))
            else:
                setattr(self, option, True)

        # Do we need to just include hosts that match a pattern?
        try:
            pattern_include = config.get('ec2', 'pattern_include')
            if pattern_include and len(pattern_include) > 0:
                self.pattern_include = re.compile(pattern_include)
            else:
                self.pattern_include = None
        except configparser.NoOptionError:
            self.pattern_include = None

        # Do we need to exclude hosts that match a pattern?
        try:
            pattern_exclude = config.get('ec2', 'pattern_exclude');
            if pattern_exclude and len(pattern_exclude) > 0:
                self.pattern_exclude = re.compile(pattern_exclude)
            else:
                self.pattern_exclude = None
        except configparser.NoOptionError:
            self.pattern_exclude = None

        # Instance filters (see boto and EC2 API docs). Ignore invalid filters.
        self.ec2_instance_filters = defaultdict(list)
        if config.has_option('ec2', 'instance_filters'):
            for instance_filter in config.get('ec2', 'instance_filters', '').split(','):
                instance_filter = instance_filter.strip()
                if not instance_filter or '=' not in instance_filter:
                    continue
                filter_key, filter_value = [x.strip() for x in instance_filter.split('=', 1)]
                if not filter_key:
                    continue
                self.ec2_instance_filters[filter_key].append(filter_value)