Internal method to filter into the formatter before being passed to the main Python logger
436680
def _log(self, lvl, msg, type, args, kwargs):
        
        extra = kwargs.get('extra', {})
        extra.setdefault("fastlog-type", type)
        extra.setdefault("fastlog-indent", self._indent)
        kwargs['extra'] = extra

        self._lastlevel = lvl

        self.inner.log(lvl, msg, *args, **kwargs)
268561
def describe_filter(f):
    """"""
    if f.__class__ is logging.Filter:  # using type() breaks in Python <= 2.6
        return 'name=%r' % f.name
    return repr(f)
274790
def plot_filter_transmissions(log, filterList):
    
    ################ > IMPORTS ################
    ## STANDARD LIB ##
    ## THIRD PARTY ##
    import matplotlib.pyplot as plt
    import numpy as np
    ## LOCAL APPLICATION ##

    ################ > VARIABLE SETTINGS ######
    ################ >ACTION(S) ################
    for filterFile in filterList:
        data = np.genfromtxt(filterFile)
        plt.plot(data[:, 0], data[:, 1])

    plt.show()
    return
1051248
def get_formatter(self, handler):
        
        if isinstance(handler, logging.handlers.SysLogHandler):
            formatter = '[%(levelname)-9s]'
        else:
            formatter = '[%(asctime)s] [%(levelname)-9s]'

        for p in self.prefix:
            formatter += ' [%s]' % (p)
        formatter = formatter + ' %(message)s'
        return logging.Formatter(formatter)
970714
def filter(self, **kwargs):
        '''
        '''
        # So we don't return anything more than once.
        yielded = set()

        dunder = '__'
        filter_items = set()
        for k, v in kwargs.items():
            if dunder in k:
                k, op = k.split(dunder)
                try:
                    handler = getattr(self, 'handle__%s' % op)
                except AttributeError:
                    msg = '%s has no %r method to handle operator %r.'
                    raise NonExistentHandler(msg % (self, handler, op))
                for dicty in self:
                    if handler(k, v, dicty):
                        dicty_id = id(dicty)
                        if dicty_id not in yielded:
                            yield dicty
                            yielded.add(dicty_id)
            else:
                filter_items.add((k, v))

        for dicty in self:
            dicty_items = set(dicty.items())
            if filter_items.issubset(dicty_items):
                yield dicty
847918
def _get_formatter(fmt):
    
    fmt = _replace_and_pad(fmt, "%(timezone)s", LogManager.spec.timezone)
    return logging.Formatter(fmt)
991434
def with_formatter(formatter):
    
    def _decorator_after_args(unwrapped):
        def _wrapped(self, *args, **kwargs):
            logging.debug('unwrapped: %s', unwrapped)
            logging.debug('self: %s', self)
            logging.debug('args: %s', args)
            logging.debug('kwargs: %s', kwargs)
            return_value = unwrapped(self, *args, **kwargs)
            if 'raw' in kwargs and kwargs['raw']:
                return return_value
            else:
                return formatter(return_value)
        return _wrapped
    return _decorator_after_args
1184727
def _build_text_filter(self):
        
        text_filter = TextFilter(logger=self.logger)
        self.log(u"Created TextFilter object")
        for key, cls, param_name in [
                (
                    gc.PPN_TASK_IS_TEXT_FILE_IGNORE_REGEX,
                    TextFilterIgnoreRegex,
                    "regex"
                ),
                (
                    gc.PPN_TASK_IS_TEXT_FILE_TRANSLITERATE_MAP,
                    TextFilterTransliterate,
                    "map_file_path"
                )
        ]:
            cls_name = cls.__name__
            param_value = gf.safe_get(self.parameters, key, None)
            if param_value is not None:
                self.log([u"Creating %s object...", cls_name])
                params = {
                    param_name: param_value,
                    "logger": self.logger
                }
                try:
                    inner_filter = cls(**params)
                    text_filter.add_filter(inner_filter)
                    self.log([u"Creating %s object... done", cls_name])
                except ValueError as exc:
                    self.log_exc(u"Creating %s object failed" % (cls_name), exc, False, None)
        return text_filter
757370
def init_log(logger, filename=None, loglevel=None):
    
    
    formatter = logging.Formatter(
                    '[%(asctime)s] %(levelname)s: %(name)s: %(message)s'
                )
    
    if loglevel:
        logger.setLevel(getattr(logging, loglevel))

    # We will allways print warnings and higher to stderr
    ch = logging.StreamHandler()
    ch.setLevel('WARNING')
    ch.setFormatter(formatter)
    
    if filename:
        fi = logging.FileHandler(filename, encoding='utf-8')
        if loglevel:
            fi.setLevel(getattr(logging, loglevel))
        fi.setFormatter(formatter)
        logger.addHandler(fi)
    # If no logfile is provided we print all log messages that the user has 
    # defined to stderr
    else:
        if loglevel:
            ch.setLevel(getattr(logging, loglevel))

    logger.addHandler(ch)
1112598
def filter(args):
    
    p = OptionParser(filter.__doc__)
    opts, args = p.parse_args(args)

    if len(args) != 2:
        sys.exit(not p.print_help())

    frgfile, idsfile = args
    assert frgfile.endswith(".frg")

    fp = open(idsfile)
    allowed = set(x.strip() for x in fp)
    logging.debug("A total of {0} allowed ids loaded.".format(len(allowed)))

    newfrgfile = frgfile.replace(".frg", ".filtered.frg")
    fp = open(frgfile)
    fw = open(newfrgfile, "w")

    nfrags, discarded_frags = 0, 0
    nmates, discarded_mates = 0, 0
    for rec in iter_records(fp):
        if rec.type == "FRG":
            readname = rec.get_field("acc")
            readname = readname.rstrip("ab")
            nfrags += 1
            if readname not in allowed:
                discarded_frags += 1
                continue
        if rec.type == "LKG":
            readname = rec.get_field("frg")
            readname = readname.rstrip("ab")
            nmates += 1
            if readname not in allowed:
                discarded_mates += 1
                continue
        print(rec, file=fw)

    # Print out a summary
    survived_frags = nfrags - discarded_frags
    survived_mates = nmates - discarded_mates
    print("Survived fragments: {0}".\
            format(percentage(survived_frags, nfrags)), file=sys.stderr)
    print("Survived mates: {0}".\
            format(percentage(survived_mates, nmates)), file=sys.stderr)
1166514
def pre_filter(self):
        
        if self._name in self.PRE_FILTER_FIELDS:
            return '"{}={},value={}"'.format(
                   self._rt_cmp, self.PRE_FILTER_FIELDS[self._name], int(self._value))
        return ''
909736
def log_formatter(request=None):
    

    if request:
        format_str = ('%(asctime)s {ip} {name}:  ENV={env} '
                      'REMOTE_IP=%(remote_ip)s REQUEST_ID=%(request_id)s '
                      '%(message)s')
    else:
        format_str = '%(asctime)s {ip} {name}:  ENV={env} %(message)s'

    try:
        hostname = socket.gethostname()
    except socket.gaierror:
        hostname = 'localhost'

    try:
        ip = socket.gethostbyname(hostname)
    except socket.gaierror:
        ip = '127.0.0.1'

    formatter = logging.Formatter(
            format_str.format(ip=ip, name=options.name, env=options.env),
            datefmt='%Y-%m-%dT%H:%M:%S')
    logging.Formatter.converter = time.gmtime

    return formatter
459762
def pre_filter(self):
        
        if self._name not in self.PRE_FILTER_FIELDS or self._template:
            return ''
        if not self._value:
            return '"equal={},cat="'.format(self.PRE_FILTER_FIELDS[self._name])

        if self._is_regex:
            needle = self._value[1:-1]
            needle = self.CLEAN_PRE_VAL_RE.sub(' ', needle)
            needle = self.SPLIT_PRE_VAL_RE.split(needle)
        else:
            needle = self.CLEAN_PRE_VAL_RE.sub(' ', self._value)
            needle = self.SPLIT_PRE_GLOB_RE.split(needle)
        needle = list(sorted(needle, key=len))[-1]

        if needle:
            try:
                needle.encode('ascii')
            except UnicodeEncodeError:
                return ''
            else:
                return r'"string.contains_i=${},\"{}\""'.format(
                       self.PRE_FILTER_FIELDS[self._name], needle.replace('"', r'\\\"'))

        return ''
471639
def dump(self, format=None, **kwargs):
        
        formatter = Formatters.get(format, None)
        if not format:
            return self.display()
        elif not formatter:
            raise FormattingException('Formatter %s not available' % format)
        else:
            return formatter(self, **kwargs)
271329
def filter_data(self, min_len, max_len):
        
        logging.info(f'Filtering data, min len: {min_len}, max len: {max_len}')

        initial_len = len(self.src)
        filtered_src = []
        for src in self.src:
            if min_len <= len(src) <= max_len:
                filtered_src.append(src)

        self.src = filtered_src
        filtered_len = len(self.src)
        logging.info(f'Pairs before: {initial_len}, after: {filtered_len}')
979311
def _filter_markdown(source, filters):
    """"""
    lines = source.splitlines()
    # Filters is a list of 'hN' strings where 1 <= N <= 6.
    headers = [_replace_header_filter(filter) for filter in filters]
    lines = [line for line in lines if line.startswith(tuple(headers))]
    return '\n'.join(lines)
816613
def init_logger():
    
    formatter = logging.Formatter('%(levelname)s: %(message)s')
    console = logging.StreamHandler(stream=sys.stdout)
    console.setLevel(logging.INFO)
    console.setFormatter(formatter)
    logger.addHandler(console)