Setup redis and tldextract
436299
def setup(self, settings):
        '''
        
        '''
        self.extract = tldextract.TLDExtract()
        self.redis_conn = redis.Redis(host=settings['REDIS_HOST'],
                                      port=settings['REDIS_PORT'],
                                      db=settings.get('REDIS_DB'))

        try:
            self.redis_conn.info()
            self.logger.debug("Connected to Redis in ZookeeperHandler")
        except ConnectionError:
            self.logger.error("Failed to connect to Redis in ZookeeperHandler")
            # plugin is essential to functionality
            sys.exit(1)
1099265
function connectToRedis(db, opts) {

    // get the redis client
    var client = redis.createClient(opts.port, opts.host);

    // log any errors
    client.on('error', function (err) {
        redisOffline = true;
        console.log('connectToRedis error: ' + err);
    });

    client.on('ready', function () {
        redisOffline = false;
    });

    // if there is a password, do auth
    var deferred = Q.defer();
    if (opts.password) {
        client.auth(opts.password, function (authErr) {
            if (authErr) {
                deferred.reject(authErr);
                return;
            }

            client.select(db, function (err) {
                err ? deferred.reject(err) : deferred.resolve(client);
            });
        });
    }
    else {
        client.select(db, function (err) {
            err ? deferred.reject(err) : deferred.resolve(client);
        });
    }

    return deferred.promise;
}
568127
def main(args=None):
    """"""
    # example './cli.py install redis --app-name tambeta'
    parser = get_cli_parser()
    if args is None:
        args = sys.argv[1:]
    args = parser.parse_args(args)

    # set the credential in the environment
    if args.webfaction_user:
        os.environ['WEBFACTION_USER'] = args.webfaction_user
    if args.webfaction_pass:
        os.environ['WEBFACTION_PASS'] = args.webfaction_pass
    if args.webfaction_host:
        os.environ['WEBFACTION_HOST'] = args.webfaction_host

    if args.action == "install":
        if args.name == 'redis':
            if not args.app_name:
                print("Install Redis requires app name")
                sys.exit(1)
        else:
            print("Unknown install %s" % args.name)
    elif args.action == "secure":
        print("Converting websites in the domain %s to HTTPS" % (args.name or '*'))
        w = WebfactionWebsiteToSsl(force=args.force)
        w.secure(domain=args.name)
    elif args.action == "renew":
        domain = args.name
        if not domain:
            print("Renew certificates for all the domains")
        else:
            print("Renew certificates for %s" % domain)
        w = WebfactionWebsiteToSsl(force=args.force)
        w.sync_certificates(subdomains=args.name)
1158569
def _connect(host=None, port=None, db=None, password=None):
    '''
    
    '''
    if not host:
        host = __salt__['config.option']('redis.host')
    if not port:
        port = __salt__['config.option']('redis.port')
    if not db:
        db = __salt__['config.option']('redis.db')
    if not password:
        password = __salt__['config.option']('redis.password')

    return redis.StrictRedis(host, port, db, password, decode_responses=True)
273581
def parse_with_toml(data):
    """"""
    try:
        return toml.loads("key={}".format(data), DynaBox).key
    except toml.TomlDecodeError:
        return data
628783
def load_data(fname):
    
    print('Loading ' + fname + ' to redis')
    r = redis.StrictRedis(host = '127.0.0.1', port = 6379, db = 0);
    with open(fname, 'r') as f:
        for line_num, row in enumerate(f): 
            if row.strip('') != '':
                if line_num < 100000000:
                    l_key, l_val = parse_n3(row, 'csv')
                    if line_num % 1000 == 0: 
                        print('loading line #', line_num, 'key=', l_key, ' = ', l_val)
                    if l_key != '':
                        r.set(l_key, l_val)
307747
def _execute_lua(self, keys, args, client):
        
        lua, lua_globals = Script._import_lua(self.load_dependencies)
        lua_globals.KEYS = self._python_to_lua(keys)
        lua_globals.ARGV = self._python_to_lua(args)

        def _call(*call_args):
            # redis-py and native redis commands are mostly compatible argument
            # wise, but some exceptions need to be handled here:
            if str(call_args[0]).lower() == 'lrem':
                response = client.call(
                    call_args[0], call_args[1],
                    call_args[3],  # "count", default is 0
                    call_args[2])
            else:
                response = client.call(*call_args)
            return self._python_to_lua(response)

        lua_globals.redis = {"call": _call}
        return self._lua_to_python(lua.execute(self.script), return_status=True)
115979
def copy_tcltk(src, dest, symlink):
    
    if majver == 2:
        libver = '8.5'
    else:
        libver = '8.6'
    for name in ['tcl', 'tk']:
        srcdir = src + '/tcl/' + name + libver
        dstdir = dest + '/tcl/' + name + libver
        copyfileordir(srcdir, dstdir, symlink)
1131008
def redis_client(address=None, connection_pool=None, timeout=None,
                 parser=None, **kwargs):
    '''
    '''
    if not connection_pool:
        if timeout == 0:
            if not async:
                raise ImportError('Asynchronous connection requires async '
                                  'bindings installed.')
            return async.pool.redis(address, **kwargs)
        else:
            kwargs['socket_timeout'] = timeout
            return Redis(address[0], address[1], **kwargs)
    else:
        return Redis(connection_pool=connection_pool)
1158971
def run(self, conn, tmp, module_name, module_args, inject):
        '''  '''

        # load up options
        options = utils.parse_kv(module_args)
        source  = options.get('src', None)
        dest    = options.get('dest', None)

        if dest.endswith("/"):
            base = os.path.basename(source)
            dest = os.path.join(dest, base)

        if (source is None and not 'first_available_file' in inject) or dest is None:
            result=dict(failed=True, msg="src and dest are required")
            return ReturnData(conn=conn, result=result)

        # if we have first_available_file in our vars
        # look up the files and use the first one we find as src
        if 'first_available_file' in inject:
            found = False
            for fn in inject.get('first_available_file'):
                fn = utils.template(self.runner.basedir, fn, inject)
                fn = utils.path_dwim(self.runner.basedir, fn)
                if os.path.exists(fn):
                    source = fn
                    found = True
                    break
            if not found:
                results=dict(failed=True, msg="could not find src in first_available_file list")
                return ReturnData(conn=conn, result=results)
        else:
            source = utils.template(self.runner.basedir, source, inject)
            source = utils.path_dwim(self.runner.basedir, source)

        local_md5 = utils.md5(source)
        if local_md5 is None:
            result=dict(failed=True, msg="could not find src=%s" % source)
            return ReturnData(conn=conn, result=result)

        remote_md5 = self.runner._remote_md5(conn, tmp, dest)

        exec_rc = None
        if local_md5 != remote_md5:
            # transfer the file to a remote tmp location
            tmp_src = tmp + os.path.basename(source)
            conn.put_file(source, tmp_src)
            # fix file permissions when the copy is done as a different user
            if self.runner.sudo and self.runner.sudo_user != 'root':
                self.runner._low_level_exec_command(conn, "chmod a+r %s" % tmp_src, tmp)

            # run the copy module
            module_args = "%s src=%s" % (module_args, tmp_src)
            return self.runner._execute_module(conn, tmp, 'copy', module_args, inject=inject)

        else:
            # no need to transfer the file, already correct md5, but still need to call
            # the file module in case we want to change attributes

            tmp_src = tmp + os.path.basename(source)
            module_args = "%s src=%s" % (module_args, tmp_src)
            return self.runner._execute_module(conn, tmp, 'file', module_args, inject=inject)
550197
def main():
    from IPython import embed
    

    """ Python 3.5.3 (pypy3)
    libttl 2.387338638305664
    ttl 1.3430471420288086
    diff lt - ttl 1.0442914962768555
    librdfxml 24.70371127128601
    rdfxml 17.85916304588318
    diff lr - rl 6.844548225402832
    simple time 18.32300615310669
    """

    # well I guess that answers that question ...
    # librdf much faster for cpython, not for pypy3

    from time import time
    rdflib.plugin.register('librdfxml', rdflib.parser.Parser,
                        'librdflib', 'libRdfxmlParser')
    rdflib.plugin.register('libttl', rdflib.parser.Parser,
                        'librdflib', 'libTurtleParser')

    p1 = Path('~/git/NIF-Ontology/ttl/NIF-Molecule.ttl').expanduser()
    start = time()
    graph = rdflib.Graph().parse(p1.as_posix(), format='libttl')
    stop = time()
    lttime = stop - start
    print('libttl', lttime)
    #serialize(graph)

    start = time()
    graph = rdflib.Graph().parse(p1.as_posix(), format='turtle')
    stop = time()
    ttltime = stop - start
    print('ttl', ttltime)
    print('diff lt - ttl', lttime - ttltime)

    p2 = Path('~/git/NIF-Ontology/ttl/external/uberon.owl').expanduser()
    start = time()
    graph2 = rdflib.Graph().parse(p2.as_posix(), format='librdfxml')
    stop = time()
    lrtime = stop - start
    print('librdfxml', lrtime)
    if True:
        start = time()
        graph2 = rdflib.Graph().parse(p2.as_posix(), format='xml')
        stop = time()
        rltime = stop - start
        print('rdfxml', rltime)
        print('diff lr - rl', lrtime - rltime)

    if True:
        file_uri = p2.as_uri()
        parser = RDF.Parser(name='rdfxml')
        stream = parser.parse_as_stream(file_uri)
        start = time()
        # t = list(stream)
        t = tuple(statement_to_tuple(statement) for statement in stream)
        stop = time()
        stime = stop - start
        print('simple time', stime)

    embed()
712761
function initRedis() {
    return when.promise((resolve, reject) => {
      redis.get(name, (err, res) => {
        if(err) {
          memStore = {};
        } else if(res) {
          memStore = JSON.parse(res);
        } else {
          memStore = {};
        }
        resolve(true);
      });
    });
  }
1229104
def run(self, conn, tmp, module_name, module_args, inject):
        '''  '''

        tokens  = shlex.split(module_args)
        source  = tokens[0]
        # FIXME: error handling
        args    = " ".join(tokens[1:])
        source  = utils.template(self.runner.basedir, source, inject)
        source  = utils.path_dwim(self.runner.basedir, source)

        # transfer the file to a remote tmp location
        source  = source.replace('\x00','') # why does this happen here?
        args    = args.replace('\x00','') # why does this happen here?
        tmp_src = os.path.join(tmp, os.path.basename(source))
        tmp_src = tmp_src.replace('\x00', '') 

        conn.put_file(source, tmp_src)

        # fix file permissions when the copy is done as a different user
        if self.runner.sudo and self.runner.sudo_user != 'root':
            prepcmd = 'chmod a+rx %s' % tmp_src
        else:
            prepcmd = 'chmod +x %s' % tmp_src

        # add preparation steps to one ssh roundtrip executing the script
        module_args = prepcmd + '; ' + tmp_src + ' ' + args

        handler = utils.plugins.action_loader.get('raw', self.runner)
        result = handler.run(conn, tmp, 'raw', module_args, inject)

        # clean up after
        if tmp.find("tmp") != -1 and C.DEFAULT_KEEP_REMOTE_FILES != '1':
            self.runner._low_level_exec_command(conn, 'rm -rf %s >/dev/null 2>&1' % tmp, tmp)

        return result
355320
def renamenx(self, src, dst):
        ""
        with self.pipe as pipe:
            return pipe.renamenx(self.redis_key(src), self.redis_key(dst))
956372
function parseRawLines(rawText, extendedSyntax, parsedRawLinesCallback) {
        var entries = rawText.replace(reBlank, '').split(/[\r\n]+/);
        var currentLang = '*';
        var genericLang = lang.split('-', 1)[0];
        var skipLang = false;
        var match = '';

        function nextEntry() {
          // Use infinite loop instead of recursion to avoid reaching the
          // maximum recursion limit for content with many lines.
          while (true) {
            if (!entries.length) {
              parsedRawLinesCallback();
              return;
            }
            var line = entries.shift();

            // comment or blank line?
            if (reComment.test(line))
              continue;

            // the extended syntax supports [lang] sections and @import rules
            if (extendedSyntax) {
              match = reSection.exec(line);
              if (match) { // section start?
                // RFC 4646, section 4.4, "All comparisons MUST be performed
                // in a case-insensitive manner."

                currentLang = match[1].toLowerCase();
                skipLang = (currentLang !== '*') &&
                    (currentLang !== lang) && (currentLang !== genericLang);
                continue;
              } else if (skipLang) {
                continue;
              }
              match = reImport.exec(line);
              if (match) { // @import rule?
                loadImport(baseURL + match[1], nextEntry);
                return;
              }
            }

            // key-value pair
            var tmp = line.match(reSplit);
            if (tmp && tmp.length == 3) {
              dictionary[tmp[1]] = evalString(tmp[2]);
            }
          }
        }
        nextEntry();
      }
263115
def main(arguments=None):
    
    # setup the command-line util settings
    su = tools(
        arguments=arguments,
        docString=__doc__,
        logLevel="DEBUG",
        options_first=False,
        projectName="tastic"
    )
    arguments, settings, log, dbConn = su.setup()

    # unpack remaining cl arguments using `exec` to setup the variable names
    # automatically
    for arg, val in arguments.iteritems():
        if arg[0] == "-":
            varname = arg.replace("-", "") + "Flag"
        else:
            varname = arg.replace("<", "").replace(">", "")
        if varname == "import":
            varname = "iimport"
        if isinstance(val, str) or isinstance(val, unicode):
            exec(varname + " = '%s'" % (val,))
        else:
            exec(varname + " = %s" % (val,))
        if arg == "--dbConn":
            dbConn = val
        log.debug('%s = %s' % (varname, val,))

    ## START LOGGING ##
    startTime = times.get_now_sql_datetime()
    log.info(
        '--- STARTING TO RUN THE cl_utils.py AT %s' %
        (startTime,))

    if init:
        from os.path import expanduser
        home = expanduser("~")
        filepath = home + "/.config/tastic/tastic.yaml"
        try:
            cmd = """open %(filepath)s""" % locals()
            p = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True)
        except:
            pass
        try:
            cmd = """start %(filepath)s""" % locals()
            p = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True)
        except:
            pass

    # CALL FUNCTIONS/OBJECTS
    if sort or archive:

        ws = workspace(
            log=log,
            settings=settings,
            fileOrWorkspacePath=pathToFileOrWorkspace
        )
    if sort:
        ws.sort()
    if archive:
        ws.archive_done()

    if sync:
        tp = syncc(
            log=log,
            settings=settings,
            workspaceRoot=pathToWorkspace,
            workspaceName=workspaceName,
            syncFolder=pathToSyncFolder,
            editorialRootPath=editorialRootPath,
            includeFileTags=fileTagsFlag
        )
        tp.sync()

    if reminders:
        r = reminderss(
            log=log,
            settings=settings
        )
        r.import_list(
            listName=listName,
            pathToTaskpaperDoc=pathToTaskpaperDoc
        )

    if "dbConn" in locals() and dbConn:
        dbConn.commit()
        dbConn.close()
    ## FINISH LOGGING ##
    endTime = times.get_now_sql_datetime()
    runningTime = times.calculate_time_difference(startTime, endTime)
    log.info('-- FINISHED ATTEMPT TO RUN THE cl_utils.py AT %s (RUNTIME: %s) --' %
             (endTime, runningTime, ))

    return
89262
def runctx(self, cmd, globals, locals):
        """"""
        with self():
            exec(cmd, globals, locals)
        return self