PATHS: paths to individual .csv files or to directories containing .csvs

    DBNAME: name of the SQLite database file to create
976664
def cli(
    paths,
    dbname,
    separator,
    quoting,
    skip_errors,
    replace_tables,
    table,
    extract_column,
    date,
    datetime,
    datetime_format,
    primary_key,
    fts,
    index,
    shape,
    filename_column,
    no_index_fks,
    no_fulltext_fks,
):
    
    # make plural for more readable code:
    extract_columns = extract_column
    del extract_column

    if extract_columns:
        click.echo("extract_columns={}".format(extract_columns))
    if dbname.endswith(".csv"):
        raise click.BadParameter("dbname must not end with .csv")
    if "." not in dbname:
        dbname += ".db"

    db_existed = os.path.exists(dbname)

    conn = sqlite3.connect(dbname)

    dataframes = []
    csvs = csvs_from_paths(paths)
    sql_type_overrides = None
    for name, path in csvs.items():
        try:
            df = load_csv(path, separator, skip_errors, quoting, shape)
            df.table_name = table or name
            if filename_column:
                df[filename_column] = name
                if shape:
                    shape += ",{}".format(filename_column)
            sql_type_overrides = apply_shape(df, shape)
            apply_dates_and_datetimes(df, date, datetime, datetime_format)
            dataframes.append(df)
        except LoadCsvError as e:
            click.echo("Could not load {}: {}".format(path, e), err=True)

    click.echo("Loaded {} dataframes".format(len(dataframes)))

    # Use extract_columns to build a column:(table,label) dictionary
    foreign_keys = {}
    for col in extract_columns:
        bits = col.split(":")
        if len(bits) == 3:
            foreign_keys[bits[0]] = (bits[1], bits[2])
        elif len(bits) == 2:
            foreign_keys[bits[0]] = (bits[1], "value")
        else:
            foreign_keys[bits[0]] = (bits[0], "value")

    # Now we have loaded the dataframes, we can refactor them
    created_tables = {}
    refactored = refactor_dataframes(
        conn, dataframes, foreign_keys, not no_fulltext_fks
    )
    for df in refactored:
        # This is a bit trickier because we need to
        # create the table with extra SQL for foreign keys
        if replace_tables and table_exists(conn, df.table_name):
            drop_table(conn, df.table_name)
        if table_exists(conn, df.table_name):
            df.to_sql(df.table_name, conn, if_exists="append", index=False)
        else:
            to_sql_with_foreign_keys(
                conn,
                df,
                df.table_name,
                foreign_keys,
                sql_type_overrides,
                primary_keys=primary_key,
                index_fks=not no_index_fks,
            )
            created_tables[df.table_name] = df
        if index:
            for index_defn in index:
                add_index(conn, df.table_name, index_defn)

    # Create FTS tables
    if fts:
        fts_version = best_fts_version()
        if not fts_version:
            conn.close()
            raise click.BadParameter(
                "Your SQLite version does not support any variant of FTS"
            )
        # Check that columns make sense
        for table, df in created_tables.items():
            for fts_column in fts:
                if fts_column not in df.columns:
                    raise click.BadParameter(
                        'FTS column "{}" does not exist'.format(fts_column)
                    )

        generate_and_populate_fts(conn, created_tables.keys(), fts, foreign_keys)

    conn.close()

    if db_existed:
        click.echo(
            "Added {} CSV file{} to {}".format(
                len(csvs), "" if len(csvs) == 1 else "s", dbname
            )
        )
    else:
        click.echo(
            "Created {} from {} CSV file{}".format(
                dbname, len(csvs), "" if len(csvs) == 1 else "s"
            )
        )
150807
def _GetDirectory(self):
    
    if self.entry_type != definitions.FILE_ENTRY_TYPE_DIRECTORY:
      return None
    return SQLiteBlobDirectory(self._file_system, self.path_spec)
19644
def get_file(db_folder, file_name):
    """"""
    if not os.path.isdir(db_folder):
        return
    file_name = file_name.lower().strip()
    for cand_name in os.listdir(db_folder):
        if cand_name.lower().strip() == file_name:
            return os.path.join(db_folder, cand_name)
784543
def _get_result_paths(self, data):
        

        result = {}

        result['Output'] = ResultPath(
            Path=self.Parameters['--output'].Value,
            IsWritten=self.Parameters['--output'].isOn())

        result['ClusterFile'] = ResultPath(
            Path=self.Parameters['--uc'].Value,
            IsWritten=self.Parameters['--uc'].isOn())

        # uchime 3-way global alignments
        result['Output_aln'] = ResultPath(
            Path=self.Parameters['--uchimealns'].Value,
            IsWritten=self.Parameters['--uchimealns'].isOn())

        # uchime tab-separated format
        result['Output_tabular'] = ResultPath(
            Path=self.Parameters['--uchimeout'].Value,
            IsWritten=self.Parameters['--uchimeout'].isOn())

        # chimeras fasta file output
        result['Output_chimeras'] = ResultPath(
            Path=self.Parameters['--chimeras'].Value,
            IsWritten=self.Parameters['--chimeras'].isOn())

        # nonchimeras fasta file output
        result['Output_nonchimeras'] = ResultPath(
            Path=self.Parameters['--nonchimeras'].Value,
            IsWritten=self.Parameters['--nonchimeras'].isOn())

        # log file
        result['LogFile'] = ResultPath(
            Path=self.Parameters['--log'].Value,
            IsWritten=self.Parameters['--log'].isOn())

        return result
219842
@InterfaceAudience.Private
    private static String nameOfDatabaseAtPath(String path) {
        String name = FileDirUtils.getDatabaseNameFromPath(path);
        return isWindows() ? name.replace('/', '.') : name.replace('/', ':');
    }
1057391
def build_db():
    
    if not os.path.exists(MISC_PATH):
        os.makedirs(MISC_PATH)
    cities_path = os.path.join(MISC_PATH, "cities1000.txt")
    cities_msgpack = os.path.join(MISC_PATH, "cities1000.bin")
    if not os.path.isfile(cities_path):
        download()
    if not os.path.isfile(cities_msgpack):
        with open(cities_path) as f:
            lines = f.readlines()

        db = []
        for i, line in enumerate(lines):
            l = line.strip().split('\t')
            db.append({'latitude': float(l[4]),
                       'longitude': float(l[5]),
                       'linenr': i})
        packed = msgpack.packb(db, use_bin_type=True)
        with open(cities_msgpack, 'wb') as f:
            f.write(packed)
    else:
        with open(cities_msgpack, 'rb') as f:
            content = f.read()
        db = msgpack.unpackb(content)
    return db
1098230
def db_url_from_hass_config(path):
    """"""
    config = load_hass_config(path)
    default_path = os.path.join(path, "home-assistant_v2.db")
    default_url = "sqlite:///{}".format(default_path)

    recorder = config.get("recorder")

    if recorder:
        db_url = recorder.get("db_url")
        if db_url is not None:
            return db_url

    if not os.path.isfile(default_path):
        raise ValueError(
            "Unable to determine DB url from hass config at {}".format(path)
        )

    return default_url
1130510
function SQLiteHelper (schemaPath, dbPath) {

    let dbFilePath = dbPath,
        dbDirPath = path.dirname(dbFilePath),
        sql;

    /**
     * Saves a sql database to the disk.
     *
     * @param sql {Database}
     */
    function saveDB (sql) {
        let data = sql.export();
        let buffer = new Buffer(data);

        mkdirp.sync(dbDirPath, '0755');
        fs.writeFileSync(dbFilePath, buffer);
    }

    /**
     * Prefixes object params with ':'.
     * If something other than an object is given the will be returned without being changed.
     *
     * @param params {[]|{}|string}
     * @returns {*} returns the updated params object
     */
    function normalizeParams(params) {
        if (params && params.constructor !== Array && typeof params === 'object') {
            params = _.mapKeys(params, function (value, key) {
                return ':' + _.trimStart(key, ':');
            });
        }
        return params;
    }

    /**
     * Executes a command returning the results.
     *
     * @param query {string} the query string
     * @param params {[]|{}} the parameters to be bound to the query
     * @returns {[]} an array of rows
     */
    function query (query, params) {
        let rows = [];
        let stmt = sql.prepare(query);
        params = normalizeParams(params);
        stmt.bind(params);
        while(stmt.step()) {
            rows.push(stmt.getAsObject());
        }
        stmt.free();
        return rows;
    }

    /**
     * Executes a command ignoring the results.
     *
     * @param query {string} the run string
     * @param params {[]|{}} the parameters to be bound to the query
     */
    function run(query, params) {
        let stmt = sql.prepare(query);
        params = normalizeParams(params);
        stmt.run(params);
        stmt.free();
    }

    if (!fs.existsSync(dbFilePath)) {
        let schema = fs.readFileSync(schemaPath);

        sql = new SQL.Database();
        sql.run(schema);

        saveDB(sql);
    } else {
        let buffer = fs.readFileSync(dbFilePath);

        sql = new SQL.Database(buffer);
    }
    
    return {query: query, save: saveDB.bind(null, sql), run: run};
}
956171
def downloadDatabases(self, savepath=None, unpack=False):
        
        url = self.url('/diagnostics/databases')
        filepath = utils.download(url, self._token, None, savepath, self._session, unpack=unpack)
        return filepath
283639
def path(args):
  """"""

  from .query import Database
  db = Database()

  output = sys.stdout
  if args.selftest:
    from bob.db.utils import null
    output = null()

  r = db.paths(args.id, prefix=args.directory, suffix=args.extension)
  for path in r: output.write('%s\n' % path)

  if not r: return 1

  return 0
889403
def csv_to_sqlite(
    input_filename,
    output_filename,
    samples=None,
    dialect=None,
    batch_size=10000,
    encoding="utf-8",
    callback=None,
    force_types=None,
    chunk_size=8388608,
    table_name="table1",
    schema=None,
):
    ""

    # TODO: automatically detect encoding if encoding == `None`
    # TODO: should be able to specify fields
    # TODO: if table_name is "2019" the final name will be "field_2019" - must
    #       be "table_2019"
    # TODO: if schema is provided and the names are in uppercase, this function
    #       will fail

    if dialect is None:  # Get a sample to detect dialect
        fobj = open_compressed(input_filename, mode="rb")
        sample = fobj.read(chunk_size)
        dialect = rows.plugins.csv.discover_dialect(sample, encoding=encoding)
    elif isinstance(dialect, six.text_type):
        dialect = csv.get_dialect(dialect)

    if schema is None:  # Identify data types
        fobj = open_compressed(input_filename, encoding=encoding)
        data = list(islice(csv.DictReader(fobj, dialect=dialect), samples))
        schema = rows.import_from_dicts(data).fields
        if force_types is not None:
            schema.update(force_types)

    # Create lazy table object to be converted
    # TODO: this lazyness feature will be incorported into the library soon so
    #       we can call here `rows.import_from_csv` instead of `csv.reader`.
    reader = csv.reader(
        open_compressed(input_filename, encoding=encoding), dialect=dialect
    )
    header = make_header(next(reader))  # skip header
    table = rows.Table(fields=OrderedDict([(field, schema[field]) for field in header]))
    table._rows = reader

    # Export to SQLite
    return rows.export_to_sqlite(
        table,
        output_filename,
        table_name=table_name,
        batch_size=batch_size,
        callback=callback,
    )
392201
def deploy_db(rollback=False):
    
    if not rollback:

        if env.DEFAULT_DATABASE_ENGINE=='django.db.backends.sqlite3':
            db_dir = '/'.join([deployment_root(),'database'])
            db_name = ''.join([env.project_name,'_','site_1','.db'])
            dest_db_path = '/'.join([db_dir,db_name])
            if exists(dest_db_path): return
            if env.verbosity:
                print env.host,"DEPLOYING DEFAULT SQLITE DATABASE"
            if not env.DEFAULT_DATABASE_NAME:
                print "ERROR: A database name has not been defined in your Django settings file"
                sys.exit(1)

            if env.DEFAULT_DATABASE_NAME[0] not in [os.path.sep,'.']: #relative path
                db_path = os.path.join(os.getcwd(),env.project_package_name,env.DEFAULT_DATABASE_NAME)

            elif env.DEFAULT_DATABASE_NAME[:2] == '..':
                print "ERROR: Use a full expanded path to the database in your Django settings"
                sys.exit(1)
            else:
                db_path = env.DEFAULT_DATABASE_NAME

            if not db_path or not os.path.exists(db_path):
                print "ERROR: the database %s does not exist. \nRun python manage.py syncdb to create your database locally first, or check your settings."% db_path
                sys.exit(1)

            db_name = os.path.split(db_path)[1]  
            run('mkdir -p '+db_dir)
            put(db_path,dest_db_path)
            #directory and file must be writable by webserver
            sudo("chown -R %s:www-data %s"% (env.user,db_dir))
            sudo("chmod -R ug+w %s"% db_dir)
        
        elif env.DEFAULT_DATABASE_ENGINE=='django.db.backends.':
            print "ERROR: The default database engine has not been defined in your Django settings file"
            print "At a minimum you must define an sqlite3 database for woven to deploy, or define a database that is managed outside of woven."
            sys.exit(1)
    elif rollback and env.DEFAULT_DATABASE_ENGINE=='django.db.backends.sqlite3':
        if env.INTERACTIVE:
            delete = confirm('DELETE the database on the host?',default=False)
            if delete:
                run('rm -f '+db_name)
    return
58965
def CreateStorageReaderForFile(cls, path):
    
    if sqlite_file.SQLiteStorageFile.CheckSupportedFormat(
        path, check_readable_only=True):
      return sqlite_reader.SQLiteStorageFileReader(path)

    return None
920506
def _get_result_paths(self, data):
        
        # access data through self.Parameters so we know it's been cast
        # to a FilePath
        wd = self.WorkingDir
        db_name = self.Parameters['-n'].Value
        log_name = self.Parameters['-l'].Value
        result = {}
        result['log'] = ResultPath(Path=wd + log_name, IsWritten=True)
        if self.Parameters['-p'].Value == 'F':
            extensions = ['nhr', 'nin', 'nsq', 'nsd', 'nsi']
        else:
            extensions = ['phr', 'pin', 'psq', 'psd', 'psi']
        for extension in extensions:
            for file_path in glob(wd + (db_name + '*' + extension)):
                # this will match e.g. nr.01.psd and nr.psd
                key = file_path.split(db_name + '.')[1]
                result_path = ResultPath(Path=file_path, IsWritten=True)
                result[key] = result_path
        return result
200917
def DemoDB(keys_per_column=None, **kwargs):
    
    _ROOT = os.path.abspath(os.path.dirname(__file__))
    chinook = os.path.join(_ROOT, 'data', 'chinook.sqlite')
    return DB(filename=chinook, dbtype='sqlite', keys_per_column=keys_per_column, **kwargs)
1362575
def create_results_dirs(base_path):
    

    if not os.path.exists(base_path):
        print("Creating directory {} for results data.".format(base_path))
        os.mkdir(base_path)
    if not os.path.exists(os.path.join(base_path, 'results')):
        os.mkdir(os.path.join(base_path, 'results'))
    if not os.path.exists(os.path.join(base_path, 'plots')):
        os.mkdir(os.path.join(base_path, 'plots'))
    if not os.path.exists(os.path.join(base_path, 'info')):
        os.mkdir(os.path.join(base_path, 'info'))
    if not os.path.exists(os.path.join(base_path, 'log')):
        os.mkdir(os.path.join(base_path, 'log'))
1307054
def pkgPath(root, path, rpath="/"):
    
    global data_files
    if not os.path.exists(path):
        return
    files = []
    for spath in os.listdir(path):
        # Ignore test directories
        if spath == 'test':
            continue
        subpath = os.path.join(path, spath)
        spath = os.path.join(rpath, spath)
        if os.path.isfile(subpath):
            files.append(subpath)
        if os.path.isdir(subpath):
            pkgPath(root, subpath, spath)
    data_files.append((root + rpath, files))