Write out the measurement tables found in paleoData and chronData
        :return:
425131
def __write_data(self, idx):
        
        pair = self.noaa_data_sorted["Data"][idx]
        # Run once for each pair (paleo+chron) of tables that was gathered earlier.
        # for idx, pair in enumerate(self.noaa_data_sorted["Data"]):
        lst_pc = ["chron", "paleo"]
        # loop once for paleo, once for chron
        for pc in lst_pc:
            # safeguard in case the table is an empty set.
            table = pair[pc]
            if pc == "paleo":
                self.__write_variables_1(table)
                self.__write_divider()
                self.__write_columns(pc, table)
            elif pc == "chron":
                # self.__write_variables_1(table)
                self.__write_columns(pc, table)
                self.__write_divider(nl=False)
        return
804520
def get_csv_from_metadata(dsn, d):
    
    logger_csvs.info("enter get_csv_from_metadata")
    _csvs = OrderedDict()
    _d = copy.deepcopy(d)

    try:
        if "paleoData" in _d:
            # Process paleoData section
            _d["paleoData"], _csvs = _get_csv_from_section(_d["paleoData"], "{}.paleo".format(dsn), _csvs)

        if "chronData" in _d:
            _d["chronData"], _csvs = _get_csv_from_section(_d["chronData"], "{}.chron".format(dsn), _csvs)

    except Exception as e:
        print("Error: get_csv_from_metadata: {}, {}".format(dsn, e))
        logger_csvs.error("get_csv_from_metadata: {}, {}".format(dsn, e))

    logger_csvs.info("exit get_csv_from_metadata")
    return _d, _csvs
55012
def _export_section(sections, pc):
    
    logger_jsons.info("enter export_data: {}".format(pc))
    _sections = []

    for name, section in sections.items():

        # Process chron models
        if "model" in section:
            section["model"] = _export_model(section["model"])

        # Process the chron measurement table
        if "measurementTable" in section:
            section["measurementTable"] = _idx_table_by_num(section["measurementTable"])

        # Add only the table to the output list
        _sections.append(section)

    logger_jsons.info("exit export_data: {}".format(pc))
    return _sections
1296267
def write_points_with_precision(self, data, time_precision='s'):
        
        warnings.warn(
            "write_points_with_precision is deprecated, and will be removed "
            "in future versions. Please use "
            "``DataFrameClient.write_points(time_precision='..')`` instead.",
            FutureWarning)
        return self.write_points(data, time_precision='s')
223003
def range_metadata(start, end, dst_folder, num_worker_threads=0, writers=[file_writer], geometry_check=None):
    

    assert isinstance(start, date)
    assert isinstance(end, date)

    delta = end - start

    dates = []

    for i in range(delta.days + 1):
        dates.append(start + timedelta(days=i))

    days = len(dates)

    total_counter = {
        'days': days,
        'products': 0,
        'saved_tiles': 0,
        'skipped_tiles': 0,
        'skipped_tiles_paths': []
    }

    def update_counter(counter):
        for key in iterkeys(total_counter):
            if key in counter:
                total_counter[key] += counter[key]

    for d in dates:
        logger.info('Getting metadata of {0}-{1}-{2}'.format(d.year, d.month, d.day))
        update_counter(daily_metadata(d.year, d.month, d.day, dst_folder, writers, geometry_check,
                                      num_worker_threads))

    return total_counter
853276
def _count_sizes(self):
        
        format_list = []
        header_sepa_format_list = []
        # actual widths of columns
        self.col_widths = {}

        for col in self.col_list:
            col_length = self.col_longest[col]
            col_width = col_length + self._separate()
            # -2 is for implicit separator -- spaces around
            format_list.append(" {%s:%d} " % (col, col_width - 2))
            header_sepa_format_list.append("{%s:%d}" % (col, col_width))
            self.col_widths[col] = col_width

        logger.debug("column widths %s", self.col_widths)

        self.format_str = "|".join(format_list)

        self.header_format_str = "+".join(header_sepa_format_list)
        self.header_data = {}
        for k in self.col_widths:
            self.header_data[k] = "-" * self.col_widths[k]
214615
def write_table(self, d):
        
        self._flushbits()
        table_data = AMQPWriter()
        for k, v in d.items():
            table_data.write_shortstr(k)
            if isinstance(v, basestring):
                if isinstance(v, unicode):
                    v = v.encode('utf-8')
                table_data.write(byte(83)) # 'S'
                table_data.write_longstr(v)
            elif isinstance(v, (int, long)):
                table_data.write(byte(73)) # 'I'
                table_data.write(pack('>i', v))
            elif isinstance(v, Decimal):
                table_data.write(byte(68)) # 'D'
                sign, digits, exponent = v.as_tuple()
                v = 0
                for d in digits:
                    v = (v * 10) + d
                if sign:
                    v = -v
                table_data.write_octet(-exponent)
                table_data.write(pack('>i', v))
            elif isinstance(v, datetime):
                table_data.write(byte(84)) # 'T'
                table_data.write_timestamp(v)
                ## FIXME: timezone ?
            elif isinstance(v, dict):
                table_data.write(byte(70)) # 'F'
                table_data.write_table(v)
            else:
                raise ValueError('%s not serializable in AMQP' % repr(v))
        table_data = table_data.getvalue()
        self.write_long(len(table_data))
        self.out.write(table_data)
718186
def _write_measurements():
    
    global _timeout, _writing

    future = concurrent.Future()

    if _writing:
        LOGGER.warning('Currently writing measurements, skipping write')
        future.set_result(False)
    elif not _pending_measurements():
        future.set_result(True)
    elif not _sample_batch():
        LOGGER.debug('Skipping batch submission due to sampling')
        future.set_result(True)

    # Exit early if there's an error condition
    if future.done():
        return future

    if not _http_client or _dirty:
        _create_http_client()

    # Keep track of the futures for each batch submission
    futures = []

    # Submit a batch for each database
    for database in _measurements:
        url = '{}?db={}&precision=ms'.format(_base_url, database)

        # Get the measurements to submit
        measurements = _measurements[database][:_max_batch_size]

        # Pop them off the stack of pending measurements
        _measurements[database] = _measurements[database][_max_batch_size:]

        # Create the request future
        LOGGER.debug('Submitting %r measurements to %r',
                     len(measurements), url)
        request = _http_client.fetch(
            url, method='POST', body='\n'.join(measurements).encode('utf-8'))

        # Keep track of each request in our future stack
        futures.append((request, str(uuid.uuid4()), database, measurements))

    # Start the wait cycle for all the requests to complete
    _writing = True
    _futures_wait(future, futures)

    return future
158911
void dumpStats() {
        long wallTimeNanos = totalStopwatch.elapsed(TimeUnit.NANOSECONDS);
        long dbtime = 0;
        for (String name : methodCalls.keySet()) {
            long calls = methodCalls.get(name);
            long time = methodTotalTime.get(name);
            dbtime += time;
            long average = time / calls;
            double proportion = (time + 0.0) / (wallTimeNanos + 0.0);
            log.info(name + " c:" + calls + " r:" + time + " a:" + average + " p:" + String.format("%.2f", proportion));
        }
        double dbproportion = (dbtime + 0.0) / (wallTimeNanos + 0.0);
        double hitrate = (hit + 0.0) / (hit + miss + 0.0);
        log.info("Cache size:" + utxoCache.size() + " hit:" + hit + " miss:" + miss + " rate:"
                + String.format("%.2f", hitrate));
        bloom.printStat();
        log.info("hasTxOut call:" + hasCall + " True:" + hasTrue + " False:" + hasFalse);
        log.info("Wall:" + totalStopwatch + " percent:" + String.format("%.2f", dbproportion));
        String stats = db.getProperty("leveldb.stats");
        System.out.println(stats);

    }
962767
def standardize_table(cls, table):
        '''
        
        '''

        print(cls)
        identifiers = {n+'-id':table[n] for n in cls.identifier_keys}

        # create skycoord objects
        N = len(table)
        coordinates = dict(  ra=table['_RAJ2000'].data.data*u.deg,
                             dec=table['_DEJ2000'].data.data*u.deg,
                             pm_ra_cosdec=table['pmRA'].data.data*u.arcsec/u.year,
                             pm_dec=table['pmDE'].data.data*u.arcsec/u.year,
                             radial_velocity=np.nan*np.ones(N)*u.km/u.s,
                             distance=np.nan*np.ones(N)*u.pc,#distance=1000*u.pc/table['parallax'].data, # weirdly, messed with RA + Dec signs if parallax is zero
                             obstime=cls.epoch*np.ones(N)*u.year)#Time(, format='decimalyear'))
        for k in coordinates:
            print(k, coordinates[k])
        magnitudes = {f+'-mag':table[f+'mag'].data for f in cls.filters}

        standardized = hstack([Table(identifiers),
                               Table(coordinates),
                               Table(magnitudes)])
        standardized.meta['catalog'] = 'Gaia'

        return standardized
840111
def submit_influxdb_measurement(self):
        """"""
        measurement = influxdb.Measurement(*self.influxdb)
        measurement.set_timestamp(time.time())
        for key, value in self.measurement.counters.items():
            measurement.set_field(key, value)
        for key, value in self.measurement.tags.items():
            measurement.set_tag(key, value)
        for key, value in self.measurement.values.items():
            measurement.set_field(key, value)

        for key, values in self.measurement.durations.items():
            if len(values) == 1:
                measurement.set_field(key, values[0])
            elif len(values) > 1:
                measurement.set_field('{}-average'.format(key),
                                      sum(values) / len(values))
                measurement.set_field('{}-max'.format(key), max(values))
                measurement.set_field('{}-min'.format(key), min(values))
                measurement.set_field('{}-median'.format(key),
                                      utils.percentile(values, 50))
                measurement.set_field('{}-95th'.format(key),
                                      utils.percentile(values, 95))

        influxdb.add_measurement(measurement)
        LOGGER.debug('InfluxDB Measurement: %r', measurement.marshall())
426281
function buildRegionAndTimeColumns(item, dimensions) {
  var regionDimension = getDimensionById(dimensions, item.regionDimensionId);
  var timePeriodDimension = getDimensionById(
    dimensions,
    item.timePeriodDimensionId
  );
  if (!defined(regionDimension) && !defined(timePeriodDimension)) {
    // No region dimension (with the actual region values in it) AND no time dimension - we're done.
    return [];
  }
  var regionValues = [];
  var timePeriodValues = [];
  var regionCount = defined(regionDimension)
    ? regionDimension.values.length
    : 1;
  var timePeriodCount = defined(timePeriodDimension)
    ? timePeriodDimension.values.length
    : 1;
  for (
    var timePeriodIndex = 0;
    timePeriodIndex < timePeriodCount;
    timePeriodIndex++
  ) {
    for (var regionIndex = 0; regionIndex < regionCount; regionIndex++) {
      if (defined(regionDimension)) {
        regionValues.push(regionDimension.values[regionIndex].id);
      }
      if (defined(timePeriodDimension)) {
        timePeriodValues.push(timePeriodDimension.values[timePeriodIndex].id);
      }
    }
  }
  var timePeriodColumn;
  if (defined(timePeriodDimension)) {
    var thisColumnOptions = clone(item._columnOptions);
    if (timePeriodCount === 1) {
      thisColumnOptions.type = VarType.ENUM; // Don't trigger timeline off a single-valued time dimension.
    }
    timePeriodColumn = new TableColumn(
      "date",
      timePeriodValues,
      thisColumnOptions
    );
  }
  if (!defined(regionDimension)) {
    return [timePeriodColumn];
  }
  // If there are multiple region types in the data, only use the first region type.
  var regionColumnName = getRegionColumnName(item, dimensions, 0);
  var regionColumn = new TableColumn(
    regionColumnName,
    regionValues,
    item._columnOptions
  );
  if (defined(timePeriodDimension) && defined(regionDimension)) {
    return [timePeriodColumn, regionColumn];
  } else {
    return [regionColumn];
  }
}
1289708
public void writeData(int l, Type[] types, Object[] data,
                          HashMappedList cols, int[] primaryKeys) {

        boolean hasPK = primaryKeys != null && primaryKeys.length != 0;
        int     limit = hasPK ? primaryKeys.length
                              : l;

        for (int i = 0; i < limit; i++) {
            int    j = hasPK ? primaryKeys[i]
                             : i;
            Object o = data[j];
            Type   t = types[j];

            if (cols != null) {
                ColumnSchema col = (ColumnSchema) cols.get(j);

                writeFieldPrefix();
                writeString(col.getName().statementName);
            }

            if (o == null) {
                writeNull(t);

                continue;
            }

            writeFieldType(t);

            switch (t.typeCode) {

                case Types.SQL_ALL_TYPES :
                    throw Error.runtimeError(
                        ErrorCode.U_S0500, "RowOutputBase");
                case Types.SQL_CHAR :
                case Types.SQL_VARCHAR :
                case Types.VARCHAR_IGNORECASE :
                    writeChar((String) o, t);
                    break;

                case Types.TINYINT :
                case Types.SQL_SMALLINT :
                    writeSmallint((Number) o);
                    break;

                case Types.SQL_INTEGER :
                    writeInteger((Number) o);
                    break;

                case Types.SQL_BIGINT :
                    writeBigint((Number) o);
                    break;

                case Types.SQL_REAL :
                case Types.SQL_FLOAT :
                case Types.SQL_DOUBLE :
                    writeReal((Double) o);
                    break;

                case Types.SQL_NUMERIC :
                case Types.SQL_DECIMAL :
                    writeDecimal((BigDecimal) o, t);
                    break;

                case Types.SQL_BOOLEAN :
                    writeBoolean((Boolean) o);
                    break;

                case Types.SQL_DATE :
                    writeDate((TimestampData) o, t);
                    break;

                case Types.SQL_TIME :
                case Types.SQL_TIME_WITH_TIME_ZONE :
                    writeTime((TimeData) o, t);
                    break;

                case Types.SQL_TIMESTAMP :
                case Types.SQL_TIMESTAMP_WITH_TIME_ZONE :
                    writeTimestamp((TimestampData) o, t);
                    break;

                case Types.SQL_INTERVAL_YEAR :
                case Types.SQL_INTERVAL_YEAR_TO_MONTH :
                case Types.SQL_INTERVAL_MONTH :
                    writeYearMonthInterval((IntervalMonthData) o, t);
                    break;

                case Types.SQL_INTERVAL_DAY :
                case Types.SQL_INTERVAL_DAY_TO_HOUR :
                case Types.SQL_INTERVAL_DAY_TO_MINUTE :
                case Types.SQL_INTERVAL_DAY_TO_SECOND :
                case Types.SQL_INTERVAL_HOUR :
                case Types.SQL_INTERVAL_HOUR_TO_MINUTE :
                case Types.SQL_INTERVAL_HOUR_TO_SECOND :
                case Types.SQL_INTERVAL_MINUTE :
                case Types.SQL_INTERVAL_MINUTE_TO_SECOND :
                case Types.SQL_INTERVAL_SECOND :
                    writeDaySecondInterval((IntervalSecondData) o, t);
                    break;

                case Types.OTHER :
                    writeOther((JavaObjectData) o);
                    break;

                case Types.SQL_BLOB :
                    writeBlob((BlobData) o, t);
                    break;

                case Types.SQL_CLOB :
                    writeClob((ClobData) o, t);
                    break;

                case Types.SQL_BINARY :
                case Types.SQL_VARBINARY :
                    writeBinary((BinaryData) o);
                    break;

                case Types.SQL_BIT :
                case Types.SQL_BIT_VARYING :
                    writeBit((BinaryData) o);
                    break;

                default :
                    throw Error.runtimeError(
                        ErrorCode.U_S0500,
                        t.getNameString());
            }
        }
    }
273843
def format_stats(self, stats:TensorOrNumList)->None:
        ""
        str_stats = []
        for name,stat in zip(self.names,stats):
            str_stats.append('#na#' if stat is None else str(stat) if isinstance(stat, int) else f'{stat:.6f}')
        if self.add_time: str_stats.append(format_time(time() - self.start_epoch))
        if not self.silent: self.pbar.write(str_stats, table=True)
142484
def reflect_table_data(table, mapping=None, engine_name='default'):
    
    table = reflect_table(table, engine_name)
    mapping = mapping or {}

    from uliweb.utils.sorteddict import SortedDict

    field_type_map = {'VARCHAR':'str', 'VARCHAR2':'str', 'INTEGER':'int', 'FLOAT':'float'}

    meta = {}
    columns = SortedDict()
    #write columns
    _primary_key = None
    for k, v in table.columns.items():
        column_type = v.type
        type_name = column_type.__class__.__name__.lower()
        kwargs = SortedDict()
        field_type = type_name.upper()

        if type_name in ('char', 'varchar'):
            kwargs['max_length'] = column_type.length
        elif type_name in ('text', 'blob', 'integer', 'float', 'bigint'):
            pass
        elif type_name == 'long':
            field_type = 'bigint'
        elif type_name in ('clob',):
            field_type = 'TEXT'
        elif type_name in ('decimal', 'float'):
            kwargs['precision'] = v.type.precision
            kwargs['scale'] = v.type.scale
        elif type_name == 'raw': #oracle
            field_type = 'binary'
            kwargs['max_length'] = column_type.length
        elif type_name == 'number':
            if v.type.scale:
                kwargs['precision'] = v.type.precision
                kwargs['scale'] = v.type.scale
                field_type = 'DECIMAL'
            else:
                field_type = 'int'
        elif type_name == 'numeric':
            field_type = 'DECIMAL'
            kwargs['precision'] = v.type.precision
            kwargs['scale'] = v.type.scale
        elif type_name in ('timestamp',):
            field_type = 'TIMESTAMP'
        elif type_name in ('datetime', 'date', 'time'):
            pass
        #for tinyint will be treated as bool
        elif type_name in ('tinyint', 'boolean'):
            field_type = 'bool'
        else:
            raise ValueError("Don't support column [{0}] for type [{1}] when parsing {2}".format(k, type_name, table.name))

        if v.primary_key:
            kwargs['primary_key'] = True
            _primary_key = k
            if v.autoincrement:
                kwargs['autoincrement'] = True
        if not v.nullable:
            kwargs['nullable'] = False
        if v.server_default:
            server_default = v.server_default.arg
            kwargs['server_default'] = server_default
        if v.index:
            kwargs['index'] = True
        if v.unique:
            kwargs['unique'] = True
        if sa_version >= '1.2' and v.comment:
            kwargs['verbose_name'] = v.comment

        #convert field_type to common python data type
        field_type = field_type_map.get(field_type, field_type)
        columns[k] = field_type, kwargs

    meta['columns'] = columns

    indexes = []
    indexes_names = []
    for index in table.indexes:
        cols = list(index.columns)
        _len = len(cols)
        #if only one column it'll be set to Property
        if _len == 1:
            column_name = cols[0].name
            d = {'index':True}
            if index.unique:
                d['unique'] = index.unique
            columns[column_name][1].update(d)
        else:
            if not index.name in indexes_names:
                indexes.append({'name':index.name, 'columns':[x.name for x in index.columns],
                            'unique':index.unique})
                indexes_names.append(index.name)

    meta['indexes'] = indexes
    return meta
131546
def show_formats():
    
    fmts = {
        "ann": "Kvis annotation",
        "reg": "DS9 regions file",
        "fits": "FITS Binary Table",
        "csv": "Comma separated values",
        "tab": "tabe separated values",
        "tex": "LaTeX table format",
        "html": "HTML table",
        "vot": "VO-Table",
        "xml": "VO-Table",
        "db": "Sqlite3 database",
        "sqlite": "Sqlite3 database"}
    supported = get_table_formats()
    print("Extension |     Description       | Supported?")
    for k in sorted(fmts.keys()):
        print("{0:10s} {1:24s} {2}".format(k, fmts[k], k in supported))
    return
490112
def main(host='localhost', port=8086, nb_day=15):
    """"""
    nb_day = 15  # number of day to generate time series
    timeinterval_min = 5  # create an event every x minutes
    total_minutes = 1440 * nb_day
    total_records = int(total_minutes / timeinterval_min)
    now = datetime.datetime.today()
    metric = "server_data.cpu_idle"
    series = []

    for i in range(0, total_records):
        past_date = now - datetime.timedelta(minutes=i * timeinterval_min)
        value = random.randint(0, 200)
        hostName = "server-%d" % random.randint(1, 5)
        # pointValues = [int(past_date.strftime('%s')), value, hostName]
        pointValues = {
            "time": int(past_date.strftime('%s')),
            "measurement": metric,
            "fields": {
                "value": value,
            },
            "tags": {
                "hostName": hostName,
            },
        }
        series.append(pointValues)

    print(series)

    client = InfluxDBClient(host, port, USER, PASSWORD, DBNAME)

    print("Create database: " + DBNAME)
    try:
        client.create_database(DBNAME)
    except InfluxDBClientError:
        # Drop and create
        client.drop_database(DBNAME)
        client.create_database(DBNAME)

    print("Create a retention policy")
    retention_policy = 'server_data'
    client.create_retention_policy(retention_policy, '3d', 3, default=True)

    print("Write points #: {0}".format(total_records))
    client.write_points(series, retention_policy=retention_policy)

    time.sleep(2)

    query = "SELECT MEAN(value) FROM {} WHERE \
            time > now() - 10d GROUP BY time(500m)".format(metric)
    result = client.query(query, database=DBNAME)
    print(result)
    print("Result: {0}".format(result))

    print("Drop database: {}".format(DBNAME))
    client.drop_database(DBNAME)