Import data from a stat file.
976097
def _import_data(self):
        
        # set default state to ironpython for very old ironpython (2.7.0)
        iron_python = True
        try:
            iron_python = True if platform.python_implementation() == 'IronPython' \
                else False
        except ValueError as e:
            # older versions of IronPython fail to parse version correctly
            # failed to parse IronPython sys.version: '2.7.5 (IronPython 2.7.5 (2.7.5.0)
            # on .NET 4.0.30319.42000 (64-bit))'
            if 'IronPython' in str(e):
                iron_python = True

        if iron_python:
            statwin = codecs.open(self.file_path, 'r')
        else:
            statwin = codecs.open(self.file_path, 'r', encoding='utf-8', errors='ignore')
        try:
            line = statwin.readline()
            # import header with location
            self._header = [line] + [statwin.readline() for i in xrange(9)]
            self._body = statwin.read()
        except Exception as e:
            import traceback
            raise Exception('{}\n{}'.format(e, traceback.format_exc()))
        else:

            # import location data
            loc_name = self._header[2].strip().replace('Location -- ', '')
            if ' - ' in loc_name:
                city = ' '.join(loc_name.split(' - ')[:-1])
            else:
                # for US stat files it is full name separated by spaces
                city = ' '.join(loc_name.split()[:-2])
            country = loc_name.split(' ')[-1]
            source = self._header[6].strip().replace('Data Source -- ', '')
            station_id = self._header[8].strip().replace('WMO Station ', '')
            if iron_python:
                # IronPython
                coord_pattern = re.compile(r"{([NSEW])(\s*\d*)deg(\s*\d*)")
                matches = coord_pattern.findall(self._header[3].replace('\xb0', 'deg'))
            else:
                # CPython
                coord_pattern = re.compile(r"{([NSEW])(\s*\d*) (\s*\d*)")
                matches = coord_pattern.findall(self._header[3])
            lat_sign = -1 if matches[0][0] == 'S' else 1
            latitude = lat_sign * (float(matches[0][1]) + (float(matches[0][2]) / 60))
            lon_sign = -1 if matches[1][0] == 'W' else 1
            longitude = lon_sign * (float(matches[1][1]) + (float(matches[1][2]) / 60))
            time_zone = self._regex_check(r"{GMT\s*(\S*)\s*Hours}", self._header[3])
            elev_pattern = re.compile(r"Elevation\s*[-]*\s*(\d*)m\s*(\S*)")
            elev_matches = elev_pattern.findall(self._header[4])
            if len(elev_matches) == 0:
                elev_pattern = re.compile(r"Elevation\s*[-]*\s*(\d*)\s*m\s*(\S*)")
                elev_matches = elev_pattern.findall(self._header[4])
            elev_sign = -1 if elev_matches[0][-1].lower() == 'below' else 1
            elevation = elev_sign * float(elev_matches[0][0])
            self._location = Location()
            self._location.city = city
            self._location.country = country
            self._location.source = source
            self._location.station_id = station_id
            self._location.latitude = latitude
            self._location.longitude = longitude
            self._location.time_zone = time_zone
            self._location.elevation = elevation

            # pull out individual properties
            self._stand_press_at_elev = self._regex_check(
                r"Elevation\s*[-]*\s*(\d*)", self._header[5])
            self._ashrae_climate_zone = self._regex_check(
                r'Climate type\s"(\S*)"\s\(A', self._body)
            self._koppen_climate_zone = self._regex_check(
                r'Climate type\s"(\S*)"\s\(K', self._body)

            # pull out extreme and seasonal weeks.
            self._extreme_hot_week = self._regex_week_parse(
                r"Extreme Hot Week Period selected:"
                "\s*(\w{3})\s*(\d{1,2}):\s*(\w{3})\s*(\d{1,2}),")
            self._extreme_cold_week = self._regex_week_parse(
                r"Extreme Cold Week Period selected:"
                "\s*(\w{3})\s*(\d{1,2}):\s*(\w{3})\s*(\d{1,2}),")
            self._typical_weeks = self._regex_typical_week_parse()

            # pull out annual design days
            winter_vals = self._regex_parse(r"Heating\s(\d.*)")
            for key, val in zip(DesignDay.heating_keys, winter_vals):
                self._winter_des_day_dict[key] = val
            summer_vals = self._regex_parse(r"Cooling\s(\d.*)")
            for key, val in zip(DesignDay.cooling_keys, summer_vals):
                self._summer_des_day_dict[key] = val

            # Pull out relevant monthly information
            self._monthly_tau_beam = self._regex_parse(r"taub \(beam\)(.*)")
            self._monthly_tau_diffuse = self._regex_parse(r"taud \(diffuse\)(.*)")
            self._monthly_db_50 = self._regex_parse(r"Drybulb 5.0%(.*)")
            self._monthly_wb_50 = self._regex_parse(r"Coincident Wetbulb 5.0%(.*)")
            self._monthly_db_100 = self._regex_parse(r"Drybulb 10.%(.*)")
            self._monthly_wb_100 = self._regex_parse(r"Coincident Wetbulb 10.%(.*)")
            self._monthly_db_20 = self._regex_parse(r"Drybulb 2.0%(.*)")
            self._monthly_wb_20 = self._regex_parse(r"Coincident Wetbulb 2.0%(.*)")
            self._monthly_db_04 = self._regex_parse(r"Drybulb 0.4%(.*)")
            self._monthly_wb_04 = self._regex_parse(r"Coincident Wetbulb 0.4%(.*)")
            self._monthly_db_range_50 = self._regex_parse(r"Drybulb range - DB 5%(.*)")
            self._monthly_wb_range_50 = self._regex_parse(r"Wetbulb range - DB 5%(.*)")
            self._monthly_wind = self._regex_parse(
                r"Monthly Statistics for Wind Speed[\s\S]*Daily Avg(.*)")
            for direction in self._wind_dir_names:
                re_string = r"Monthly Wind Direction %[\s\S]*" + direction + r"\s(.*)"
                dirs = self._regex_parse(re_string)
                if dirs != []:
                    self._monthly_wind_dirs.append(dirs)
            if self._monthly_wind_dirs == []:
                self._monthly_wind_dirs = [[0] * 12 for i in xrange(8)]

        finally:
            statwin.close()
1312035
def fromfile(file, dtype, count, *args, **kwargs):
    """"""

    try:
        return np.fromfile(file, dtype=dtype, count=count, *args, **kwargs)
    except (TypeError, IOError, UnsupportedOperation):
        return np.frombuffer(
            file.read(count * np.dtype(dtype).itemsize),
            dtype=dtype, count=count, *args, **kwargs)
173461
def export(self, name, columns, points):
        """"""
        for i in range(len(columns)):
            if not isinstance(points[i], Number):
                continue
            else:
                data = {'host': self.hostname, 'service': name + " " + columns[i], 'metric': points[i]}
                logger.debug(data)
                try:
                    self.client.send(data)
                except Exception as e:
                    logger.error("Cannot export stats to Riemann (%s)" % e)
479579
def db_import(ctx, schema, uuid, object_filter, import_format, filename, all_schemata, dry):
    """"""

    import_format = import_format.upper()

    with open(filename, 'r') as f:
        json_data = f.read()
    data = json.loads(json_data)  # , parse_float=True, parse_int=True)

    if schema is None:
        if all_schemata is False:
            log('No schema given. Read the help', lvl=warn)
            return
        else:
            schemata = data.keys()
    else:
        schemata = [schema]

    from hfos import database
    database.initialize(ctx.obj['dbhost'], ctx.obj['dbname'])

    all_items = {}
    total = 0

    for schema_item in schemata:
        model = database.objectmodels[schema_item]

        objects = data[schema_item]
        if uuid:
            for item in objects:
                if item['uuid'] == uuid:
                    items = [model(item)]
        else:
            items = []
            for item in objects:
                thing = model(item)
                items.append(thing)

        schema_total = len(items)
        total += schema_total

        if dry:
            log('Would import', schema_total, 'items of', schema_item)
        all_items[schema_item] = items

    if dry:
        log('Would import', total, 'objects.')
    else:
        log('Importing', total, 'objects.')
        for schema_name, item_list in all_items.items():
            log('Importing', len(item_list), 'objects of type', schema_name)
            for item in item_list:
                item._fields['_id'] = bson.objectid.ObjectId(item._fields['_id'])
                item.save()
660447
def dump_stats(filename):
    # type: (str) -> None
    
    res = _dump_impl()
    f = open(filename, 'w')
    json.dump(res, f, indent=4)
    f.close()
78270
def read_stat():
    
    data = []
    with open("/proc/stat", "rb") as stat_file:
        for line in stat_file:
            cpu_stat = line.split()
            if cpu_stat[0][:3] != b"cpu":
                break
            # First cpu line is aggregation of following lines, skip it
            if len(cpu_stat[0]) == 3:
                continue
            data.append(
                {
                    "times": {
                        "user": int(cpu_stat[1]),
                        "nice": int(cpu_stat[2]),
                        "sys": int(cpu_stat[3]),
                        "idle": int(cpu_stat[4]),
                        "irq": int(cpu_stat[6]),
                    }
                }
            )
    return data
547969
def statstable(args):
    
    p = OptionParser(statstable.__doc__)
    opts, args = p.parse_args(args)

    if len(args) < 1:
        sys.exit(not p.print_help())

    gff_files = args
    for metric in metrics:
        logging.debug("Parsing files in `{0}`..".format(metric))

        table = {}
        for x in gff_files:
            pf = op.basename(x).split(".")[0]
            numberfile = op.join(metric, pf + ".txt")
            ar = [int(x.strip()) for x in open(numberfile)]
            sum = SummaryStats(ar).todict().items()
            keys, vals = zip(*sum)
            keys = [(pf, x) for x in keys]
            table.update(dict(zip(keys, vals)))

        print(tabulate(table), file=sys.stderr)
890408
def import_data(self, filenames=None):
        """"""
        title = _("Import data")
        if filenames is None:
            if self.filename is None:
                basedir = getcwd_or_home()
            else:
                basedir = osp.dirname(self.filename)
            filenames, _selfilter = getopenfilenames(self, title, basedir,
                                                     iofunctions.load_filters)
            if not filenames:
                return
        elif is_text_string(filenames):
            filenames = [filenames]

        for filename in filenames:
            self.filename = to_text_string(filename)
            ext = osp.splitext(self.filename)[1].lower()

            if ext not in iofunctions.load_funcs:
                buttons = QMessageBox.Yes | QMessageBox.Cancel
                answer = QMessageBox.question(self, title,
                            _("<b>Unsupported file extension '%s'</b><br><br>"
                              "Would you like to import it anyway "
                              "(by selecting a known file format)?"
                              ) % ext, buttons)
                if answer == QMessageBox.Cancel:
                    return
                formats = list(iofunctions.load_extensions.keys())
                item, ok = QInputDialog.getItem(self, title,
                                                _('Open file as:'),
                                                formats, 0, False)
                if ok:
                    ext = iofunctions.load_extensions[to_text_string(item)]
                else:
                    return

            load_func = iofunctions.load_funcs[ext]
                
            # 'import_wizard' (self.setup_io)
            if is_text_string(load_func):
                # Import data with import wizard
                error_message = None
                try:
                    text, _encoding = encoding.read(self.filename)
                    base_name = osp.basename(self.filename)
                    editor = ImportWizard(self, text, title=base_name,
                                  varname=fix_reference_name(base_name))
                    if editor.exec_():
                        var_name, clip_data = editor.get_data()
                        self.editor.new_value(var_name, clip_data)
                except Exception as error:
                    error_message = str(error)
            else:
                QApplication.setOverrideCursor(QCursor(Qt.WaitCursor))
                QApplication.processEvents()
                error_message = self.shellwidget.load_data(self.filename, ext)
                self.shellwidget._kernel_reply = None
                QApplication.restoreOverrideCursor()
                QApplication.processEvents()
    
            if error_message is not None:
                QMessageBox.critical(self, title,
                                     _("<b>Unable to load '%s'</b>"
                                       "<br><br>Error message:<br>%s"
                                       ) % (self.filename, error_message))
            self.refresh_table()
457054
def import_table(self, source, table_name):
        """"""
        query = "SELECT * FROM `%s`" % table_name.lower()
        df = pandas.read_sql(query, source.connection)
        df.to_sql(table_name, con=self.own_connection)
731712
def copystat(src, dst):
    """"""
    st = os.stat(src)
    mode = stat.S_IMODE(st.st_mode)
    if hasattr(os, 'utime'):
        os.utime(dst, (st.st_atime, st.st_mtime))
    if hasattr(os, 'chmod'):
        os.chmod(dst, mode)
    if hasattr(os, 'chflags') and hasattr(st, 'st_flags'):
        try:
            os.chflags(dst, st.st_flags)
        except OSError as why:
            if (not hasattr(errno, 'EOPNOTSUPP') or
                why.errno != errno.EOPNOTSUPP):
                raise
863856
def load_data(self, filename, *args, **kwargs):
        
        # workbook read from file
        workbook = open_workbook(filename, verbosity=True)
        data = {}  # an empty dictionary to store data
        # iterate through sheets in parameters
        # iterate through the parameters on each sheet
        for param, pval in self.parameters.iteritems():
            sheet = pval['extras']['sheet']
            # get each worksheet from the workbook
            worksheet = workbook.sheet_by_name(sheet)
            # split the parameter's range elements
            prng0, prng1 = pval['extras']['range']
            # missing "units", json ``null`` and Python ``None`` all OK!
            # convert to str from unicode, None to '' (dimensionless)
            punits = str(pval.get('units') or '')
            # replace None with empty list
            if prng0 is None:
                prng0 = []
            if prng1 is None:
                prng1 = []
            # FIXME: Use duck-typing here instead of type-checking!
            # if both elements in range are `int` then parameter is a cell
            if isinstance(prng0, int) and isinstance(prng1, int):
                datum = worksheet.cell_value(prng0, prng1)
            # if the either element is a `list` then parameter is a slice
            elif isinstance(prng0, list) and isinstance(prng1, int):
                datum = worksheet.col_values(prng1, *prng0)
            elif isinstance(prng0, int) and isinstance(prng1, list):
                datum = worksheet.row_values(prng0, *prng1)
            # if both elements are `list` then parameter is 2-D
            else:
                datum = []
                for col in xrange(prng0[1], prng1[1]):
                    datum.append(worksheet.col_values(col, prng0[0],
                                                      prng1[0]))
            # duck typing that datum is real
            try:
                npdatum = np.array(datum, dtype=np.float)
            except ValueError as err:
                # check for iterable:
                # if `datum` can't be coerced to float, then it must be
                # *string* & strings *are* iterables, so don't check!
                # check for strings:
                # data must be real or *all* strings!
                # empty string, None or JSON null also OK
                # all([]) == True but any([]) == False
                if not datum:
                    data[param] = None  # convert empty to None
                elif all(isinstance(_, basestring) for _ in datum):
                    data[param] = datum  # all str is OK (EG all 'TMY')
                elif all(not _ for _ in datum):
                    data[param] = None  # convert list of empty to None
                else:
                    raise err  # raise ValueError if not all real or str
            else:
                data[param] = npdatum * UREG(punits)
            # FYI: only put one statement into try-except test otherwise
            # might catch different error than expected. use ``else`` as
            # option to execute only if exception *not* raised.
        return data
10395
def data_import(self, json_response):
        """"""
        if 'data' not in json_response:
            raise PyVLXException('no element data found: {0}'.format(
                json.dumps(json_response)))
        data = json_response['data']

        for item in data:
            if 'category' not in item:
                raise PyVLXException('no element category: {0}'.format(
                    json.dumps(item)))
            category = item['category']
            if category == 'Window opener':
                self.load_window_opener(item)
            elif category in ['Roller shutter', 'Dual Shutter']:
                self.load_roller_shutter(item)
            elif category in ['Blind']:
                self.load_blind(item)
            else:
                self.pyvlx.logger.warning(
                    'WARNING: Could not parse product: %s', category)
1361467
def export(self, name, columns, points):
        """"""

        # Check for completion of loop for all exports
        if name == self.plugins_to_export()[0] and self.buffer != {}:
            # One whole loop has been completed
            # Flush stats to file
            logger.debug("Exporting stats ({}) to JSON file ({})".format(
                listkeys(self.buffer),
                self.json_filename)
            )

            # Export stats to JSON file
            data_json = json.dumps(self.buffer)
            self.json_file.write("{}\n".format(data_json))

            # Reset buffer
            self.buffer = {}

        # Add current stat to the buffer
        self.buffer[name] = dict(zip(columns, points))
147552
def stat(self, filename, timeout=None):
    """"""
    transport = StatFilesyncTransport(self.stream)
    transport.write_data('STAT', filename, timeout)
    stat_msg = transport.read_message(timeout)
    stat_msg.assert_command_is('STAT')
    return DeviceFileStat(filename, stat_msg.mode, stat_msg.size, stat_msg.time)
873711
def write_stats(stats, fname, header=None):
    """"""
    # Write stats output to file

    for bg in list(stats.values())[0].keys():
        f = open(fname.format(bg), "w")
        if header:
            f.write(header)
        
        stat_keys = sorted(list(list(stats.values())[0].values())[0].keys())
        f.write("{}\t{}\n".format("Motif", "\t".join(stat_keys)))

        for motif in stats:
            m_stats = stats.get(str(motif), {}).get(bg)
            if m_stats:
                f.write("{}\t{}\n".format(
                    "_".join(motif.split("_")[:-1]),
                    "\t".join([str(m_stats[k]) for k in stat_keys])
                    ))
            else:
                logger.warn("No stats for motif {0}, skipping this motif!".format(motif.id))
            #motifs.remove(motif)
        f.close()

    return
1071306
def pickle_data(data, picklefile):
    """"""
    with open(picklefile, 'wb') as f:
        pickle.dump(data, f, protocol=2)
149405
def stat(self, filename):
        """"""
        # NOTE: Size of the file is given by ContentLength from S3,
        # but we convert to .length
        client = boto3.client("s3")
        bucket, path = self.bucket_and_path(filename)
        try:
            obj = client.head_object(Bucket=bucket, Key=path)
            return StatData(obj['ContentLength'])
        except botocore.exceptions.ClientError as exc:
            if exc.response['Error']['Code'] == '404':
                raise errors.NotFoundError(None, None, "Could not find file")
            else:
                raise