读取历史财务数据文件，并返回pandas结果 ， 类似gpcw20171231.zip格式，具体字段含义参考

        https://github.com/rainx/pytdx/issues/133

        :param data_file: 数据文件地址， 数据文件类型可以为 .zip 文件，也可以为解压后的 .dat
        :return: pandas DataFrame格式的历史财务数据
198857
def get_df(self, data_file):
        

        crawler = QAHistoryFinancialCrawler()

        with open(data_file, 'rb') as df:
            data = crawler.parse(download_file=df)

        return crawler.to_df(data)
474132
def _previewFile(self):
        
        dataFrame = self._loadCSVDataFrame()
        dataFrameModel = DataFrameModel(dataFrame, filePath=self._filename)
        dataFrameModel.enableEditing(True)
        self._previewTableView.setModel(dataFrameModel)
        columnModel = dataFrameModel.columnDtypeModel()
        columnModel.changeFailed.connect(self.updateStatusBar)
        self._datatypeTableView.setModel(columnModel)
682161
def index(self, symbol='incon.dat', group=False):
        '''
        
        '''
        reader = BlockReader()
        symbol = os.path.join(self.tdxdir, symbol)

        if symbol is not None:
            return reader.get_df(symbol, group)

        return None
428157
def record(self, timestamp, *args, **kwargs):
        
        if self.output_file is None:
            return

        data = {'datetime': timestamp}

        # append all data
        if len(args) == 1:
            if isinstance(args[0], dict):
                data.update(dict(args[0]))
            elif isinstance(args[0], pd.DataFrame):
                data.update(args[0][-1:].to_dict(orient='records')[0])

        # add kwargs
        if kwargs:
            data.update(dict(kwargs))

        data['datetime'] = timestamp
        # self.rows.append(pd.DataFrame(data=data, index=[timestamp]))

        new_data = {}
        if "symbol" not in data.keys():
            new_data = dict(data)
        else:
            sym = data["symbol"]
            new_data["symbol"] = data["symbol"]
            for key in data.keys():
                if key not in ['datetime', 'symbol_group', 'asset_class']:
                    new_data[sym + '_' + str(key).upper()] = data[key]

        new_data['datetime'] = timestamp

        # append to rows
        self.rows.append(pd.DataFrame(data=new_data, index=[timestamp]))

        # create dataframe
        recorded = pd.concat(self.rows, sort=True)

        if "symbol" not in recorded.columns:
            return


        # group by symbol
        recorded['datetime'] = recorded.index
        data = recorded.groupby(['symbol', 'datetime'], as_index=False).sum()
        data.set_index('datetime', inplace=True)

        symbols = data['symbol'].unique().tolist()
        data.drop(columns=['symbol'], inplace=True)


        # cleanup:

        # remove symbols
        recorded.drop(['symbol'] + [sym + '_SYMBOL' for sym in symbols],
                      axis=1, inplace=True)

        # remove non-option data if not working with options
        for sym in symbols:
            try:
                opt_cols = recorded.columns[
                    recorded.columns.str.startswith(sym + '_OPT_')].tolist()
                if len(opt_cols) == len(recorded[opt_cols].isnull().all()):
                    recorded.drop(opt_cols, axis=1, inplace=True)
            except Exception as e:
                pass

        # group df
        recorded = recorded.groupby(recorded['datetime']).first()

        # shift position
        for sym in symbols:
            recorded[sym + '_POSITION'] = recorded[sym + '_POSITION'
                                                   ].shift(1).fillna(0)

        # make this public
        self.recorded = recorded.copy()

        # cleanup columns names before saving...
        recorded.columns = [col.replace('_FUT_', '_').replace(
                            '_OPT_OPT_', '_OPT_') for col in recorded.columns]

        # save
        if ".csv" in self.output_file:
            recorded.to_csv(self.output_file)
        elif ".h5" in self.output_file:
            recorded.to_hdf(self.output_file, 0)
        elif (".pickle" in self.output_file) | (".pkl" in self.output_file):
            recorded.to_pickle(self.output_file)

        chmod(self.output_file)
1260831
def _load_mtdata(gzfile):
    
    path = os.path.join(os.path.dirname(__file__), 'tests', 'data', gzfile)
    return np.loadtxt(gzip.open(path))
477789
def load_df_state(path_csv: Path)->pd.DataFrame:
    '''
    '''

    df_state = pd.read_csv(
        path_csv,
        header=[0, 1],
        index_col=[0, 1],
        parse_dates=True,
        infer_datetime_format=True,
    )
    return df_state
1336181
def rawfile_within_timeframe(rawfile, timeframe):
    
    matches = re.search(r'-(\d{8})-', rawfile)
    if matches:
        ftime = datetime.strptime(matches.group(1), "%Y%m%d")
        ftime = pytz.utc.localize(ftime)

    return ftime.date() >= timeframe[0].date() and ftime.date() <= timeframe[1].date()
38335
def as_dataframe(self, absolute_time=False):
        

        import pandas as pd

        # When absolute_time is True,
        # use the wf_start_time as offset for the time_track()
        try:
            time = self.time_track(absolute_time)
        except KeyError:
            time = None
        if self.channel is None:
            return pd.DataFrame.from_items(
                [(ch.channel, pd.Series(ch.data))
                    for ch in self.tdms_file.group_channels(self.group)])
        else:
            return pd.DataFrame(self._data, index=time, columns=[self.path])
977683
def dframe(self):
        
        self.param.warning("The MultiDimensionalMapping.dframe method is "
                           "deprecated and should no longer be used. "
                           "Use a more specific subclass which does support "
                           "the dframe method instead, e.g. a HoloMap.")
        try:
            import pandas
        except ImportError:
            raise Exception("Cannot build a DataFrame without the pandas library.")
        labels = self.dimensions('key', True) + [self.group]
        return pandas.DataFrame(
            [dict(zip(labels, k + (v,))) for (k, v) in self.data.items()])
604266
def _fuzz_data_file(self, data_file):
        
        buf = bytearray(open(os.path.abspath(data_file), 'rb').read())
        fuzzed = fuzzer(buf, self.fuzz_factor)
        try:
            _, fuzz_output = mkstemp(prefix='fuzzed_')
            open(fuzz_output, 'wb').write(fuzzed)
        finally:
            pass
        return fuzz_output
81911
def prepare_csv_to_dataframe(data_file, config, use_target=True):
    
    names, dtypes = [], []
    model = config.get_data_model()
    for feature in model:
        assert feature.get_name() not in names, "Two features can't have the same name."
        if not use_target and feature.is_target():
            continue
        names.append(feature.get_name())
    data = pd.read_csv(data_file, names=names)

    transform_categorical_features(config, data, use_target)

    return data
446661
def _maybe_pandas_data(data, feature_names, feature_types):
    

    if not isinstance(data, DataFrame):
        return data, feature_names, feature_types

    data_dtypes = data.dtypes
    if not all(dtype.name in PANDAS_DTYPE_MAPPER for dtype in data_dtypes):
        bad_fields = [data.columns[i] for i, dtype in
                      enumerate(data_dtypes) if dtype.name not in PANDAS_DTYPE_MAPPER]

        msg = """DataFrame.dtypes for data must be int, float or bool.
                Did not expect the data types in fields """
        raise ValueError(msg + ', '.join(bad_fields))

    if feature_names is None:
        if isinstance(data.columns, MultiIndex):
            feature_names = [
                ' '.join([str(x) for x in i])
                for i in data.columns
            ]
        else:
            feature_names = data.columns.format()

    if feature_types is None:
        feature_types = [PANDAS_DTYPE_MAPPER[dtype.name] for dtype in data_dtypes]

    data = data.values.astype('float')

    return data, feature_names, feature_types
608195
def __build_dataframe(self, timing, project_name=None, org_name=None):
        
        date_list = []
        uuid_list = []
        name_list = []
        contribs_list = []
        latest_ts_list = []
        logger.debug(self.__log_prefix + " timing: " + timing.key_as_string)

        for author in timing[self.AUTHOR_UUID].buckets:
            latest_ts_list.append(timing[self.LATEST_TS].value_as_string)
            date_list.append(timing.key_as_string)
            uuid_list.append(author.key)
            if author[self.AUTHOR_NAME] and author[self.AUTHOR_NAME].buckets \
                    and len(author[self.AUTHOR_NAME].buckets) > 0:
                name_list.append(author[self.AUTHOR_NAME].buckets[0].key)
            else:
                name_list.append("Unknown")
            contribs_list.append(author[self.CONTRIBUTIONS].value)

        df = pandas.DataFrame()
        df[self.TIMEFRAME] = date_list
        df[self.AUTHOR_UUID] = uuid_list
        df[self.AUTHOR_NAME] = name_list
        df[self.CONTRIBUTIONS] = contribs_list
        df[self.TIMESTAMP] = latest_ts_list

        if not project_name:
            project_name = "_Global_"
        df[self.PROJECT] = project_name

        if not org_name:
            org_name = "_Global_"
        df[self.AUTHOR_ORG] = org_name

        return df
1156427
def has_data_for_dates(series_or_df, first_date, last_date):
    
    dts = series_or_df.index
    if not isinstance(dts, pd.DatetimeIndex):
        raise TypeError("Expected a DatetimeIndex, but got %s." % type(dts))
    first, last = dts[[0, -1]]
    return (first <= first_date) and (last >= last_date)
1015351
def add_dataframe(df, name, pkg=None, description=''):
    
    from warnings import warn
    from metapack.cli.core import alt_col_name, type_map
    import numpy as np

    if name is None or df is None:
        warn("Did not find dataframe for reference '{}' ".format(ref))
        return

    pkg = pkg or open_source_package()

    resource_ref = 'file:' + get_notebook_rel_path(pkg) + '#' + name

    t = pkg.find_first('Root.Datafile', value=resource_ref)
    col_props = {}

    if t:
        print("Datafile exists for url '{}', deleting".format(resource_ref))

        if t.schema_term:
            col_props = { c['name']:c for c in t.columns()}
            pkg.remove_term(t.schema_term)

        pkg.remove_term(t)

    t = pkg['Resources'].new_term('Root.Datafile', resource_ref, name=name, description=description)

    st = pkg['Schema'].new_term('Table', t.schema_name, description=description)

    for i, name in enumerate(df.columns):

        props = col_props.get(name,{})

        try:
            native_type = type(np.asscalar(df[name].dtype.type(0))).__name__
        except ValueError:
            native_type = df[name].dtype.name
        except AttributeError:
            native_type = type(df[name][0]).__name__

        for pn in 'datatype name pos header'.split():
            if pn in props:
                del props[pn]

        if 'altname' in props:
            altname = props['altname']
            del props['altname']
        else:
            raw_alt_name = alt_col_name(name, i)
            altname = raw_alt_name if raw_alt_name != name else ''

        col = df[name]

        if hasattr(col, 'description'):  # custom property
            props['description'] = col.description

        t = st.new_child('Column', name,
                            datatype=type_map.get(native_type, native_type),
                            altname=altname,
                            **props)


    pkg.write_csv()
54521
def retrieve_data(self):
        

        #==== Retrieve data ====#

        df = self.manager.get_historic_data(self.start.date(), self.end.date())
        df.replace(0, np.nan, inplace=True)
        return df
222355
def load_data_file(filename, encoding='utf-8'):
    
    data = pkgutil.get_data(PACKAGE_NAME, os.path.join(DATA_DIR, filename))
    return data.decode(encoding).splitlines()