convert the response map to a single data frame with Multi-Index columns
327741
def response_as_single(self, copy=0):
        
        arr = []
        for sid, frame in self.response.iteritems():
            if copy:
                frame = frame.copy()
            'security' not in frame and frame.insert(0, 'security', sid)
            arr.append(frame.reset_index().set_index(['date', 'security']))
        return concat(arr).unstack()
239277
def dflin2dfbinarymap(dflin,col1,col2,params_df2submap={'aggfunc':'sum','binary':True,'binaryby':'nan'},test=False):
    
    # get the submap ready
    df_map=df2submap(df=dflin,
              col=col1,idx=col2,**params_df2submap)
    if test:           
        logging.debug(df_map.unstack().unique())
    # make columns and index symmetric
    df_map_symm=dfmap2symmcolidx(df_map)
    df_map_symm=df_map_symm.fillna(False)
    if test:           
        logging.debug(df_map_symm.unstack().unique())
    df_map_symm=(df_map_symm+df_map_symm.T)/2
    if test:           
        logging.debug(df_map_symm.unstack().unique())
    df_map_symm=df_map_symm!=0
    if test:           
        logging.debug(df_map_symm.unstack().unique())
    return df_map_symm
1265165
def single_row_or_col_df_to_series(desired_type: Type[T], single_rowcol_df: pd.DataFrame, logger: Logger, **kwargs)\
        -> pd.Series:
    
    if single_rowcol_df.shape[0] == 1:
        # one row
        return single_rowcol_df.transpose()[0]
    elif single_rowcol_df.shape[1] == 2 and isinstance(single_rowcol_df.index, pd.RangeIndex):
        # two columns but the index contains nothing but the row number : we can use the first column
        d = single_rowcol_df.set_index(single_rowcol_df.columns[0])
        return d[d.columns[0]]
    elif single_rowcol_df.shape[1] == 1:
        # one column and one index
        d = single_rowcol_df
        return d[d.columns[0]]
    else:
        raise ValueError('Unable to convert provided dataframe to a series : '
                         'expected exactly 1 row or 1 column, found : ' + str(single_rowcol_df.shape) + '')
426787
def dataframe(df, **kwargs):
    
    table(df.values, list(df.columns), **kwargs)
1013061
def results_to_table(self):
        """"""
        # Get Labels
        baseline_labels = self.nearest_targets['origin_id'].values
        mcf_labels = self.results['model_labels']['labels'].values

        # Create the outcomes
        outcome = pd.DataFrame({
                'the_geom': [
                    'SRID=4326;Point({lng} {lat})'.format(lng=v[0], lat=v[1])
                    for v in zip(self.results['model_labels']['lng'].values, self.results['model_labels']['lat'].values)],
                'target_lng': self.results['model_labels']['lng'].values,
                'target_lng': self.results['model_labels']['lat'].values,
                'origin_lng': self.origins.reindex(baseline_labels)['lng'].values,
                'origin_lat': self.origins.reindex(baseline_labels)['lat'].values,
                'target_id': self.results['model_labels'].target_id,
                'sales': self.results['model_labels'][self.demand_col].values,
                'labels': baseline_labels
            }
        )

        outcomes2 = pd.DataFrame({
                'the_geom': [
                    'SRID=4326;Point({lng} {lat})'.format(lng=v[0], lat=v[1])
                    for v in zip(
                        self.results['model_labels']['lng'].values,
                        self.results['model_labels']['lat'].values
                    )
                ],
                'target_lng': self.results['model_labels']['lng'].values,
                'target_lat': self.results['model_labels']['lat'].values,
                'origin_lng': self.origins.reindex(mcf_labels)['lng'].values,
                'origin_lat': self.origins.reindex(mcf_labels)['lat'].values,
                'target_id': self.results['model_labels'].target_id,
                'sales': self.results['model_labels'][self.demand_col].values,
                'labels': mcf_labels
            },
            index=self.results['model_labels'].target_id
        )
        now = datetime.datetime.now()
        out_table = 'mincostflow_{}'.format(now.strftime("%Y_%m_%d_%H_%M_%S"))
        logging.info('Writing output to {}'.format(out_table))
        self.context.write(outcomes2.reset_index(drop=True), out_table)
        logging.info('Table {} written to CARTO'.format(out_table))
        return out_table
497790
def to_df(self) -> pd.DataFrame:
        """"""
        df = pd.DataFrame(index=RangeIndex(0, self.shape[0], name=None))
        for key in self.keys():
            value = self[key]
            for icolumn, column in enumerate(value.T):
                df['{}{}'.format(key, icolumn+1)] = column
        return df
212087
def process_one(df, *keys, columns=['key', 'agent_id'], values='value',
                fill=True, index=['t_step',],
                aggfunc='first', **kwargs):
    '''
    
    '''
    if df is None:
        return df
    if keys:
        df = df[df['key'].isin(keys)]

    df = df.pivot_table(values=values, index=index, columns=columns,
                        aggfunc=aggfunc, **kwargs)
    if fill:
        df = fillna(df)
    return df
71711
def from_response(cls, response, attrs):
        
        proj = response['Projection']
        hash_key = attrs.get(response['KeySchema'][0]['AttributeName'])
        range_key = None
        if len(response['KeySchema']) > 1:
            range_key = attrs[response['KeySchema'][1]['AttributeName']]
        throughput = Throughput.from_response(
            response['ProvisionedThroughput'])
        index = cls(proj['ProjectionType'], response['IndexName'], hash_key,
                    range_key, proj.get('NonKeyAttributes'), throughput)
        index.response = response
        return index
1158380
def index_map(data):
  """"""
  (entry, text_fn) = data
  text = text_fn()

  logging.debug("Got %s", entry.filename)
  for s in split_into_sentences(text):
    for w in split_into_words(s.lower()):
      yield (w, entry.filename)
981693
def _to_str_columns(self):
        
        frame = self.tr_frame
        # may include levels names also

        str_index = self._get_formatted_index(frame)

        if not is_list_like(self.header) and not self.header:
            stringified = []
            for i, c in enumerate(frame):
                fmt_values = self._format_col(i)
                fmt_values = _make_fixed_width(fmt_values, self.justify,
                                               minimum=(self.col_space or 0),
                                               adj=self.adj)
                stringified.append(fmt_values)
        else:
            if is_list_like(self.header):
                if len(self.header) != len(self.columns):
                    raise ValueError(('Writing {ncols} cols but got {nalias} '
                                      'aliases'
                                      .format(ncols=len(self.columns),
                                              nalias=len(self.header))))
                str_columns = [[label] for label in self.header]
            else:
                str_columns = self._get_formatted_column_labels(frame)

            if self.show_row_idx_names:
                for x in str_columns:
                    x.append('')

            stringified = []
            for i, c in enumerate(frame):
                cheader = str_columns[i]
                header_colwidth = max(self.col_space or 0,
                                      *(self.adj.len(x) for x in cheader))
                fmt_values = self._format_col(i)
                fmt_values = _make_fixed_width(fmt_values, self.justify,
                                               minimum=header_colwidth,
                                               adj=self.adj)

                max_len = max(max(self.adj.len(x) for x in fmt_values),
                              header_colwidth)
                cheader = self.adj.justify(cheader, max_len, mode=self.justify)
                stringified.append(cheader + fmt_values)

        strcols = stringified
        if self.index:
            strcols.insert(0, str_index)

        # Add ... to signal truncated
        truncate_h = self.truncate_h
        truncate_v = self.truncate_v

        if truncate_h:
            col_num = self.tr_col_num
            strcols.insert(self.tr_col_num + 1, [' ...'] * (len(str_index)))
        if truncate_v:
            n_header_rows = len(str_index) - len(frame)
            row_num = self.tr_row_num
            for ix, col in enumerate(strcols):
                # infer from above row
                cwidth = self.adj.len(strcols[ix][row_num])
                is_dot_col = False
                if truncate_h:
                    is_dot_col = ix == col_num + 1
                if cwidth > 3 or is_dot_col:
                    my_str = '...'
                else:
                    my_str = '..'

                if ix == 0:
                    dot_mode = 'left'
                elif is_dot_col:
                    cwidth = 4
                    dot_mode = 'right'
                else:
                    dot_mode = 'right'
                dot_str = self.adj.justify([my_str], cwidth, mode=dot_mode)[0]
                strcols[ix].insert(row_num + n_header_rows, dot_str)
        return strcols
577683
def as_dataframe(self, fillna=True, subjects=None):
        
        entries = []
        selected_subjects = self.subjects
        if subjects is not None:
            selected_subjects = subjects
            
        for s in selected_subjects:
            vmap = {}
            for c in self.inferred_types(s):
                vmap[c] = 1
            entries.append(vmap)
        logging.debug("Creating DataFrame")
        df = pd.DataFrame(entries, index=selected_subjects)
        if fillna:
            logging.debug("Performing fillna...")
            df = df.fillna(0)
        return df
615746
def dmap2lin(df,idxn='index',coln='column',colvalue_name='value'):
    
#     df.columns.name=coln
    df.index.name=idxn
    return df.reset_index().melt(id_vars=[df.index.name],
                             var_name=coln,value_name=colvalue_name)
814633
def from_response(cls, response, attrs):
        
        proj = response['Projection']
        index = cls(proj['ProjectionType'], response['IndexName'],
                    attrs[response['KeySchema'][1]['AttributeName']],
                    proj.get('NonKeyAttributes'))
        index.response = response
        return index
831612
def decode_offset_response(cls, data):
        
        ((correlation_id, num_topics), cur) = relative_unpack('>ii', data, 0)

        for _ in range(num_topics):
            (topic, cur) = read_short_string(data, cur)
            ((num_partitions,), cur) = relative_unpack('>i', data, cur)

            for _ in range(num_partitions):
                ((partition, error, num_offsets,), cur) = \
                    relative_unpack('>ihi', data, cur)

                offsets = []
                for k in range(num_offsets):
                    ((offset,), cur) = relative_unpack('>q', data, cur)
                    offsets.append(offset)

                yield OffsetResponse(topic, partition, error, tuple(offsets))
640888
@NonNull
    @Override
    public Map<String, Object> toMap() {
        final List<Object> values = toList();
        final Map<String, Object> dict = new HashMap<>();
        for (String name : rs.getColumnNames().keySet()) {
            final int index = indexForColumnName(name);
            if (index >= 0) { dict.put(name, values.get(index)); }
        }
        return dict;
    }
1102214
def map(self, func, *columns):
        
        if not columns:
            return map(func, self.rows)
        else:
            values = (self.values(column) for column in columns)
            result = [map(func, v) for v in values]
            if len(columns) == 1:
                return result[0]
            else:
                return result
1129847
def map(self, response):
        
        inserts = []

        parse = {}
        # custom remapping, cleanup, and filling in holes
        parse['i-id'] = response.get('keys', {}).get('i-id', -1)
        self._parse_id = max(self._parse_id + 1, parse['i-id'])
        parse['parse-id'] = self._parse_id
        parse['run-id'] = response.get('run', {}).get('run-id', -1)
        if 'tokens' in response:
            parse['p-input'] = response['tokens'].get('initial')
            parse['p-tokens'] = response['tokens'].get('internal')
            if 'ninputs' not in response:
                toks = response.tokens('initial')
                if toks is not None:
                    response['ninputs'] = len(toks.tokens)
            if 'ntokens' not in response:
                toks = response.tokens('internal')
                if toks is not None:
                    response['ntokens'] = len(toks.tokens)
        if 'readings' not in response and 'results' in response:
            response['readings'] = len(response['results'])
        # basic mapping
        for key in self._parse_keys:
            if key in response:
                parse[key] = response[key]
        inserts.append(('parse', parse))

        for result in response.get('results', []):
            d = {'parse-id': self._parse_id}
            if 'flags' in result:
                d['flags'] = SExpr.format(result['flags'])
            for key in self._result_keys:
                if key in result:
                    d[key] = result[key]
            inserts.append(('result', d))

        if 'run' in response:
            run_id = response['run'].get('run-id', -1)
            # check if last run was not closed properly
            if run_id not in self._runs and self._last_run_id in self._runs:
                last_run = self._runs[self._last_run_id]
                if 'end' not in last_run:
                    last_run['end'] = datetime.now()
            self._runs[run_id] = response['run']
            self._last_run_id = run_id

        return inserts