Parameters
		----------
		corpus

		Returns
		-------
		float, pd.Series
		float: point on x-axis at even characteristicness
		pd.Series: term -> value between 0 and 1, sorted by score in a descending manner
		Background scores from corpus
554052
def get_scores(self, corpus):
		'''
		
		'''
		term_ranks = self.term_ranker(corpus).get_ranks()

		freq_df = pd.DataFrame({
			'corpus': term_ranks.sum(axis=1),
			'standard': self.background_frequencies.get_background_frequency_df()['background']}
		).dropna()
		corpus_rank = rankdata(freq_df.corpus, 'dense')
		standard_rank = rankdata(freq_df.standard, 'dense')
		scores = corpus_rank/corpus_rank.max() - standard_rank/standard_rank.max()


		#scores = RankDifference().get_scores(bg['corpus'], bg['bg']).sort_values()
		# import pdb; pdb.set_trace()
		if self.rerank_ranks:
			rank_scores, zero_marker = self._rerank_scores(scores)
			freq_df['score'] = pd.Series(rank_scores, index=freq_df.index)
		else:
			if scores.min() < 0 and scores.max() > 0:
				zero_marker = -scores.min() / (scores.max() - scores.min())
			elif scores.min() > 0:
				zero_marker = 0
			else:
				zero_marker = 1
			freq_df['score'] = scale(scores)
		return zero_marker, freq_df.sort_values(by='score', ascending=False)['score']
537400
def sd(series):
    

    if np.issubdtype(series.dtype, np.number):
        return series.std()
    else:
        return np.nan
683628
def draw(self,
             category,
             num_top_words_to_annotate=4,
             words_to_annotate=[],
             scores=None,
             transform=percentile_alphabetical):
        '''
        '''
        try:
            import matplotlib.pyplot as plt
        except:
            raise Exception("matplotlib and mpld3 need to be installed to use this function.")
        try:
            from mpld3 import plugins, fig_to_html
        except:
            raise Exception("mpld3 need to be installed to use this function.")
        all_categories, other_categories = self._get_category_names(category)
        df = self._term_rank_score_and_frequency_df(all_categories, category, other_categories, scores)
        if self.x_coords is None:
            df['x'], df['y'] = self._get_coordinates_from_transform_and_jitter_frequencies \
                (category, df, other_categories, transform)
        df_to_annotate = df[(df['not category score rank'] <= num_top_words_to_annotate)
                            | (df['category score rank'] <= num_top_words_to_annotate)
                            | df['term'].isin(words_to_annotate)]
        words = list(df['term'])

        font = {'family': 'sans-serif',
                'color': 'black',
                'weight': 'normal',
                'size': 'large'}

        fig, ax = plt.subplots()
        plt.figure(figsize=(10, 10))
        plt.gcf().subplots_adjust(bottom=0.2)
        plt.gcf().subplots_adjust(right=0.2)

        points = ax.scatter(self.x_coords,
                            self.y_coords,
                            c=-df['color_scores'],
                            cmap='seismic',
                            s=10,
                            edgecolors='none',
                            alpha=0.9)
        tooltip = plugins.PointHTMLTooltip(points,
                                           ['<span id=a>%s</span>' % w for w in words],
                                           css='#a {background-color: white;}')
        plugins.connect(fig, tooltip)
        ax.set_ylim([-.2, 1.2])
        ax.set_xlim([-.2, 1.2])
        ax.xaxis.set_ticks([0., 0.5, 1.])
        ax.yaxis.set_ticks([0., 0.5, 1.])
        ax.set_ylabel(category.title() + ' Frequency Percentile', fontdict=font, labelpad=20)
        ax.set_xlabel('Not ' + category.title() + ' Frequency Percentile', fontdict=font, labelpad=20)

        for i, row in df_to_annotate.iterrows():
            # alignment_criteria = row['category score rank'] < row['not category score rank']
            alignment_criteria = i % 2 == 0
            horizontalalignment = 'right' if alignment_criteria else 'left'
            verticalalignment = 'bottom' if alignment_criteria else 'top'
            term = row['term']
            ax.annotate(term,
                        (self.x_coords[i], self.y_data[i]),
                        size=15,
                        horizontalalignment=horizontalalignment,
                        verticalalignment=verticalalignment,
                        )
        # texts.append(
        # ax.text(row['dem freq scaled'], row['rep freq scaled'], row['word'])
        # )
        # adjust_text(texts, arrowprops=dict(arrowstyle="->", color='r', lw=0.5))
        plt.show()
        return df, fig_to_html(fig)
1007782
def raw_corpus_length_ratio(hypotheses: Iterable[str], references: Iterable[str]) -> float:
    
    ratios = [len(h.split())/len(r.split()) for h, r in zip(hypotheses, references)]
    return sum(ratios)/len(ratios) if len(ratios) else 0.0
1248486
def set_background_corpus(self, background):
        '''
        

        '''
        if issubclass(type(background), TermDocMatrixWithoutCategories):
            self._background_corpus = pd.DataFrame(background
                                                   .get_term_freq_df()
                                                   .sum(axis=1),
                                                   columns=['background']).reset_index()
            self._background_corpus.columns = ['word', 'background']
        elif (type(background) == pd.DataFrame
              and set(background.columns) == set(['word', 'background'])):
            self._background_corpus = background
        else:
            raise Exception('The argument named background must be a subclass of TermDocMatrix or a ' \
                            + 'DataFrame with columns "word" and "background", where "word" ' \
                            + 'is the term text, and "background" is its frequency.')
256455
def argmax_with_score(logits, axis=None):
  """"""
  axis = axis or len(logits.get_shape()) - 1
  predictions = tf.argmax(logits, axis=axis)

  logits_shape = shape_list(logits)
  prefix_shape, vocab_size = logits_shape[:-1], logits_shape[-1]
  prefix_size = 1
  for d in prefix_shape:
    prefix_size *= d

  # Flatten to extract scores
  flat_logits = tf.reshape(logits, [prefix_size, vocab_size])
  flat_predictions = tf.reshape(predictions, [prefix_size])
  flat_indices = tf.stack(
      [tf.range(tf.to_int64(prefix_size)),
       tf.to_int64(flat_predictions)],
      axis=1)
  flat_scores = tf.gather_nd(flat_logits, flat_indices)

  # Unflatten
  scores = tf.reshape(flat_scores, prefix_shape)

  return predictions, scores
184651
def _dt_to_float_ordinal(dt):
    
    if (isinstance(dt, (np.ndarray, Index, ABCSeries)
                   ) and is_datetime64_ns_dtype(dt)):
        base = dates.epoch2num(dt.asi8 / 1.0E9)
    else:
        base = dates.date2num(dt)
    return base
247498
def topic_search(text, corpus=doc_topic_vectors, pcaer=pcaer, corpus_text=corpus):
    
    tokens = tokenize(text, vocabulary=corpus.columns)
    tfidf_vector_query = np.array(tfidfer.transform([' '.join(tokens)]).todense())[0]
    topic_vector_query = pcaer.transform([tfidf_vector_query])
    query_series = pd.Series(topic_vector_query, index=corpus.columns)
    return corpus_text[query_series.dot(corpus.T).values.argmax()]
545751
def scores(self, text: str) -> Dict[str, float]:
        
        values = extract(text)
        input_fn = _to_func(([values], []))
        prediction = self._classifier.predict_proba(input_fn=input_fn)
        probabilities = next(prediction).tolist()
        sorted_languages = sorted(self.languages)
        return dict(zip(sorted_languages, probabilities))
378299
def score_hist(df, columns=None, groupby=None, threshold=0.7, stacked=True,
               bins=20, percent=True, alpha=0.33, show=True, block=False, save=False):
    
    df = df if columns is None else df[([] if groupby is None else [groupby]) + list(columns)].copy()
    if groupby is not None or threshold is not None:
        df = groups_from_scores(df, groupby=groupby, threshold=threshold)
    percent = 100. if percent else 1.
    if isinstance(df, pd.core.groupby.DataFrameGroupBy):
        df = df_from_groups(df, columns=columns) * percent
    columns = df.columns if columns is None else columns
    if bins is None:
        bins = 20
    if isinstance(bins, int):
        bins = np.linspace(np.min(df.min()), np.max(df.max()), bins)
    log.debug('bins: {}'.format(bins))
    figs = []

    df.plot(kind='hist', alpha=alpha, stacked=stacked, bins=bins)
    # for col in df.columns:
    #     series = df[col] * percent
    #     log.debug('{}'.format(series))
    #     figs.append(plt.hist(series, bins=bins, alpha=alpha,
    #                 weights=percent * np.ones_like(series) / len(series.dropna()),
    #                 label=stringify(col)))
    plt.legend()
    plt.xlabel('Score (%)')
    plt.ylabel('Percent')
    plt.title('{} Scores for {}'.format(np.sum(df.count()), columns))
    plt.draw()
    if save or not show:
        fig = plt.gcf()
        today = datetime.datetime.today()
        fig.savefig(os.path.join(IMAGES_PATH, 'score_hist_{:04d}-{:02d}-{:02d}_{:02d}{:02d}.jpg'.format(*today.timetuple())))
    if show:
        plt.show(block=block)
    return figs
997261
def get_temporal_score_df(self):
		'''
		

		'''
		scoredf = {}
		tdf = self.term_ranker(self.corpus).get_ranks()
		for cat in sorted(self.corpus.get_categories()):
			if cat >= self.starting_time_step:
				negative_categories = self._get_negative_categories(cat, tdf)
				scores = self.term_scorer.get_scores(
					tdf[cat + ' freq'].astype(int),
					tdf[negative_categories].sum(axis=1)
				)
				scoredf[cat + ' score'] = scores
				scoredf[cat + ' freq'] = tdf[cat + ' freq'].astype(int)
		return pd.DataFrame(scoredf)
996670
def raw_corpus_bleu(hypotheses: Iterable[str], references: Iterable[str], offset: Optional[float] = 0.01) -> float:
    
    return sacrebleu.raw_corpus_bleu(hypotheses, [references], smooth_floor=offset).score / 100.0
778987
def _value_format(self, value, serie, index):
        
        sum_ = serie.points[index][1]
        if serie in self.series and (
                self.stack_from_top
                and self.series.index(serie) == self._order - 1
                or not self.stack_from_top and self.series.index(serie) == 0):
            return super(StackedLine, self)._value_format(value)
        return '%s (+%s)' % (self._y_format(sum_), self._y_format(value))
1119045
def plot(self, entities: List[str]):
        

        ax = self._sensors_num_df[entities].plot(figsize=[12, 6])
        ax.legend(loc="center left", bbox_to_anchor=(1, 0.5))
        ax.set_xlabel("Date")
        ax.set_ylabel("Reading")
        return
694265
def zscore(self, axis=1):
        
        if axis == 1:
            return self.map(lambda x: (x - mean(x)) / std(x))
        elif axis == 0:
            meanval = self.mean().toarray()
            stdval = self.std().toarray()
            return self.map(lambda x: (x - meanval) / stdval)
        else:
            raise Exception('Axis must be 0 or 1')
1304041
def baseline(y_true, y_score=None):
    
    if len(y_true) > 0:
        return np.nansum(y_true)/count(y_true, countna=False)
    else:
        return 0.0
173426
def get_corpus(self):
        

        # 正向判定
        corpus = []
        cd = 0
        tag = None
        for i in range(0, self.init_corpus[0][0]):
            init_unit = self.unit_raw[self.init_corpus[0][0] - i]
            cdm = CDM(init_unit)
            alpha = cdm.get_alpha()

            if cd <= self.cd_min and cdm.NC is not 0:
                tag = True
            if cd > self.cd_max or cdm.NC == 0:
                tag = False
            if cd in range(self.cd_min + 1, self.cd_max) and cdm.NC is not 0:
                if alpha > 0:
                    tag = True
                else:
                    tag = False

            if cdm.NC == 0:
                cd += 1
            else:
                cd = 0

            if tag == True:
                corpus.append(init_unit)
            elif tag == False:
                if alpha < 0 or cd > self.cd_max:
                    break
                else:
                    continue
        corpus = list(reversed(corpus))
        try:
            self.index = self.init_corpus[0][0] - i + 1
        except UnboundLocalError:
            log('err', '正向判定完成，索引定位出错')
            self.index = self.init_corpus[0][0]

        # 反向判定
        cd = 0
        tag = None
        for i in range(1, len(self.unit_raw) - self.init_corpus[0][0]):
            init_unit = self.unit_raw[self.init_corpus[0][0] + i]
            cdm = CDM(init_unit)
            alpha = cdm.get_alpha()

            if cd <= self.cd_min and cdm.NC is not 0:
                tag = True
            if cd > self.cd_max or cdm.NC == 0:
                tag = False
            if cd in range(self.cd_min + 1, self.cd_max) and cdm.NC is not 0:
                if alpha > 0:
                    tag = True
                else:
                    tag = False

            if cdm.NC == 0:
                cd += 1
            else:
                cd = 0

            if tag == True:
                corpus.append(init_unit)
            elif tag == False:
                if alpha < 0 or cd > self.cd_max:
                    break
                else:
                    continue
        log('debug', '\n获取语料库成功:【{}】\n'.format(corpus))
        return ''.join(corpus)