Pickles the Corpus object in a file in tgt_dir.
436786
def pickle(self) -> None:
        

        pickle_path = self.tgt_dir / "corpus.p"
        logger.debug("pickling %r object and saving it to path %s", self, pickle_path)
        with pickle_path.open("wb") as f:
            pickle.dump(self, f)
540787
def make_data_splits(label_type, train_rec_type="text_and_wordlist", max_samples=1000,
                     seed=0, tgt_dir=TGT_DIR):
    

    feat_dir = os.path.join(tgt_dir, "feat")
    test_prefix_fn=os.path.join(tgt_dir, "test_prefixes.txt")
    valid_prefix_fn=os.path.join(tgt_dir, "valid_prefixes.txt")
    with open(test_prefix_fn) as f:
        prefixes = f.readlines()
        test_prefixes = [("TEXT/" + prefix).strip() for prefix in prefixes]
    with open(valid_prefix_fn) as f:
        prefixes = f.readlines()
        valid_prefixes = [("TEXT/" + prefix).strip() for prefix in prefixes]

    label_dir = os.path.join(tgt_dir, "label")
    prefixes = get_story_prefixes(label_type, label_dir=label_dir)
    prefixes = list(set(prefixes) - set(valid_prefixes))
    prefixes = list(set(prefixes) - set(test_prefixes))
    prefixes = utils.filter_by_size(
        feat_dir, prefixes, "fbank", max_samples)

    if train_rec_type == "text":
        train_prefixes = prefixes
    else:
        wordlist_prefixes = [prefix for prefix in os.listdir(os.path.join(label_dir, "WORDLIST"))
                             if prefix.endswith("phonemes")]
        wordlist_prefixes = [os.path.splitext(os.path.join("WORDLIST", prefix))[0]
                             for prefix in wordlist_prefixes]
        wordlist_prefixes = utils.filter_by_size(
                feat_dir, wordlist_prefixes, "fbank", max_samples)
        if train_rec_type == "wordlist":
            prefixes = wordlist_prefixes
        elif train_rec_type == "text_and_wordlist":
            prefixes.extend(wordlist_prefixes)
        else:
            raise PersephoneException("train_rec_type='%s' not supported." % train_rec_type)
        train_prefixes = prefixes
    random.seed(seed)
    random.shuffle(train_prefixes)

    return train_prefixes, valid_prefixes, test_prefixes
758684
def corpus_page_generator(corpus_files, tmp_dir, max_page_size_exp):
  
  for remote_filepath in corpus_files:

    filepath = maybe_copy_file_to_directory(remote_filepath, tmp_dir)
    tf.logging.info("Reading from " + filepath)

    command = ["7z", "x", "-so", filepath]
    tf.logging.info("Running command: %s", command)

    p = subprocess.Popen(command, stdout=subprocess.PIPE, bufsize=-1)

    for page in file_page_generator(p.stdout, 2**max_page_size_exp):
      yield page
406271
def gather_data(mypath, key="width"):
    
    filenames = [join(mypath, f) for f in listdir(mypath) if isfile(join(mypath, f))]
    filenames = [f for f in filenames if f.endswith(".p")] 
    W = []
    for filename in filenames:
        with open(filename, 'r') as f:
            data = pickle.load(f)
            W.append(np.expand_dims(data[key], axis=-1) )

    W = np.concatenate(W, axis=-1)
    return W
251231
def load_classifier():
    """"""
    path = os.path.join(l.TOPDIR, 'clf.pickle')
    obj = pickle.load(open(path, 'r'))

    return obj['tfidf_model'], obj['clf'], obj['target_names']
780382
def get_corpus_line(self, item):
        
        t1 = ""
        t2 = ""
        assert item < self.corpus_lines
        if self.on_memory:
            sample = self.sample_to_doc[item]
            t1 = self.all_docs[sample["doc_id"]][sample["line"]]
            t2 = self.all_docs[sample["doc_id"]][sample["line"]+1]
            # used later to avoid random nextSentence from same doc
            self.current_doc = sample["doc_id"]
            return t1, t2
        else:
            if self.line_buffer is None:
                # read first non-empty line of file
                while t1 == "" :
                    t1 = next(self.file).strip()
                    t2 = next(self.file).strip()
            else:
                # use t2 from previous iteration as new t1
                t1 = self.line_buffer
                t2 = next(self.file).strip()
                # skip empty rows that are used for separating documents and keep track of current doc id
                while t2 == "" or t1 == "":
                    t1 = next(self.file).strip()
                    t2 = next(self.file).strip()
                    self.current_doc = self.current_doc+1
            self.line_buffer = t2

        assert t1 != ""
        assert t2 != ""
        return t1, t2
460587
def csv_to_pickle(path=ROOT / "raw", clean=False):
    """"""

    for file in os.listdir(path):
        base, ext = os.path.splitext(file)
        if ext != ".csv":
            continue

        LOG(f"converting {file} to pickle")
        df = pd.read_csv(path / file, low_memory=True)
        WRITE_DF(df, path / (base + "." + FILE_EXTENSION), **WRITE_DF_OPTS)
        if clean:
            os.remove(path / file)
            LOG(f"removed {file}")
670023
def _reset_corpus_iterator(self):
        
        self.__context = etree.iterparse(self.urml_file, events=('end',),
                                         tag='document', recover=False)
1266201
def load_corpus(*data_file_paths):
    
    for file_path in data_file_paths:
        corpus = []
        corpus_data = read_corpus(file_path)

        conversations = corpus_data.get('conversations', [])
        corpus.extend(conversations)

        categories = corpus_data.get('categories', [])

        yield corpus, categories, file_path
478051
def load_fegg(filepath, update=True):
    
    try:
        egg = FriedEgg(**dd.io.load(filepath))
    except ValueError as e:
        print(e)
        # if error, try loading old format
        with open(filepath, 'rb') as f:
            egg = pickle.load(f)

    if update:
        return egg.crack()
    else:
        return egg
937056
def from_pickle(cls, filename):
        """"""
        with open(filename, 'rb') as f:
            mesh = pickle.load(f).copy()
        return mesh
360501
def load(file_or_path):
    
    from pickle import UnpicklingError
    _python3 = True
    try: 
        import cPickle as pickle
        _python3 = False
    except ImportError: #python3
        import pickle
   
    try:
        if _python3:
            strcl = str
            p3kw = dict(encoding='latin1')
            return _unpickle(file_or_path, pickle, strcl, p3kw)
        else:
            strcl = basestring
            p3kw = {}
            return _unpickle(file_or_path, pickle, strcl, p3kw)
    
    except UnpicklingError: # pragma: no coverage
        import pickle
        return _unpickle(file_or_path, pickle, strcl, p3kw)
232943
def extract_from_files(
        files: List[Path],
        languages: Dict[str, List[str]]) -> DataSet:
    
    enumerator = enumerate(sorted(languages.items()))
    rank_map = {ext: rank for rank, (_, exts) in enumerator for ext in exts}

    with multiprocessing.Pool(initializer=_process_init) as pool:
        file_iterator = ((path, rank_map) for path in files)
        arrays = _to_arrays(pool.starmap(_extract_features, file_iterator))

    LOGGER.debug("Extracted arrays count: %d", len(arrays[0]))
    return arrays
252223
def save_ldamodel_to_pickle(picklefile, model, vocab, doc_labels, dtm=None, **kwargs):
    """"""
    pickle_data({'model': model, 'vocab': vocab, 'doc_labels': doc_labels, 'dtm': dtm}, picklefile)
1075426
def import_training_data(self,
            positive_corpus_file=os.path.join(os.path.dirname(__file__),
                "positive.txt"),
            negative_corpus_file=os.path.join(os.path.dirname(__file__),
                "negative.txt")
            ):
        

        positive_corpus = open(positive_corpus_file)
        negative_corpus = open(negative_corpus_file)

        # for line in positive_corpus: 
        #     self.training_data.append((line, True))

        # for line in negative_corpus: 
        #     self.training_data.append((line, False))

        # The following code works. Need to profile this to see if this is an
        # improvement over the code above. 
        positive_training_data = list(map(lambda x: (x, True), positive_corpus))
        negative_training_data = list(map(lambda x: (x, False), negative_corpus))
        self.training_data = positive_training_data + negative_training_data
645870
def read_object_from_pickle(desired_type: Type[T], file_path: str, encoding: str,
                            fix_imports: bool = True, errors: str = 'strict', *args, **kwargs) -> Any:
    
    import pickle
    file_object = open(file_path, mode='rb')
    try:
        return pickle.load(file_object, fix_imports=fix_imports, encoding=encoding, errors=errors)
    finally:
        file_object.close()
537948
def pickle_dump(self):
        """"""
        with open(os.path.join(self.workdir, self.PICKLE_FNAME), mode="wb") as fh:
            pickle.dump(self, fh)