Read all the files and tokenize their text into words and
        accumulate all the words in a list.
437487
def start_indexing(self):
        
        for filepath in self.filepaths:
            with open(filepath) as fp:
                blob = fp.read()
                self.words.extend(self.tokenize(blob))
782844
def from_vocabfile(filename):
    
    word_count = [x.strip().split() for x in _open(filename, 'r').read().splitlines()]
    word_count = {w:int(c) for w,c in word_count}
    return CountedVocabulary(word_count=word_count)
893632
def read_files(filenames):
    """"""
    if isinstance(filenames, list):
        for filename in filenames:
            with open(filename, 'r') as infile:
                return infile.read()
    else:
        with open(filenames, 'r') as infile:
            return infile.read()
188482
def words(self: object, fileids: str):
        
        for sent in self.sents(fileids):
            for token in word_tokenize(sent):
                yield token
34680
function _readAllFiles(sDir = '', aWhitelist = []) {
  // read all files in current directory
  const aFiles = fs.readdirSync(sDir)

  // loop at all files
  const aContents = aFiles.reduce((aContents, sFileName) => {
    // get file stats
    const oFile = fs.statSync(`${sDir}/${sFileName}`)
    if (oFile.isDirectory()) {
      // append files of directory to list
      return aContents.concat(_readAllFiles(`${sDir}/${sFileName}`))
    }

    // append file content to list (if contained in whitelist)
    return aWhitelist.length === 0 || aWhitelist.indexOf(sFileName) !== -1
      ? aContents.concat(fs.readFileSync(`${sDir}/${sFileName}`, 'utf8'))
      : aContents
  }, [])

  return aContents
}
912016
def merge(files, accumulate=True):
        
        metrics = {}
        for f in files:
            parts = os.path.basename(f).split('_')
            typ = parts[0]
            d = MmapedDict(f, read_mode=True)
            for key, value in d.read_all_values():
                metric_name, name, labels = json.loads(key)
                labels_key = tuple(sorted(labels.items()))

                metric = metrics.get(metric_name)
                if metric is None:
                    metric = Metric(metric_name, 'Multiprocess metric', typ)
                    metrics[metric_name] = metric

                if typ == 'gauge':
                    pid = parts[2][:-3]
                    metric._multiprocess_mode = parts[1]
                    metric.add_sample(name, labels_key + (('pid', pid),), value)
                else:
                    # The duplicates and labels are fixed in the next for.
                    metric.add_sample(name, labels_key, value)
            d.close()

        for metric in metrics.values():
            samples = defaultdict(float)
            buckets = {}
            for s in metric.samples:
                name, labels, value = s.name, s.labels, s.value
                if metric.type == 'gauge':
                    without_pid = tuple(l for l in labels if l[0] != 'pid')
                    if metric._multiprocess_mode == 'min':
                        current = samples.setdefault((name, without_pid), value)
                        if value < current:
                            samples[(s.name, without_pid)] = value
                    elif metric._multiprocess_mode == 'max':
                        current = samples.setdefault((name, without_pid), value)
                        if value > current:
                            samples[(s.name, without_pid)] = value
                    elif metric._multiprocess_mode == 'livesum':
                        samples[(name, without_pid)] += value
                    else:  # all/liveall
                        samples[(name, labels)] = value

                elif metric.type == 'histogram':
                    bucket = tuple(float(l[1]) for l in labels if l[0] == 'le')
                    if bucket:
                        # _bucket
                        without_le = tuple(l for l in labels if l[0] != 'le')
                        buckets.setdefault(without_le, {})
                        buckets[without_le].setdefault(bucket[0], 0.0)
                        buckets[without_le][bucket[0]] += value
                    else:
                        # _sum/_count
                        samples[(s.name, labels)] += value

                else:
                    # Counter and Summary.
                    samples[(s.name, labels)] += value

            # Accumulate bucket values.
            if metric.type == 'histogram':
                for labels, values in buckets.items():
                    acc = 0.0
                    for bucket, value in sorted(values.items()):
                        sample_key = (
                            metric.name + '_bucket',
                            labels + (('le', floatToGoString(bucket)),),
                        )
                        if accumulate:
                            acc += value
                            samples[sample_key] = acc
                        else:
                            samples[sample_key] = value
                    if accumulate:
                        samples[(metric.name + '_count', labels)] = acc

            # Convert to correct sample format.
            metric.samples = [Sample(name_, dict(labels), value) for (name_, labels), value in samples.items()]
        return metrics.values()
756186
def get_common(self, filename):
        '''  '''
        word_list = []
        words = open(filename)
        for word in words.readlines():
            word_list.append(word.strip())
        return word_list
789708
def combine_word_list(word_list):
    
    bag_of_words = collections.defaultdict(int)

    for word in word_list:
        bag_of_words[word] += 1

    return bag_of_words
1158380
def index_map(data):
  """"""
  (entry, text_fn) = data
  text = text_fn()

  logging.debug("Got %s", entry.filename)
  for s in split_into_sentences(text):
    for w in split_into_words(s.lower()):
      yield (w, entry.filename)
378136
def load_common(self, wlist):
        """"""
        self.com_dict = os.path.join(self.base_dir, 'dicts', 'EN_vocab.txt')
        with open(self.com_dict) as words_file:
            data = words_file.read()
        self.common_words = set(data.splitlines())
        if wlist:
            new_words = ''
            for wl in wlist:
                with open(wl) as f:
                    new_words += f.read()
            if new_words:
                new_dict = set(data.splitlines())
                self.common_words.update(new_dict)
1174921
public static void load(String dir, List<String[]> sentences, List<PennTreebankPOS[]> labels) {
        List<File> files = new ArrayList<>();
        walkin(new File(dir), files);

        for (File file : files) {
            try {
                FileInputStream stream = new FileInputStream(file);
                BufferedReader reader = new BufferedReader(new InputStreamReader(stream));
                String line = null;
                List<String> sent = new ArrayList<>();
                List<PennTreebankPOS> label = new ArrayList<>();
                while ((line = reader.readLine()) != null) {
                    line = line.trim();
                    if (line.isEmpty()) {
                        if (!sent.isEmpty()) {
                            sentences.add(sent.toArray(new String[sent.size()]));
                            labels.add(label.toArray(new PennTreebankPOS[label.size()]));
                            sent.clear();
                            label.clear();
                        }
                    } else if (!line.startsWith("===") && !line.startsWith("*x*")) {
                        String[] words = line.split("\\s");
                        for (String word : words) {
                            String[] w = word.split("/");
                            if (w.length == 2) {
                                sent.add(w[0]);
                                
                                int pos = w[1].indexOf('|');
                                String tag = pos == -1 ? w[1] : w[1].substring(0, pos);
                                if (tag.equals("PRP$R")) tag = "PRP$";
                                if (tag.equals("JJSS")) tag = "JJS";
                                label.add(PennTreebankPOS.getValue(tag));
                            }
                        }
                    }
                }
                
                if (!sent.isEmpty()) {
                    sentences.add(sent.toArray(new String[sent.size()]));
                    labels.add(label.toArray(new PennTreebankPOS[label.size()]));
                    sent.clear();
                    label.clear();
                }
                
                reader.close();
            } catch (Exception e) {
                logger.error("Failed to load training data {}", file, e);
            }
        }
    }
1299592
def read_content(path: str, limit: Optional[int] = None) -> Iterator[List[str]]:
    
    with smart_open(path) as indata:
        for i, line in enumerate(indata):
            if limit is not None and i == limit:
                break
            yield list(get_tokens(line))
402541
def process_files():
    

    with enlighten.Counter(total=100, desc='Simple', unit='ticks') as pbar:
        for num in range(100):  # pylint: disable=unused-variable
            time.sleep(0.05)
            pbar.update()
1156853
def load(self, f, skip):
        """"""
        array = self.get()
        counter = 0
        counter_limit = array.size
        convert = array.dtype.type
        while counter < counter_limit:
            line = f.readline()
            words = line.split()
            for word in words:
                if counter >= counter_limit:
                    raise FileFormatError("Wrong array data: too many values.")
                if not skip:
                    array.flat[counter] = convert(word)
                counter += 1
731242
private void readAll() {
        final List<CompletableFuture<Void>> futures = new ArrayList<>();
        for(File file : directory.listFiles()) {
            
            final Runnable loadFile = () -> {
                if(file.getName().endsWith(SUFFIX)) {
                    try(FileReader reader = new FileReader(file)) {
                        Properties props = new Properties();
                        props.load(reader);
                        Map.Entry<Key,HttpCookie> entry = fromProperties(props);
                        if(entry != null) {
                            add(entry.getKey(), entry.getValue());
                        }
                        else {
                            file.delete();
                        }
                    }
                    catch(IOException ioe) {
                        throw new RuntimeException(ioe);
                    }
                } };

            futures.add(CompletableFuture.runAsync(loadFile, executor));
        }

        try {
            CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).get();
        }
        catch(InterruptedException | ExecutionException e) {
            throw new RuntimeException(e);
        }
    }
1054398
def load_word_file(filename):
    """"""
    words_file = resource_filename(__name__, "words/%s" % filename)
    handle = open(words_file, 'r')
    words = handle.readlines()
    handle.close()
    return words
724726
def extract_words(lines):
    
    for line in lines:
        for word in re.findall(r"\w+", line):
            yield word