Loads a text file and generate a language profile from its content. The input text file is supposed to be encoded in UTF-8.
@param lang target language name.
@param textFile input text file.
@return Language profile instance
188333
public static LangProfile generate(String lang, File textFile) {
        LangProfile profile = new LangProfile(lang);

        InputStream is = null;
        try {
            is = new BufferedInputStream(new FileInputStream(textFile));
            if (textFile.getName().endsWith(".gz")) is = new GZIPInputStream(is);

            BufferedReader reader = new BufferedReader(new InputStreamReader(is, Charset.forName("UTF-8")));
            String line;
            while ((line = reader.readLine()) != null) {
                TextObject textObject = textObjectFactory.forText(" "+line+" ");
                Util.addCharSequence(profile, textObject);
            }
        } catch (IOException e) {
            throw new RuntimeException("Can't open training database file '" + textFile.getName() + "'", e);
        } finally {
            IOUtils.closeQuietly(is);
        }
        return profile;
    }
474969
public static MonolingualTextValue makeMonolingualTextValue(String text,
			String languageCode) {
		return factory.getMonolingualTextValue(text, languageCode);
	}
531806
@Nonnull
    public static String readFile(@Nonnull File file, @Nonnull Charset charset) throws IOException {
        if (file == null)
            throw new NullPointerException("Given 'file' can not be null");
        if (!file.exists())
            throw new IllegalArgumentException("File " + file.getAbsolutePath() + " does not exist");
        if (file.isDirectory())
            throw new IllegalArgumentException("File " + file.getAbsolutePath() + " can NOT be a directory");

        InputStream in = new BufferedInputStream(new FileInputStream(file));
        ByteArrayOutputStream bos = new ByteArrayOutputStream();
        copy(in, bos);
        return new String(bos.toByteArray(), charset);

    }
471290
public static String readFile(File file, String charsetName) throws IOException {
		byte[] bytes = readAllBytes(file.toPath());
		return new String(bytes, charsetName);
	}
106447
public void generateProfile() {
        File directory = new File(arglist.get(0));
        String lang = arglist.get(1);
        File file = searchFile(directory, lang + "wiki-.*-abstract\\.xml.*");
        if (file == null) {
            System.err.println("Not Found text file : lang = " + lang);
            return;
        }

        try(FileOutputStream outputStream = new FileOutputStream(new File(lang))) {
            LangProfile profile = GenProfile.load(lang, file);
            profile.omitLessFreq();
            new LangProfileWriter().write(profile, outputStream);
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
892736
public LanguageProfile read(InputStream inputStream) throws IOException {
        return OldLangProfileConverter.convert(internalReader.read(inputStream));
    }
570040
protected LinkedHashMap<Pattern, String> load(File propertyFile) {
    LinkedHashMap<Pattern, String> result = new LinkedHashMap<Pattern, String>();
    BufferedReader reader = null;
    try {
      reader =
          new BufferedReader(new InputStreamReader(new FileInputStream(propertyFile), "UTF-8"));
      String line;
      while ((line = reader.readLine()) != null) {
        if (line.length() == 0) {
          continue;
        }
        char firstChar = line.charAt(0);
        if (firstChar == '#' || firstChar == '!') {
          continue;
        }
        int pos = line.indexOf('=');
        if (pos < 0) {
          continue;
        }
        String key = line.substring(0, pos);
        String value = line.substring(pos + 1, line.length());
        int pos2 = line.indexOf('@');
        if (pos2 == 0) {
          key = key.substring(1);
        } else if (pos2 > 0) {
          key = Pattern.quote(key.substring(0, pos2)) + key.substring(pos2);
        }
        result.put(Pattern.compile(key), value);
      }
    } catch (IOException e) {
      throw new GenException(Message.DOMAGEN9001, e, e);
    } finally {
      IOUtil.close(reader);
    }
    return result;
  }
1265634
public static Pair<InMemoryLookupTable, VocabCache> loadTxt(File vectorsFile)
            throws IOException, UnsupportedEncodingException {

        AbstractCache cache = new AbstractCache<>();
        BufferedReader reader = createReader(vectorsFile);
        LineIterator iter = IOUtils.lineIterator(reader);
        String line = null;
        boolean hasHeader = false;
        if (iter.hasNext()) {
            line = iter.nextLine(); // skip header line
            //look for spaces
            if (!line.contains(" ")) {
                log.debug("Skipping first line");
                hasHeader = true;
            } else {
                // we should check for something that looks like proper word vectors here. i.e: 1 word at the 0 position, and bunch of floats further
                String[] split = line.split(" ");
                try {
                    long[] header = new long[split.length];
                    for (int x = 0; x < split.length; x++) {
                        header[x] = Long.parseLong(split[x]);
                    }
                    if (split.length < 4)
                        hasHeader = true;
                    // now we know, if that's all ints - it's just a header
                    // [0] - number of words
                    // [1] - vectorSize
                    // [2] - number of documents <-- DL4j-only value
                    if (split.length == 3)
                        cache.incrementTotalDocCount(header[2]);

                    printOutProjectedMemoryUse(header[0], (int) header[1], 1);

                    hasHeader = true;

                    try {
                        reader.close();
                    } catch (Exception ex) {
                    }
                } catch (Exception e) {
                    // if any conversion exception hits - that'll be considered header
                    hasHeader = false;

                }
            }

        }

        //reposition buffer to be one line ahead
        if (hasHeader) {
            line = "";
            iter.close();
            //reader = new BufferedReader(new FileReader(vectorsFile));
            reader = createReader(vectorsFile);
            iter = IOUtils.lineIterator(reader);
            iter.nextLine();
        }

        List<INDArray> arrays = new ArrayList<>();
        long[] vShape = new long[]{1, -1};
        while (iter.hasNext()) {
            if (line.isEmpty())
                line = iter.nextLine();
            String[] split = line.split(" ");
            String word = decodeB64(split[0]); //split[0].replaceAll(whitespaceReplacement, " ");
            VocabWord word1 = new VocabWord(1.0, word);

            word1.setIndex(cache.numWords());

            cache.addToken(word1);

            cache.addWordToIndex(word1.getIndex(), word);

            cache.putVocabWord(word);

            float[] vector = new float[split.length - 1];

            for (int i = 1; i < split.length; i++) {
                vector[i - 1] = Float.parseFloat(split[i]);
            }

            vShape[1] = vector.length;
            INDArray row = Nd4j.create(vector, vShape);

            arrays.add(row);

            // workaround for skipped first row
            line = "";
        }

        INDArray syn = Nd4j.vstack(arrays);

        InMemoryLookupTable lookupTable =
                (InMemoryLookupTable) new InMemoryLookupTable.Builder().vectorLength(arrays.get(0).columns())
                        .useAdaGrad(false).cache(cache).useHierarchicSoftmax(false).build();

        lookupTable.setSyn0(syn);

        iter.close();

        try {
            reader.close();
        } catch (Exception e) {
        }

        return new Pair<>(lookupTable, (VocabCache) cache);
    }
487336
public static List<String> readUtf8Lines(File file) throws IORuntimeException {
		return readLines(file, CharsetUtil.CHARSET_UTF_8);
	}
1055245
public String getLanguage (String text) {
    String language = null;
    TextObject textObject = textObjectFactory.forText(text);
    com.google.common.base.Optional<LdLocale> lang = languageDetector.detect(textObject);
    
    if (lang.isPresent()) {
      language = lang.get().toString();
    }
    
    return language;
  }
597180
public final URL getBinaryTaggerDict(final String lang, final String resourcesDirectory) {
    return resourcesDirectory == null
      ? getBinaryTaggerDictFromResources(lang)
      : getBinaryTaggerDictFromDirectory(lang, resourcesDirectory);
  }
180350
public static String toLanguageTag(Locale locale) {
        return locale.getLanguage() + (hasText(locale.getCountry()) ? "-" + locale.getCountry() : EMPTY);
    }
849164
public List<LanguageProfile> read(String profileDirectory, Collection<String> profileFileNames) throws IOException {
        return read(LanguageProfileReader.class.getClassLoader(), profileDirectory, profileFileNames);
    }
557587
public static String readString(File file, Charset charset) throws IOException {
        if (file == null || !file.isFile()) {
            return null;
        }
        StringBuilder fileContent = new StringBuilder();

        BufferedReader reader = null;
        try {
            InputStreamReader is = new InputStreamReader(new FileInputStream(file), charset);
            reader = new BufferedReader(is);
            String line = null;
            while ((line = reader.readLine()) != null) {
                if (!fileContent.toString().equals("")) {
                    fileContent.append("\r\n");
                }
                fileContent.append(line);
            }
            reader.close();
            return fileContent.toString();
        } finally {
            closeQuietly(reader);
        }
    }
529212
public String loadTextFile(File source) throws IOException
	{
		StringBuffer _result = new StringBuffer();
		append(source, _result);
		return _result.toString();
	}
470813
public static File zip(File srcFile, Charset charset) throws UtilException {
		final File zipFile = FileUtil.file(srcFile.getParentFile(), FileUtil.mainName(srcFile) + ".zip");
		zip(zipFile, charset, false, srcFile);
		return zipFile;
	}
90582
def read_text_file(filename, encoding="utf-8"):
    
    try:
        with open(filename, 'r', encoding) as f:
            r = f.read()
    except TypeError:
        with open(filename, 'r') as f:
            r = f.read()
    return r