Collects text into translation source files
428174
def make_messages(context: Context, javascript=False, fuzzy=False):
    
    kwargs = {
        'all': True,
        'keep_pot': True,
        'no_wrap': True,
    }
    if fuzzy:
        kwargs['allow_fuzzy'] = True
    if javascript:
        kwargs.update(domain='djangojs', ignore_patterns=['*.bundle.js'])
    with in_dir(context.app.django_app_name):
        return context.management_command('makemessages', **kwargs)
232943
def extract_from_files(
        files: List[Path],
        languages: Dict[str, List[str]]) -> DataSet:
    
    enumerator = enumerate(sorted(languages.items()))
    rank_map = {ext: rank for rank, (_, exts) in enumerator for ext in exts}

    with multiprocessing.Pool(initializer=_process_init) as pool:
        file_iterator = ((path, rank_map) for path in files)
        arrays = _to_arrays(pool.starmap(_extract_features, file_iterator))

    LOGGER.debug("Extracted arrays count: %d", len(arrays[0]))
    return arrays
69041
def get_translation(modname, dirname=None):
    """"""
    if dirname is None:
        dirname = modname

    def translate_dumb(x):
        """Dumb function to not use translations."""
        if not is_unicode(x):
            return to_text_string(x, "utf-8")
        return x

    locale_path = get_module_data_path(dirname, relpath="locale",
                                       attr_name='LOCALEPATH')

    # If LANG is defined in Ubuntu, a warning message is displayed,
    # so in Unix systems we define the LANGUAGE variable.
    language = load_lang_conf()
    if os.name == 'nt':
        # Trying to set LANG on Windows can fail when Spyder is
        # run with admin privileges.
        # Fixes issue 6886
        try:
            os.environ["LANG"] = language      # Works on Windows
        except Exception:
            return translate_dumb
    else:
        os.environ["LANGUAGE"] = language  # Works on Linux
 
    import gettext
    try:
        _trans = gettext.translation(modname, locale_path, codeset="utf-8")
        lgettext = _trans.lgettext
        def translate_gettext(x):
            if not PY3 and is_unicode(x):
                x = x.encode("utf-8")
            y = lgettext(x)
            if is_text_string(y) and PY3:
                return y
            else:
                return to_text_string(y, "utf-8")
        return translate_gettext
    except Exception:
        return translate_dumb
530324
def get_strings_to_translate(self, po):
        
        strings = []
        for index, entry in enumerate(po):
            if not self.need_translate(entry):
                continue
            strings.append(humanize_placeholders(entry.msgid))
            if entry.msgid_plural:
                strings.append(humanize_placeholders(entry.msgid_plural))
        return strings
152987
function _extractTranslation(regexName, regex, content, file, lineNumber, results) {
      var r;
      _log.debug("---------------------------------------------------------------------------------------------------");
      _log.debug('Process extraction with regex : "' + regexName + '"');
      _log.debug(regex);
      regex.lastIndex = 0;
      while ((r = regex.exec(content)) !== null) {

        // Result expected [STRING, KEY, SOME_REGEX_STUF]
        // Except for plural hack [STRING, KEY, ARRAY_IN_STRING]
        if (r.length >= 2) {
          var translationKey, evalString;
          var translationDefaultValue = "";

          switch (regexName) {
            case 'HtmlDirectivePluralFirst':
              var tmp = r[1];
              r[1] = r[2];
              r[2] = tmp;
              /* falls through */
            case 'HtmlDirectivePluralLast':
              evalString = eval(r[2]);
              if (_.isArray(evalString) && evalString.length >= 2) {
                translationDefaultValue = "{NB, plural, one{" + evalString[0] + "} other{" + evalString[1] + "}" + (evalString[2] ? ' ' + evalString[2] : '');
              }
              translationKey = r[1].trim();
              break;
            default:
              translationKey = r[1].trim();
          }

          // Avoid empty translation
          if (translationKey === "") {
            return;
          }

          switch (regexName) {
            case "commentSimpleQuote":
            case "HtmlFilterSimpleQuote":
            case "JavascriptServiceSimpleQuote":
            case "JavascriptServiceInstantSimpleQuote":
            case "JavascriptFilterSimpleQuote":
            case "HtmlNgBindHtml":
              translationKey = translationKey.replace(/\\\'/g, "'");
              break;
            case "commentDoubleQuote":
            case "HtmlFilterDoubleQuote":
            case "JavascriptServiceDoubleQuote":
            case "JavascriptServiceInstantDoubleQuote":
            case "JavascriptFilterDoubleQuote":
              translationKey = translationKey.replace(/\\\"/g, '"');
              break;
            case "JavascriptServiceArraySimpleQuote":
            case "JavascriptServiceArrayDoubleQuote":
              var key;

              if(regexName === "JavascriptServiceArraySimpleQuote") {
                key = translationKey.replace(/'/g, '');
              } else {
                key = translationKey.replace(/"/g, '');
              }
              key = key.replace(/[\][]/g, '');
              key = key.split(',');

              key.forEach(function(item){
                item = item.replace(/\\\"/g, '"').trim();
                if (results[item]) {
                  results[item].sourceFiles.push({
                    file: file,
                    line: lineNumber
                  });
                } else {
                  results[item] = {
                    defaultValue: translationDefaultValue,
                    sourceFiles: [{
                      file: file,
                      line: lineNumber
                    }]
                  };
                }
              });
              break;
          }

          if( regexName !== "JavascriptServiceArraySimpleQuote" &&
              regexName !== "JavascriptServiceArrayDoubleQuote") {
            if (results[translationKey]) {
              results[translationKey].sourceFiles.push({
                file: file,
                line: lineNumber
              });
            } else {
              results[ translationKey ] = {
                defaultValue: translationDefaultValue,
                sourceFiles: [{
                  file: file,
                  line: lineNumber
                }]
              };
            }
          }
        }
      }
    }
940530
private function extractGettextStrings()
    {
        $translation = null;
        $translationObjects = array();
        $lookupDirectories = array(
            Strata::getVendorPath() . 'strata-mvc' . DIRECTORY_SEPARATOR . 'strata' . DIRECTORY_SEPARATOR . 'src',
            Strata::getSrcPath(),
            Strata::getThemesPath(),
        );

        foreach ($lookupDirectories as $directory) {
            $translationObjects = $this->recurseThroughDirectory($directory);

            // Merge all translation objects into a bigger one
            foreach ($translationObjects as $t) {
                if (is_null($translation)) {
                    $translation = $t;
                } else {
                    $translation->mergeWith($t);
                }
            }
        }

        return $translation;
    }
642550
def _load_sentence_list(self, path):
        

        result = {}

        for entry in textfile.read_separated_lines_generator(path, separator='\t', max_columns=3):
            if self.include_languages is None or entry[1] in self.include_languages:
                result[entry[0]] = entry[1:]

        return result
922905
def load_translation_data(dataset, bleu, args):
    
    src_lang, tgt_lang = args.src_lang, args.tgt_lang
    if dataset == 'IWSLT2015':
        common_prefix = 'IWSLT2015_{}_{}_{}_{}'.format(src_lang, tgt_lang,
                                                       args.src_max_len, args.tgt_max_len)
        data_train = nlp.data.IWSLT2015('train', src_lang=src_lang, tgt_lang=tgt_lang)
        data_val = nlp.data.IWSLT2015('val', src_lang=src_lang, tgt_lang=tgt_lang)
        data_test = nlp.data.IWSLT2015('test', src_lang=src_lang, tgt_lang=tgt_lang)
    elif dataset == 'WMT2016BPE':
        common_prefix = 'WMT2016BPE_{}_{}_{}_{}'.format(src_lang, tgt_lang,
                                                        args.src_max_len, args.tgt_max_len)
        data_train = nlp.data.WMT2016BPE('train', src_lang=src_lang, tgt_lang=tgt_lang)
        data_val = nlp.data.WMT2016BPE('newstest2013', src_lang=src_lang, tgt_lang=tgt_lang)
        data_test = nlp.data.WMT2016BPE('newstest2014', src_lang=src_lang, tgt_lang=tgt_lang)
    elif dataset == 'WMT2014BPE':
        common_prefix = 'WMT2014BPE_{}_{}_{}_{}'.format(src_lang, tgt_lang,
                                                        args.src_max_len, args.tgt_max_len)
        data_train = nlp.data.WMT2014BPE('train', src_lang=src_lang, tgt_lang=tgt_lang)
        data_val = nlp.data.WMT2014BPE('newstest2013', src_lang=src_lang, tgt_lang=tgt_lang)
        data_test = nlp.data.WMT2014BPE('newstest2014', src_lang=src_lang, tgt_lang=tgt_lang,
                                        full=args.full)
    elif dataset == 'TOY':
        common_prefix = 'TOY_{}_{}_{}_{}'.format(src_lang, tgt_lang,
                                                 args.src_max_len, args.tgt_max_len)
        data_train = _dataset.TOY('train', src_lang=src_lang, tgt_lang=tgt_lang)
        data_val = _dataset.TOY('val', src_lang=src_lang, tgt_lang=tgt_lang)
        data_test = _dataset.TOY('test', src_lang=src_lang, tgt_lang=tgt_lang)
    else:
        raise NotImplementedError
    src_vocab, tgt_vocab = data_train.src_vocab, data_train.tgt_vocab
    data_train_processed = _load_cached_dataset(common_prefix + '_train')
    if not data_train_processed:
        data_train_processed = process_dataset(data_train, src_vocab, tgt_vocab,
                                               args.src_max_len, args.tgt_max_len)
        _cache_dataset(data_train_processed, common_prefix + '_train')
    data_val_processed = _load_cached_dataset(common_prefix + '_val')
    if not data_val_processed:
        data_val_processed = process_dataset(data_val, src_vocab, tgt_vocab)
        _cache_dataset(data_val_processed, common_prefix + '_val')
    if dataset == 'WMT2014BPE':
        filename = common_prefix + '_' + str(args.full) + '_test'
    else:
        filename = common_prefix + '_test'
    data_test_processed = _load_cached_dataset(filename)
    if not data_test_processed:
        data_test_processed = process_dataset(data_test, src_vocab, tgt_vocab)
        _cache_dataset(data_test_processed, filename)
    if bleu == 'tweaked':
        fetch_tgt_sentence = lambda src, tgt: tgt.split()
        val_tgt_sentences = list(data_val.transform(fetch_tgt_sentence))
        test_tgt_sentences = list(data_test.transform(fetch_tgt_sentence))
    elif bleu == '13a' or bleu == 'intl':
        fetch_tgt_sentence = lambda src, tgt: tgt
        if dataset == 'WMT2016BPE':
            val_text = nlp.data.WMT2016('newstest2013', src_lang=src_lang, tgt_lang=tgt_lang)
            test_text = nlp.data.WMT2016('newstest2014', src_lang=src_lang, tgt_lang=tgt_lang)
        elif dataset == 'WMT2014BPE':
            val_text = nlp.data.WMT2014('newstest2013', src_lang=src_lang, tgt_lang=tgt_lang)
            test_text = nlp.data.WMT2014('newstest2014', src_lang=src_lang, tgt_lang=tgt_lang,
                                         full=args.full)
        elif dataset == 'IWSLT2015' or dataset == 'TOY':
            val_text = data_val
            test_text = data_test
        else:
            raise NotImplementedError
        val_tgt_sentences = list(val_text.transform(fetch_tgt_sentence))
        test_tgt_sentences = list(test_text.transform(fetch_tgt_sentence))
    else:
        raise NotImplementedError
    return data_train_processed, data_val_processed, data_test_processed, \
           val_tgt_sentences, test_tgt_sentences, src_vocab, tgt_vocab
1280749
public function read()
    {

        $file = $this->getFilePath();
        if (!file_exists($file)) {
            throw new tao_helpers_translation_TranslationException("The translation file '${file}' does not exist.");
        }

        // Create the translation file.
        $tf = new tao_helpers_translation_POFile();

        $fc = implode('',file($file));

        $matched = preg_match_all('/((?:#[\.:,\|]{0,1}\s+(?:.*?)\\n)*)'.
            '(msgctxt\s+(?:"(?:[^"]|\\\\")*?"\s*)+)?'.
            '(msgid\s+(?:"(?:[^"]|\\\\")*?"\s*)+)\s+' .
            '(msgstr\s+(?:"(?:[^"]|\\\\")*?(?<!\\\)"\s*)+)/',
            $fc, $matches);

        preg_match('/sourceLanguage: (.*?)\\n/s', $fc, $sourceLanguage);
        preg_match('/targetLanguage: (.*?)\\n/s', $fc, $targetLanguage);

        if (count($sourceLanguage)) {
            $tf->setSourceLanguage(substr($sourceLanguage[1],0,5));
        }
        if (count($targetLanguage)) {
            $tf->setTargetLanguage(substr($targetLanguage[1],0,5));
        }

        if ($matched) {

            for ($i = 0; $i < $matched; $i++) {

                $annotations = $matches[1][$i];
                $msgctxt = preg_replace('/\s*msgctxt\s*"(.*)"\s*/s','\\1',$matches[2][$i]);
                $msgid = preg_replace('/\s*msgid\s*"(.*)"\s*/s','\\1',$matches[3][$i]);
                $msgstr = preg_replace('/\s*msgstr\s*"(.*)"\s*/s','\\1',$matches[4][$i]);

                // Do not include meta data as a translation unit..
                if ($msgid !== ''){

                    // Sanitze the strings.
                    $msgid = tao_helpers_translation_POUtils::sanitize($msgid);
                    $msgstr = tao_helpers_translation_POUtils::sanitize($msgstr);
                    $msgctxt = tao_helpers_translation_POUtils::sanitize($msgctxt);
                    $tu = new tao_helpers_translation_POTranslationUnit();

                    // Set up source & target.
                    $tu->setSource($msgid);
                    if ($msgstr !== '') {
                        $tu->setTarget($msgstr);
                    }
                    if ($msgctxt){
                        $tu->setContext($msgctxt);
                    }

                    // Deal with annotations
                    $annotations = tao_helpers_translation_POUtils::unserializeAnnotations($annotations);
                    foreach ($annotations as $name => $value){
                        $tu->addAnnotation($name, $value);
                    }

                    $tf->addTranslationUnit($tu);
                }
			}
		}

		$this->setTranslationFile($tf);

    }
295525
def lazy_translations(cls):
        """"""
        return {
            cloudfiles.errors.NoSuchContainer: errors.NoContainerException,
            cloudfiles.errors.NoSuchObject: errors.NoObjectException,
        }
1303663
def extract_keys_from_files(localizable_files)
      keys_from_file = {}
      localizable_files.each do |file|
        lines = File.readlines(file)

        # Grab just the keys, we don't need the translation
        keys = lines.map { |e| e.split("=").first }
        # Filter newlines and comments
        keys = keys.select do |e|
          e != "\n" && !e.start_with?("/*") && !e.start_with?("//")
        end
        keys_from_file[file] = keys
      end
      keys_from_file
    end
102875
def _generate_examples(self, source_file, target_file):
    """"""
    with tf.io.gfile.GFile(source_file) as f:
      source_sentences = f.read().split("\n")
    with tf.io.gfile.GFile(target_file) as f:
      target_sentences = f.read().split("\n")

    assert len(target_sentences) == len(
        source_sentences), "Sizes do not match: %d vs %d for %s vs %s." % (len(
            source_sentences), len(target_sentences), source_file, target_file)

    source, target = self.builder_config.language_pair
    for l1, l2 in zip(source_sentences, target_sentences):
      result = {source: l1, target: l2}
      # Make sure that both translations are non-empty.
      if all(result.values()):
        yield result
1006963
def interlink_translations(content):
    '''
    '''
    lang = content.lang
    # sort translations by lang
    content.translations.sort(key=attrgetter('lang'))
    for translation in content.translations:
        relpath = relpath_to_site(lang, translation.lang)
        url = _NATIVE_CONTENT_URL_DB[translation.source_path]
        translation.override_url = posixpath.join(relpath, url)
589198
def _read_linguas_from_files(env, linguas_files=None):
    
    import SCons.Util
    import SCons.Environment
    global _re_comment
    global _re_lang
    if not SCons.Util.is_List(linguas_files) \
            and not SCons.Util.is_String(linguas_files) \
            and not isinstance(linguas_files, SCons.Node.FS.Base) \
            and linguas_files:
        # If, linguas_files==True or such, then read 'LINGUAS' file.
        linguas_files = ['LINGUAS']
    if linguas_files is None:
        return []
    fnodes = env.arg2nodes(linguas_files)
    linguas = []
    for fnode in fnodes:
        contents = _re_comment.sub("", fnode.get_text_contents())
        ls = [l for l in _re_lang.findall(contents) if l]
        linguas.extend(ls)
    return linguas
989670
def translate(text, target_lang='en', source_lang=None):
	
	url_base = 'https://www.googleapis.com/language/translate/v2'
	params = dict(
		key=translate.API_key,
		q=text,
		target=target_lang,
	)
	if source_lang:
		params['source'] = source_lang
	resp = requests.get(url_base, params=params)
	resp.raise_for_status()
	return resp.json()['data']['translations'][0]['translatedText']
604178
function fetchTranslations() {
            var api = getApi();
            var url = api.baseUrl + api.path;
            var requestOptions;
            var queryParams = { 'source_file_name': options.sourceFile };

            if (options.exportType === 'multilingual') {
                url = url + 'translations/multilingual';
                queryParams = _.extend(queryParams, { 'file_format': 'I18NEXT_MULTILINGUAL_JSON' });
            }

            if (options.exportType === 'locale') {
                url = url + 'translations';
                queryParams = _.extend(queryParams, { 'locale': options.locale });
            }

            requestOptions = {
                method: 'GET',
                url: url,
                headers: {
                    'Content-Type': 'application/json'
                },
                qs: {
                    'api_key': api.publicKey,
                    'timestamp': api.timestamp,
                    'dev_hash': api.devHash
                }
            };

            requestOptions.qs = _.extend(requestOptions.qs, queryParams);

            return request(requestOptions, onFetchTranslations);
        }
82710
def get_translation_in(self, key=None):
        
        if key is not None:
            return [
                item
                for item in self.texts.values()
                if isinstance(item, CtsTranslationMetadata) and item.lang == key
                ]
        else:
            return [
                item
                for item in self.texts.values()
                if isinstance(item, CtsTranslationMetadata)
            ]