Creates a file with a list of prefixes (identifiers) of utterances to
    include in the test set. Test utterances must never be wordlists. Assumes
    preprocessing of label dir has already been done.
540787
def make_data_splits(label_type, train_rec_type="text_and_wordlist", max_samples=1000,
                     seed=0, tgt_dir=TGT_DIR):
    

    feat_dir = os.path.join(tgt_dir, "feat")
    test_prefix_fn=os.path.join(tgt_dir, "test_prefixes.txt")
    valid_prefix_fn=os.path.join(tgt_dir, "valid_prefixes.txt")
    with open(test_prefix_fn) as f:
        prefixes = f.readlines()
        test_prefixes = [("TEXT/" + prefix).strip() for prefix in prefixes]
    with open(valid_prefix_fn) as f:
        prefixes = f.readlines()
        valid_prefixes = [("TEXT/" + prefix).strip() for prefix in prefixes]

    label_dir = os.path.join(tgt_dir, "label")
    prefixes = get_story_prefixes(label_type, label_dir=label_dir)
    prefixes = list(set(prefixes) - set(valid_prefixes))
    prefixes = list(set(prefixes) - set(test_prefixes))
    prefixes = utils.filter_by_size(
        feat_dir, prefixes, "fbank", max_samples)

    if train_rec_type == "text":
        train_prefixes = prefixes
    else:
        wordlist_prefixes = [prefix for prefix in os.listdir(os.path.join(label_dir, "WORDLIST"))
                             if prefix.endswith("phonemes")]
        wordlist_prefixes = [os.path.splitext(os.path.join("WORDLIST", prefix))[0]
                             for prefix in wordlist_prefixes]
        wordlist_prefixes = utils.filter_by_size(
                feat_dir, wordlist_prefixes, "fbank", max_samples)
        if train_rec_type == "wordlist":
            prefixes = wordlist_prefixes
        elif train_rec_type == "text_and_wordlist":
            prefixes.extend(wordlist_prefixes)
        else:
            raise PersephoneException("train_rec_type='%s' not supported." % train_rec_type)
        train_prefixes = prefixes
    random.seed(seed)
    random.shuffle(train_prefixes)

    return train_prefixes, valid_prefixes, test_prefixes
726570
def generate_strings(project_base_dir, localization_bundle_path, tmp_directory, exclude_dirs, include_strings_file,
                     special_ui_components_prefix):
    

    localization_directory = os.path.join(localization_bundle_path, DEFAULT_LANGUAGE_DIRECTORY_NAME)
    if not os.path.exists(localization_directory):
        os.makedirs(localization_directory)

    localization_file = os.path.join(localization_directory, LOCALIZATION_FILENAME)

    # Creating the same directory tree structure in the tmp directory
    tmp_localization_directory = os.path.join(tmp_directory, DEFAULT_LANGUAGE_DIRECTORY_NAME)
    tmp_localization_file = os.path.join(tmp_localization_directory, LOCALIZATION_FILENAME)

    if os.path.isdir(tmp_localization_directory):
        shutil.rmtree(tmp_localization_directory)
    os.mkdir(tmp_localization_directory)

    logging.info("Running genstrings")

    source_files = extract_source_files(project_base_dir, exclude_dirs)

    genstrings_cmd = 'genstrings -s JTLocalizedString -o %s %s' % (tmp_localization_directory, " ".join(
            ['"%s"' % (source_file,) for source_file in source_files]))

    genstrings_process = subprocess.Popen(genstrings_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                                          stdin=subprocess.PIPE, shell=True)

    genstrings_out, genstrings_err = genstrings_process.communicate()

    remove_empty_comments_from_file(tmp_localization_file)
    add_genstrings_comments_to_file(tmp_localization_file, genstrings_err)

    genstrings_rc = genstrings_process.returncode
    if genstrings_rc != 0:
        logging.fatal("genstrings returned %d, aborting run!", genstrings_rc)
        sys.exit(genstrings_rc)

    create_localized_strings_from_ib_files(project_base_dir, exclude_dirs, tmp_localization_file,
                                           special_ui_components_prefix)

    if include_strings_file:
        target = open_strings_file(tmp_localization_file, "a")
        source = open_strings_file(include_strings_file, "r")
        target.write(source.read())
        source.close()
        target.close()

    handle_duplications(tmp_localization_file)

    if os.path.isfile(localization_file):
        logging.info("Merging old localizable with new one...")
        merge_strings_files(localization_file, tmp_localization_file)
    else:
        logging.info("No Localizable yet, moving the created file...")
        shutil.move(tmp_localization_file, localization_file)
872366
def prepare_data_loader(args, dataset, vocab, test=False):
    
    # Preprocess
    dataset = dataset.transform(lambda s1, s2, label: (vocab(s1), vocab(s2), label),
                                lazy=False)

    # Batching
    batchify_fn = btf.Tuple(btf.Pad(), btf.Pad(), btf.Stack(dtype='int32'))
    data_lengths = [max(len(d[0]), len(d[1])) for d in dataset]
    batch_sampler = nlp.data.FixedBucketSampler(lengths=data_lengths,
                                                batch_size=args.batch_size,
                                                shuffle=(not test))
    data_loader = gluon.data.DataLoader(dataset=dataset,
                                        batch_sampler=batch_sampler,
                                        batchify_fn=batchify_fn)
    return data_loader
441524
public List<WordTag> process(List list) {
    List<IN> featureLabels = new ArrayList<IN>();
    for (Object o : list) {
      CoreLabel wi = new CoreLabel();
      if (o instanceof HasWord) {
        wi.setWord(((HasWord) o).word());
        if (o instanceof HasTag) {
          wi.setTag(((HasTag) o).tag());
        }
      } else {
        wi.setWord(o.toString());
      }
      featureLabels.add((IN) wi);
    }
    List<IN> tagged = classify(featureLabels);
    List<WordTag> out = new ArrayList<WordTag>();
    for (CoreLabel wi : tagged) {
      out.add(new WordTag(wi.word(), wi.get(AnswerAnnotation.class)));
    }
    return out;
  }
554758
def create_feature_dict(files):
    
    dict = {}

    def add_to_dict( feature, file ):
        if not SCons.Util.is_List( feature ):
            feature = [ feature ]

        for f in feature:
            if f not in dict:
                dict[ f ] = [ file ]
            else:
                dict[ f ].append( file )

    for file in files:
        if hasattr( file, 'PACKAGING_X_MSI_FEATURE' ):
            add_to_dict(file.PACKAGING_X_MSI_FEATURE, file)
        elif hasattr( file, 'PACKAGING_DOC' ):
            add_to_dict( 'PACKAGING_DOC', file )
        else:
            add_to_dict( 'default', file )

    return dict
1805
def create_new_file(help_string=NO_HELP, default=NO_DEFAULT, suffixes=None):
        # type: (str, Union[str, NO_DEFAULT_TYPE], Union[List[str], None]) -> str
        
        # noinspection PyTypeChecker
        return ParamFilename(
            help_string=help_string,
            default=default,
            type_name="new_file",
            suffixes=suffixes,
        )
700446
def snli_token_generator(tmp_dir, train, vocab_size):
  """"""
  _download_and_parse_dataset(tmp_dir, train)

  symbolizer_vocab = _get_or_generate_vocab(
      tmp_dir, 'vocab.subword_text_encoder', vocab_size)

  file_name = 'train' if train else 'dev'
  data_file = os.path.join(tmp_dir, file_name + '.txt')
  with tf.gfile.GFile(data_file, mode='r') as f:
    for line in f:
      sent1, sent2, label = line.strip().split('\t')
      sent1_enc = symbolizer_vocab.encode(sent1)
      sent2_enc = symbolizer_vocab.encode(sent2)

      inputs = sent1_enc + [_SEP] + sent2_enc + [_EOS]
      yield {
          'inputs': inputs,
          'targets': [_LABEL_TO_ID[label]],
      }
291587
def extend_from_instances(self,
                              params: Params,
                              instances: Iterable['adi.Instance'] = ()) -> None:
        
        min_count = params.pop("min_count", None)
        max_vocab_size = pop_max_vocab_size(params)
        non_padded_namespaces = params.pop("non_padded_namespaces", DEFAULT_NON_PADDED_NAMESPACES)
        pretrained_files = params.pop("pretrained_files", {})
        min_pretrained_embeddings = params.pop("min_pretrained_embeddings", None)
        only_include_pretrained_words = params.pop_bool("only_include_pretrained_words", False)
        tokens_to_add = params.pop("tokens_to_add", None)
        params.assert_empty("Vocabulary - from dataset")

        logger.info("Fitting token dictionary from dataset.")
        namespace_token_counts: Dict[str, Dict[str, int]] = defaultdict(lambda: defaultdict(int))
        for instance in Tqdm.tqdm(instances):
            instance.count_vocab_items(namespace_token_counts)
        self._extend(counter=namespace_token_counts,
                     min_count=min_count,
                     max_vocab_size=max_vocab_size,
                     non_padded_namespaces=non_padded_namespaces,
                     pretrained_files=pretrained_files,
                     only_include_pretrained_words=only_include_pretrained_words,
                     tokens_to_add=tokens_to_add,
                     min_pretrained_embeddings=min_pretrained_embeddings)
587146
def get_vocabulary(preprocess_output_dir, name):
  
  vocab_file = os.path.join(preprocess_output_dir, CATEGORICAL_ANALYSIS % name)
  if not file_io.file_exists(vocab_file):
    raise ValueError('File %s not found in %s' %
                     (CATEGORICAL_ANALYSIS % name, preprocess_output_dir))

  labels = python_portable_string(
      file_io.read_file_to_string(vocab_file)).split('\n')
  label_values = [x for x in labels if x]  # remove empty lines

  return label_values
1141125
def prepare_to_generate(self, data_dir, tmp_dir):
    """"""
    self.get_or_create_vocab(data_dir, tmp_dir)
    self.train_text_filepaths(tmp_dir)
    self.dev_text_filepaths(tmp_dir)
51891
def create_prefix_dir(nb_file, fmt):
    """"""
    if 'prefix' in fmt:
        nb_dir = os.path.dirname(nb_file) + os.path.sep
        if not os.path.isdir(nb_dir):
            logging.log(logging.WARNING, "[jupytext] creating missing directory %s", nb_dir)
            os.makedirs(nb_dir)
1157804
def create(technical_terms_filename, spellchecker_cache_path):
    """"""
    user_dictionary = os.path.join(os.getcwd(), "DICTIONARY")
    user_words = read_dictionary_file(user_dictionary)

    technical_terms_set = set(user_words)

    if technical_terms_filename:
        with open(technical_terms_filename) as tech_tf:
            technical_terms_set |= set(tech_tf.read().splitlines())

    return Dictionary(technical_terms_set,
                      "technical_words",
                      dictionary_sources=[technical_terms_filename,
                                          user_dictionary],
                      cache=spellchecker_cache_path)
1010193
def _create_variables_no_pretrain(self, n_features):
        
        self.encoding_w_ = []
        self.encoding_b_ = []

        for l, layer in enumerate(self.layers):

            w_name = 'enc-w-{}'.format(l)
            b_name = 'enc-b-{}'.format(l)

            if l == 0:
                w_shape = [n_features, self.layers[l]]
            else:
                w_shape = [self.layers[l - 1], self.layers[l]]

            w_init = tf.truncated_normal(shape=w_shape, stddev=0.1)
            W = tf.Variable(w_init, name=w_name)
            tf.summary.histogram(w_name, W)
            self.encoding_w_.append(W)

            b_init = tf.constant(0.1, shape=[self.layers[l]])
            b = tf.Variable(b_init, name=b_name)
            tf.summary.histogram(b_name, b)
            self.encoding_b_.append(b)
444253
def create(cls, train_ds, valid_ds, test_ds=None, path:PathOrStr='.', bs:int=32, val_bs:int=None, pad_idx=1,
               pad_first=True, device:torch.device=None, no_check:bool=False, backwards:bool=False, **dl_kwargs) -> DataBunch:
        ""
        datasets = cls._init_ds(train_ds, valid_ds, test_ds)
        val_bs = ifnone(val_bs, bs)
        collate_fn = partial(pad_collate, pad_idx=pad_idx, pad_first=pad_first, backwards=backwards)
        train_sampler = SortishSampler(datasets[0].x, key=lambda t: len(datasets[0][t][0].data), bs=bs)
        train_dl = DataLoader(datasets[0], batch_size=bs, sampler=train_sampler, drop_last=True, **dl_kwargs)
        dataloaders = [train_dl]
        for ds in datasets[1:]:
            lengths = [len(t) for t in ds.x.items]
            sampler = SortSampler(ds.x, key=lengths.__getitem__)
            dataloaders.append(DataLoader(ds, batch_size=val_bs, sampler=sampler, **dl_kwargs))
        return cls(*dataloaders, path=path, device=device, collate_fn=collate_fn, no_check=no_check)
718799
public static NamedFileChannel createNewFile(Path dir, String prefix, String suffix) throws IOException {
        return createNewFileImpl(dir, prefix, suffix, new OpenOption[]{READ, WRITE, CREATE_NEW});
    }
1266395
public List<String> generateFeats(String[] tokens) {
        tokens = Arrays.stream(tokens).filter(Objects::nonNull).filter(x -> !x.trim().isEmpty())
                .map(String::toLowerCase).toArray(String[]::new);

        if (tokens.length == 0) {
            return new ArrayList<>();
        }

        List<String> feats = new ArrayList<>();
        feats.add("word_count_" + String.valueOf(tokens.length));

        if (tokens[0].toLowerCase().startsWith("wh")) {
            feats.add("wh_start");
        }

        // Verify, Ask, Say - imperative
        boolean isImperativeStart = IMPERATIVE_KEYWORDS.contains(tokens[0]);
        String lastToken = tokens[tokens.length - 1];
        if (isImperativeStart) {
            feats.add("imperative_start");
            if (!Pattern.matches("\\p{Punct}+", lastToken)) {
                String[] newTokens = new String[tokens.length + 1];
                System.arraycopy(tokens, 0, newTokens, 0, tokens.length);
                tokens = newTokens;
            }
            tokens[tokens.length - 1] = "!";
        }

        // Start with question words.
        String sentence = Arrays.stream(tokens).collect(Collectors.joining(StringUtils.SPACE));
        long questionPrefixCount = QUESTION_PREFIX.stream().filter(sentence::startsWith).count();
        if (questionPrefixCount > 0) {
            feats.add("question_prefix");
            if (!Pattern.matches("\\p{Punct}+", lastToken)) {
                String[] newTokens = new String[tokens.length + 1];
                System.arraycopy(tokens, 0, newTokens, 0, tokens.length);
                tokens = newTokens;
            }
            tokens[tokens.length - 1] = "?";
        }

        feats.add("first_word_" + tokens[0]);

        if (tokens.length > 1) {
            feats.add("sec_word_" + tokens[1]);
        }

        lastToken = tokens[tokens.length - 1];
        if (Pattern.matches("\\p{Punct}+", lastToken)) {
            feats.add("punct_" + lastToken);
        }

        // whether
        long whetherKeyWord = Arrays.stream(tokens).filter("whether"::equals).count();
        if (whetherKeyWord > 0) {
            feats.add("has_whether");
        }

        List<String> biWord = new ArrayList<>();
        List<String> triWord = new ArrayList<>();

        for (int i = 0; i < tokens.length && i < 6; i++) { // 6 words maximum
            // Bigram
            if (i + 1 < tokens.length) {
                biWord.add(tokens[i] + "_" + tokens[i + 1]);
            }

            // Trigram
            if (i + 2 < tokens.length) {
                triWord.add(tokens[i] + "_" + tokens[i + 1] + "_" + tokens[i + 2]);
            }
        }

        feats.addAll(biWord);
        feats.addAll(triWord);

//        LOG.debug("feats: " + feats);
        return feats;
    }
111705
private void initializePrefixes()
	{
		if (namespaceMap == null) {
			// TODO use logger
			System.err
					.println("Cannot use whitespace filter without initializing the namespace-prefix map for the current Wikipedia language version. DISABLING FILTER.");
		}
		else {
			prefixesToAllow = new HashSet<String>();
			prefixesToReject = new HashSet<String>();

			for (Entry<Integer, String> namespace : namespaceMap.entrySet()) {
				if (allowedNamespaces.contains(namespace.getKey())) {
					prefixesToAllow.add(namespace.getValue() + ":");
				}
				else {
					prefixesToReject.add(namespace.getValue() + ":");
				}
			}
		}
	}