Generator to feed sentences to the dov2vec model.
195117
def _doc2vec_doc_stream(paths, n, tokenizer=word_tokenize, sentences=True):
    
    i = 0
    p = Progress()
    for path in paths:
        with open(path, 'r') as f:
            for line in f:
                i += 1
                p.print_progress(i/n)

                # We do minimal pre-processing here so the model can learn
                # punctuation
                line = line.lower()

                if sentences:
                    for sent in sent_tokenize(line):
                        tokens = tokenizer(sent)
                        yield LabeledSentence(tokens, ['SENT_{}'.format(i)])
                else:
                    tokens = tokenizer(line)
                    yield LabeledSentence(tokens, ['SENT_{}'.format(i)])
703954
def phrases_map(data):
  """"""
  (entry, text_fn) = data
  text = text_fn()
  filename = entry.filename

  logging.debug("Got %s", filename)
  for s in split_into_sentences(text):
    words = split_into_words(s.lower())
    if len(words) < PHRASE_LENGTH:
      yield (":".join(words), filename)
      continue
    for i in range(0, len(words) - PHRASE_LENGTH):
      yield (":".join(words[i:i+PHRASE_LENGTH]), filename)
147291
def train_doc2vec(paths, out='data/model.d2v', tokenizer=word_tokenize, sentences=False, **kwargs):
    
    kwargs = {
        'size': 400,
        'window': 8,
        'min_count': 2,
        'workers': 8
    }.update(kwargs)

    n = 0
    for path in paths:
        print('Counting lines for {0}...'.format(path))
        n += sum(1 for line in open(path, 'r'))
    print('Processing {0} lines...'.format(n))

    print('Training doc2vec model...')
    m = Doc2Vec(_doc2vec_doc_stream(paths, n, tokenizer=tokenizer, sentences=sentences), **kwargs)

    print('Saving...')
    m.save(out)
254154
public INDArray loadSingleSentence(String sentence) {
        List<String> tokens = tokenizeSentence(sentence);
        if(format == Format.CNN1D || format == Format.RNN){
            int[] featuresShape = new int[] {1, wordVectorSize, Math.min(maxSentenceLength, tokens.size())};
            INDArray features = Nd4j.create(featuresShape, (format == Format.CNN1D ? 'c' : 'f'));
            INDArrayIndex[] indices = new INDArrayIndex[3];
            indices[0] = NDArrayIndex.point(0);
            for (int i = 0; i < featuresShape[2]; i++) {
                INDArray vector = getVector(tokens.get(i));
                indices[1] = NDArrayIndex.all();
                indices[2] = NDArrayIndex.point(i);
                features.put(indices, vector);
            }
            return features;
        } else {
            int[] featuresShape = new int[] {1, 1, 0, 0};
            if (sentencesAlongHeight) {
                featuresShape[2] = Math.min(maxSentenceLength, tokens.size());
                featuresShape[3] = wordVectorSize;
            } else {
                featuresShape[2] = wordVectorSize;
                featuresShape[3] = Math.min(maxSentenceLength, tokens.size());
            }

            INDArray features = Nd4j.create(featuresShape);
            int length = (sentencesAlongHeight ? featuresShape[2] : featuresShape[3]);
            INDArrayIndex[] indices = new INDArrayIndex[4];
            indices[0] = NDArrayIndex.point(0);
            indices[1] = NDArrayIndex.point(0);
            for (int i = 0; i < length; i++) {
                INDArray vector = getVector(tokens.get(i));

                if (sentencesAlongHeight) {
                    indices[2] = NDArrayIndex.point(i);
                    indices[3] = NDArrayIndex.all();
                } else {
                    indices[2] = NDArrayIndex.all();
                    indices[3] = NDArrayIndex.point(i);
                }

                features.put(indices, vector);
            }

            return features;
        }
    }
57692
def _normalize(sentences):
    ''''''

    tfidf = TfidfVectorizer(tokenizer=_tokenize,
                            stop_words='english',
                            decode_error='ignore',
                            ngram_range=(1,2))
    return tfidf.fit_transform(sentences)
413457
def train(self, sentences, nr_iter=5):
        
        self._make_tagdict(sentences)
        self.perceptron.classes = self.classes
        for iter_ in range(nr_iter):
            c = 0
            n = 0
            for sentence in sentences:
                prev, prev2 = self.START
                context = [t[0] for t in sentence]
                for i, (token, tag) in enumerate(sentence):
                    guess = self.tagdict.get(token)
                    if not guess:
                        feats = self._get_features(i, context, prev, prev2)
                        guess = self.perceptron.predict(feats)
                        self.perceptron.update(tag, guess, feats)
                    prev2 = prev
                    prev = guess
                    c += guess == tag
                    n += 1
            random.shuffle(sentences)
            log.debug('Iter %s: %s/%s=%s' % (iter_, c, n, (float(c) / n) * 100))
        self.perceptron.average_weights()
1112579
def doc2vec(self, doc):
        """"""
        bow = self.dictionary.doc2bow(doc['tokens'])
        if self.method == 'lsi':
            return self.lsi[self.tfidf[bow]]
        elif self.method == 'lda':
            return self.lda[bow]
        elif self.method == 'lda_tfidf':
            return self.lda[self.tfidf[bow]]
        elif self.method == 'logentropy':
            return self.logent[bow]
264174
def convert_examples_to_features(examples, tokenizer, max_seq_length,
                                 is_training):
    """"""

    # Swag is a multiple choice task. To perform this task using Bert,
    # we will use the formatting proposed in "Improving Language
    # Understanding by Generative Pre-Training" and suggested by
    # @jacobdevlin-google in this issue
    # https://github.com/google-research/bert/issues/38.
    #
    # Each choice will correspond to a sample on which we run the
    # inference. For a given Swag example, we will create the 4
    # following inputs:
    # - [CLS] context [SEP] choice_1 [SEP]
    # - [CLS] context [SEP] choice_2 [SEP]
    # - [CLS] context [SEP] choice_3 [SEP]
    # - [CLS] context [SEP] choice_4 [SEP]
    # The model will output a single value for each input. To get the
    # final decision of the model, we will run a softmax over these 4
    # outputs.
    features = []
    for example_index, example in enumerate(examples):
        context_tokens = tokenizer.tokenize(example.context_sentence)
        start_ending_tokens = tokenizer.tokenize(example.start_ending)

        choices_features = []
        for ending_index, ending in enumerate(example.endings):
            # We create a copy of the context tokens in order to be
            # able to shrink it according to ending_tokens
            context_tokens_choice = context_tokens[:]
            ending_tokens = start_ending_tokens + tokenizer.tokenize(ending)
            # Modifies `context_tokens_choice` and `ending_tokens` in
            # place so that the total length is less than the
            # specified length.  Account for [CLS], [SEP], [SEP] with
            # "- 3"
            _truncate_seq_pair(context_tokens_choice, ending_tokens, max_seq_length - 3)

            tokens = ["[CLS]"] + context_tokens_choice + ["[SEP]"] + ending_tokens + ["[SEP]"]
            segment_ids = [0] * (len(context_tokens_choice) + 2) + [1] * (len(ending_tokens) + 1)

            input_ids = tokenizer.convert_tokens_to_ids(tokens)
            input_mask = [1] * len(input_ids)

            # Zero-pad up to the sequence length.
            padding = [0] * (max_seq_length - len(input_ids))
            input_ids += padding
            input_mask += padding
            segment_ids += padding

            assert len(input_ids) == max_seq_length
            assert len(input_mask) == max_seq_length
            assert len(segment_ids) == max_seq_length

            choices_features.append((tokens, input_ids, input_mask, segment_ids))

        label = example.label
        if example_index < 5:
            logger.info("*** Example ***")
            logger.info("swag_id: {}".format(example.swag_id))
            for choice_idx, (tokens, input_ids, input_mask, segment_ids) in enumerate(choices_features):
                logger.info("choice: {}".format(choice_idx))
                logger.info("tokens: {}".format(' '.join(tokens)))
                logger.info("input_ids: {}".format(' '.join(map(str, input_ids))))
                logger.info("input_mask: {}".format(' '.join(map(str, input_mask))))
                logger.info("segment_ids: {}".format(' '.join(map(str, segment_ids))))
            if is_training:
                logger.info("label: {}".format(label))

        features.append(
            InputFeatures(
                example_id = example.swag_id,
                choices_features = choices_features,
                label = label
            )
        )

    return features
1322417
def _gen_sentence(self, assetid_body_tuple):
        '''
         
        '''
        asset_id, body = assetid_body_tuple
        text = self._process(body)
        sentence = LabeledSentence(text, labels=['DOC_%s' % str(asset_id)])
        return sentence
680171
def _iter_content_generator(response, decode_unicode):
    """"""
    for chunk in response.iter_content(100 * 1024, decode_unicode=decode_unicode):
        if decode_unicode:
            # Replace CRLF newlines with LF, Python will handle
            # platform specific newlines if written to file.
            chunk = chunk.replace('\r\n', '\n')
            # Chunk could be ['...\r', '\n...'], stril trailing \r
            chunk = chunk.rstrip('\r')
        yield chunk
902822
def foreach_sentence(layer, drop_factor=1.0):
    """"""

    def sentence_fwd(docs, drop=0.0):
        sents = []
        lengths = []
        for doc in docs:
            doc_sents = [sent for sent in doc.sents if len(sent)]
            subset = [
                s for s in doc_sents if numpy.random.random() >= drop * drop_factor
            ]
            if subset:
                sents.extend(subset)
                lengths.append(len(subset))
            else:
                numpy.random.shuffle(doc_sents)
                sents.append(doc_sents[0])
                lengths.append(1)
        flat, bp_flat = layer.begin_update(sents, drop=0.0)
        output = layer.ops.unflatten(flat, lengths)

        def sentence_bwd(d_output, sgd=None):
            d_flat = layer.ops.flatten(d_output)
            d_sents = bp_flat(d_flat, sgd=sgd)
            if d_sents is None:
                return d_sents
            else:
                return layer.ops.unflatten(d_sents, lengths)

        return output, sentence_bwd

    model = wrap(sentence_fwd, layer)
    return model
1243333
def create_mnist_model(hyper_params, input_shape=(H, W, 1), num_classes=NUM_CLASSES):
    '''
    
    '''
    layers = [
        Conv2D(32, kernel_size=(3, 3), activation='relu', input_shape=input_shape),
        Conv2D(64, (3, 3), activation='relu'),
        MaxPooling2D(pool_size=(2, 2)),
        Flatten(),
        Dense(100, activation='relu'),
        Dense(num_classes, activation='softmax')
    ]

    model = Sequential(layers)

    if hyper_params['optimizer'] == 'Adam':
        optimizer = keras.optimizers.Adam(lr=hyper_params['learning_rate'])
    else:
        optimizer = keras.optimizers.SGD(lr=hyper_params['learning_rate'], momentum=0.9)
    model.compile(loss=keras.losses.categorical_crossentropy, optimizer=optimizer, metrics=['accuracy'])

    return model
1225674
def maybe_build_from_corpus(self, corpus_generator, **kwargs):
    """"""
    if self._encoder_cls is not text_lib.SubwordTextEncoder:
      return
    if self.encoder:
      return

    vocab_size = self._encoder_config.vocab_size
    self.encoder = text_lib.SubwordTextEncoder.build_from_corpus(
        corpus_generator=corpus_generator,
        target_vocab_size=vocab_size,
        **kwargs)
795392
def convert_examples_to_features(examples, label_list, max_seq_length,
                                 tokenizer, output_mode):
    """"""

    label_map = {label : i for i, label in enumerate(label_list)}

    features = []
    for (ex_index, example) in enumerate(examples):
        if ex_index % 10000 == 0:
            logger.info("Writing example %d of %d" % (ex_index, len(examples)))

        tokens_a = tokenizer.tokenize(example.text_a)

        tokens_b = None
        if example.text_b:
            tokens_b = tokenizer.tokenize(example.text_b)
            # Modifies `tokens_a` and `tokens_b` in place so that the total
            # length is less than the specified length.
            # Account for [CLS], [SEP], [SEP] with "- 3"
            _truncate_seq_pair(tokens_a, tokens_b, max_seq_length - 3)
        else:
            # Account for [CLS] and [SEP] with "- 2"
            if len(tokens_a) > max_seq_length - 2:
                tokens_a = tokens_a[:(max_seq_length - 2)]

        # The convention in BERT is:
        # (a) For sequence pairs:
        #  tokens:   [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP]
        #  type_ids: 0   0  0    0    0     0       0 0    1  1  1  1   1 1
        # (b) For single sequences:
        #  tokens:   [CLS] the dog is hairy . [SEP]
        #  type_ids: 0   0   0   0  0     0 0
        #
        # Where "type_ids" are used to indicate whether this is the first
        # sequence or the second sequence. The embedding vectors for `type=0` and
        # `type=1` were learned during pre-training and are added to the wordpiece
        # embedding vector (and position vector). This is not *strictly* necessary
        # since the [SEP] token unambiguously separates the sequences, but it makes
        # it easier for the model to learn the concept of sequences.
        #
        # For classification tasks, the first vector (corresponding to [CLS]) is
        # used as as the "sentence vector". Note that this only makes sense because
        # the entire model is fine-tuned.
        tokens = ["[CLS]"] + tokens_a + ["[SEP]"]
        segment_ids = [0] * len(tokens)

        if tokens_b:
            tokens += tokens_b + ["[SEP]"]
            segment_ids += [1] * (len(tokens_b) + 1)

        input_ids = tokenizer.convert_tokens_to_ids(tokens)

        # The mask has 1 for real tokens and 0 for padding tokens. Only real
        # tokens are attended to.
        input_mask = [1] * len(input_ids)

        # Zero-pad up to the sequence length.
        padding = [0] * (max_seq_length - len(input_ids))
        input_ids += padding
        input_mask += padding
        segment_ids += padding

        assert len(input_ids) == max_seq_length
        assert len(input_mask) == max_seq_length
        assert len(segment_ids) == max_seq_length

        if output_mode == "classification":
            label_id = label_map[example.label]
        elif output_mode == "regression":
            label_id = float(example.label)
        else:
            raise KeyError(output_mode)

        if ex_index < 5:
            logger.info("*** Example ***")
            logger.info("guid: %s" % (example.guid))
            logger.info("tokens: %s" % " ".join(
                    [str(x) for x in tokens]))
            logger.info("input_ids: %s" % " ".join([str(x) for x in input_ids]))
            logger.info("input_mask: %s" % " ".join([str(x) for x in input_mask]))
            logger.info(
                    "segment_ids: %s" % " ".join([str(x) for x in segment_ids]))
            logger.info("label: %s (id = %d)" % (example.label, label_id))

        features.append(
                InputFeatures(input_ids=input_ids,
                              input_mask=input_mask,
                              segment_ids=segment_ids,
                              label_id=label_id))
    return features
323498
def sentences(obj: BioCCollection or BioCDocument or BioCPassage) \
        -> Generator[BioCSentence, None, None]:
    
    if isinstance(obj, BioCCollection):
        for document in obj.documents:
            yield from sentences(document)
    elif isinstance(obj, BioCDocument):
        for passage in obj.passages:
            yield from sentences(passage)
    elif isinstance(obj, BioCPassage):
        yield from obj.sentences
    else:
        raise TypeError(f'Object of type {obj.__class__.__name__} must be BioCCollection, '
                        f'BioCDocument, BioCPassage, or BioCSentence')
302799
def _get_wv(sentence, ignore=False):
    '''
    
    '''
    global _vectors
    vectors = []
    for y in sentence:
        y_ = any2unicode(y).strip()
        if y_ not in _stopwords:
            syns = nearby(y_)[0]
            # print("sentence %s word: %s" %(sentence, y_))
            # print("sentence %s word nearby: %s" %(sentence, " ".join(syns)))
            c = []
            try:
                c.append(_vectors.word_vec(y_))
            except KeyError as error:
                if ignore:
                    continue
                else:
                    logging.warning("not exist in w2v model: %s" % y_)
                    # c.append(np.zeros((100,), dtype=float))
                    random_state = np.random.RandomState(seed=(hash(y_) % (2**32 - 1)))
                    c.append(random_state.uniform(low=-10.0, high=10.0, size=(100,)))
            for n in syns:
                if n is None: continue
                try:
                    v = _vectors.word_vec(any2unicode(n))
                except KeyError as error:
                    # v = np.zeros((100,), dtype=float)
                    random_state = np.random.RandomState(seed=(hash(n) % (2 ** 32 - 1)))
                    v = random_state.uniform(low=10.0, high=10.0, size=(100,))
                c.append(v)
            r = np.average(c, axis=0)
            vectors.append(r)
    return vectors
1083739
def build_net(self, is_training):
        """"""
        cfg = self.cfg
        with tf.device('/cpu:0'):
            word_embed = tf.get_variable(
                name='word_embed', initializer=self.embed, dtype=tf.float32, trainable=False)
            char_embed = tf.get_variable(name='char_embed',
                                         shape=[cfg.char_vcb_size,
                                                cfg.char_embed_dim],
                                         dtype=tf.float32)

        # [query_length, batch_size]
        self.query_word = tf.placeholder(dtype=tf.int32,
                                         shape=[None, None],
                                         name='query_word')
        self.query_mask = tf.placeholder(dtype=tf.float32,
                                         shape=[None, None],
                                         name='query_mask')
        # [batch_size]
        self.query_lengths = tf.placeholder(
            dtype=tf.int32, shape=[None], name='query_lengths')

        # [passage_length, batch_size]
        self.passage_word = tf.placeholder(
            dtype=tf.int32, shape=[None, None], name='passage_word')
        self.passage_mask = tf.placeholder(
            dtype=tf.float32, shape=[None, None], name='passage_mask')
        # [batch_size]
        self.passage_lengths = tf.placeholder(
            dtype=tf.int32, shape=[None], name='passage_lengths')

        if is_training:
            self.answer_begin = tf.placeholder(
                dtype=tf.int32, shape=[None], name='answer_begin')
            self.answer_end = tf.placeholder(
                dtype=tf.int32, shape=[None], name='answer_end')

        self.query_char_ids = tf.placeholder(dtype=tf.int32,
                                             shape=[
                                                 self.cfg.max_char_length, None, None],
                                             name='query_char_ids')
        # sequence_length, batch_size
        self.query_char_lengths = tf.placeholder(
            dtype=tf.int32, shape=[None, None], name='query_char_lengths')

        self.passage_char_ids = tf.placeholder(dtype=tf.int32,
                                               shape=[
                                                   self.cfg.max_char_length, None, None],
                                               name='passage_char_ids')
        # sequence_length, batch_size
        self.passage_char_lengths = tf.placeholder(dtype=tf.int32,
                                                   shape=[None, None],
                                                   name='passage_char_lengths')

        query_char_states = self.build_char_states(char_embed=char_embed,
                                                   is_training=is_training,
                                                   reuse=False,
                                                   char_ids=self.query_char_ids,
                                                   char_lengths=self.query_char_lengths)

        passage_char_states = self.build_char_states(char_embed=char_embed,
                                                     is_training=is_training,
                                                     reuse=True,
                                                     char_ids=self.passage_char_ids,
                                                     char_lengths=self.passage_char_lengths)

        with tf.variable_scope("encoding") as scope:
            query_states = tf.concat([tf.nn.embedding_lookup(
                word_embed, self.query_word), query_char_states], axis=2)
            scope.reuse_variables()
            passage_states = tf.concat([tf.nn.embedding_lookup(
                word_embed, self.passage_word), passage_char_states], axis=2)
        passage_states = tf.transpose(passage_states, perm=[1, 0, 2])
        query_states = tf.transpose(query_states, perm=[1, 0, 2])
        self.passage_states = passage_states
        self.query_states = query_states

        output, output2 = graph_to_network(passage_states, query_states,
                                           self.passage_lengths, self.query_lengths,
                                           self.graph, self.cfg.dropout,
                                           is_training, num_heads=cfg.num_heads,
                                           rnn_units=cfg.rnn_units)

        passage_att_mask = self.passage_mask
        batch_size_x = tf.shape(self.query_lengths)
        answer_h = tf.zeros(
            tf.concat([batch_size_x, tf.constant([cfg.ptr_dim], dtype=tf.int32)], axis=0))

        answer_context = tf.reduce_mean(output2, axis=1)

        query_init_w = tf.get_variable(
            'query_init_w', shape=[output2.get_shape().as_list()[-1], cfg.ptr_dim])
        self.query_init = query_init_w
        answer_context = tf.matmul(answer_context, query_init_w)

        output = tf.transpose(output, perm=[1, 0, 2])

        with tf.variable_scope('answer_ptr_layer'):
            ptr_att = DotAttention('ptr',
                                   hidden_dim=cfg.ptr_dim,
                                   is_vanilla=self.cfg.att_is_vanilla,
                                   is_identity_transform=self.cfg.att_is_id,
                                   need_padding=self.cfg.att_need_padding)
            answer_pre_compute = ptr_att.get_pre_compute(output)
            ptr_gru = XGRUCell(hidden_dim=cfg.ptr_dim)
            begin_prob, begin_logits = ptr_att.get_prob(output, answer_context, passage_att_mask,
                                                        answer_pre_compute, True)
            att_state = ptr_att.get_att(output, begin_prob)
            (_, answer_h) = ptr_gru.call(inputs=att_state, state=answer_h)
            answer_context = answer_h
            end_prob, end_logits = ptr_att.get_prob(output, answer_context,
                                                    passage_att_mask, answer_pre_compute,
                                                    True)

        self.begin_prob = tf.transpose(begin_prob, perm=[1, 0])
        self.end_prob = tf.transpose(end_prob, perm=[1, 0])
        begin_logits = tf.transpose(begin_logits, perm=[1, 0])
        end_logits = tf.transpose(end_logits, perm=[1, 0])

        if is_training:
            def label_smoothing(inputs, masks, epsilon=0.1):
                """Modify target for label smoothing."""
                epsilon = cfg.labelsmoothing
                num_of_channel = tf.shape(inputs)[-1]  # number of channels
                inputs = tf.cast(inputs, tf.float32)
                return (((1 - epsilon) * inputs) + (epsilon /
                                                    tf.cast(num_of_channel, tf.float32))) * masks
            cost1 = tf.reduce_mean(
                tf.losses.softmax_cross_entropy(label_smoothing(
                    tf.one_hot(self.answer_begin,
                               depth=tf.shape(self.passage_word)[0]),
                    tf.transpose(self.passage_mask, perm=[1, 0])), begin_logits))
            cost2 = tf.reduce_mean(
                tf.losses.softmax_cross_entropy(
                    label_smoothing(tf.one_hot(self.answer_end,
                                               depth=tf.shape(self.passage_word)[0]),
                                    tf.transpose(self.passage_mask, perm=[1, 0])), end_logits))

            reg_ws = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)
            l2_loss = tf.reduce_sum(reg_ws)
            loss = cost1 + cost2 + l2_loss
            self.loss = loss

            optimizer = tf.train.AdamOptimizer(learning_rate=cfg.learning_rate)
            self.train_op = optimizer.minimize(self.loss)

        return tf.stack([self.begin_prob, self.end_prob])