Static method viterbi

@param string $sentence # input sentence
@param array  $options  # other options

@return array $viterbi
181859
public static function viterbi($sentence, $options = array())
    {
        $defaults = array(
            'mode'=>'default'
        );

        $options = array_merge($defaults, $options);

        $obs = $sentence;
        $states = self::$char_state;
        $V = array();
        $V[0] = array();
        $mem_path = array();
        $mem_path[0] = array();
        $all_states = array_keys(self::$prob_trans);

        $c = mb_substr($obs, 0, 1, 'UTF-8');

        if (isset($states[$c]) && !empty($states[$c])) {
            $c_states = $states[$c];
        } else {
            $c_states = $all_states;
        }

        foreach ($c_states as $key => $state) {
            $y = $state;
            $c = mb_substr($obs, 0, 1, 'UTF-8');
            $prob_emit = 0.0;
            if (isset(self::$prob_emit[$y][$c])) {
                $prob_emit = self::$prob_emit[$y][$c];
            } else {
                $prob_emit = MIN_FLOAT;
            }
            $V[0][$y] = self::$prob_start[$y] + $prob_emit;
            $mem_path[0][$y] = '';
        }

        for ($t=1; $t<mb_strlen($obs, 'UTF-8'); $t++) {
            $c = mb_substr($obs, $t, 1, 'UTF-8');
            $V[$t] = array();
            $mem_path[$t] = array();

            $prev_states = array_keys(self::getTopStates($V[$t-1]));

            $prev_mem_path = array_keys($mem_path[$t-1]);

            $prev_states = array();

            foreach ($prev_mem_path as $mem_path_state) {
                if (count(self::$prob_trans[$mem_path_state])>0) {
                    array_push($prev_states, $mem_path_state);
                }
            }

            $prev_states_expect_next = array();

            foreach ($prev_states as $prev_state) {
                $prev_states_expect_next
                    = array_unique(
                        array_merge(
                            $prev_states_expect_next,
                            array_keys(self::$prob_trans[$prev_state])
                        )
                    );
            }

            $obs_states = array();

            if (isset($states[$c])) {
                $obs_states = $states[$c];
            } else {
                $obs_states = $all_states;
            }

            $obs_states = array_intersect($obs_states, $prev_states_expect_next);

            if (count($obs_states)==0) {
                $obs_states = $all_states;
            }


            foreach ($obs_states as $y) {
                $temp_prob_array = array();
                foreach ($prev_states as $y0) {
                    $prob_trans = 0.0;
                    if (isset(self::$prob_trans[$y0][$y])) {
                        $prob_trans = self::$prob_trans[$y0][$y];
                    } else {
                        $prob_trans = MIN_FLOAT;
                    }
                    $prob_emit = 0.0;
                    if (isset(self::$prob_emit[$y][$c])) {
                        $prob_emit = self::$prob_emit[$y][$c];
                    } else {
                        $prob_emit = MIN_FLOAT;
                    }
                    $temp_prob_array[$y0] = $V[$t-1][$y0] + $prob_trans + $prob_emit;
                }
                arsort($temp_prob_array);
                $max_prob = reset($temp_prob_array);
                $max_key = key($temp_prob_array);
                $V[$t][$y] = $max_prob;
                $mem_path[$t][$y] = $max_key;
            }
        }

        $last = array();
        $mem_path_end_keys = array_keys(end($mem_path));

        foreach ($mem_path_end_keys as $y) {
            $end_array = end($V);
            $last[$y] = $end_array[$y];
        }

        arsort($last);
        $return_prob = reset($last);
        $return_prob_key = key($last);

        $obs_length = mb_strlen($obs, 'UTF-8');

        $route = array();
        for ($t=0; $t<$obs_length; $t++) {
            array_push($route, 'None');
        }

        $i = $obs_length-1;

        while ($i >= 0) {
            $route[$i] = $return_prob_key;
            $return_prob_key = $mem_path[$i][$return_prob_key];
            $i-=1;
        }

        return array("prob"=>$return_prob, "pos_list"=>$route);
    }
324543
public static function calc($sentence, $DAG, $options = array())
    {
        $N = mb_strlen($sentence, 'UTF-8');
        self::$route = array();
        self::$route[$N] = array($N => 1.0);
        for ($i=($N-1); $i>=0; $i--) {
            $candidates = array();
            foreach ($DAG[$i] as $x) {
                $w_c = mb_substr($sentence, $i, (($x+1)-$i), 'UTF-8');
                $previous_freq = current(self::$route[$x+1]);
                if (isset(self::$FREQ[$w_c])) {
                    $current_freq = (float) $previous_freq + self::$FREQ[$w_c];
                } else {
                    $current_freq = (float) $previous_freq + self::$min_freq;
                }
                $candidates[$x] = $current_freq;
            }
            arsort($candidates);
            $max_prob = reset($candidates);
            $max_key = key($candidates);
            self::$route[$i] = array($max_key => $max_prob);
        }

        return self::$route;
    }
1353047
public static function ask(string $question, array $options) : string
    {
        self::out($question);
        fscanf(STDIN, '%s\n', $answer);
        if ($answer === null) {
            $answer = $options[true];
        }
        if (!in_array($answer, $options)) {
            return self::ask($question, $options);
        }
        return $answer;
    }
1091173
public function write($message, $options = 0)
    {
        if ($message instanceof \Exception || (interface_exists('\Throwable') && $message instanceof \Throwable)) {
            echo $message;
            return $this;
        } elseif ($message instanceof \JsonSerializable) {
            echo json_encode($message, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES);
            return $this;
        } elseif ($message instanceof \Serializable) {
            echo serialize($message);
            return $this;
        } elseif (!is_array($message)) {
            echo (string)$message;
            return $this;
        }

        if (!isset($message[0]) || !is_string($message[0])) {
            echo json_encode($message, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES);
            return $this;
        }

        if (substr_count($message[0], '%') + 1 >= ($count = count($message)) && isset($message[$count - 1])) {
            foreach ((array)$message as $k => $v) {
                if ($k === 0 || is_scalar($v) || $v === null) {
                    continue;
                }

                if ($v instanceof \Exception || (interface_exists('\Throwable') && $v instanceof \Throwable)) {
                    $message[$k] = (string)$v;
                } elseif (is_array($v) || $v instanceof \JsonSerializable) {
                    $message[$k] = json_encode($v, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES);
                }
            }
            /** @noinspection ArgumentUnpackingCanBeUsedInspection */
            echo call_user_func_array('sprintf', $message);

            return $this;
        }

        if (count($message) === 2) {
            if (isset($message[1]) && strpos($message[0], ':1') === false) {
                if (is_scalar($message[1])) {
                    echo json_encode($message, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES);
                    return $this;
                } else {
                    $message[0] = rtrim($message[0], ': ') . ': :1';
                }
            }
        } elseif (count($message) === 3) {
            if (isset($message[1], $message[2]) && strpos($message[0], ':1') === false) {
                if (is_scalar($message[1]) && !is_scalar($message[2])) {
                    $message[0] = rtrim($message[0], ': ') . ': :1 => :2';
                } else {
                    echo json_encode($message, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE);
                    return $this;
                }
            }
        }

        if (strpos($message[0], ':') === false) {
            echo $this->colorize($message[0], $options);
        } else {
            $replaces = [];

            foreach ((array)$message as $k => $v) {
                if ($k === 0) {
                    continue;
                }

                if (is_int($v)) {
                    if (!$options && strpos($v, "\033[") === false) {
                        $v = $this->colorize($v, self::FC_GREEN);
                    }
                } elseif (is_string($v)) {
                    if (!$options && strpos($v, "\033[") === false) {
                        $v = $this->colorize($v, self::FC_CYAN);
                    }
                } elseif ($v instanceof \Exception || (interface_exists('\Throwable') && $v instanceof \Throwable)) {
                    $v = (string)$v;
                } elseif (is_array($v) || $v instanceof \JsonSerializable) {
                    $v = json_encode($v, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES);
                } elseif ($v instanceof \Serializable) {
                    $v = serialize($v);
                } elseif ($v === null || is_scalar($v)) {
                    $v = json_encode($v);
                } else {
                    $v = (string)$v;
                }

                $replaces[':' . $k] = $v;
            }

            echo $this->colorize(strtr($message[0], $replaces), $options);
        }

        return $this;
    }
503447
public function split($text)
    {
        $this->probabilityCalculator->setAbbreviations($this->getAbbreviations());

        $tokens = $this->lexer->run($text);
        $probabilities = $this->probabilityCalculator->calculate($tokens);
        $sentences = $this->sentenceBuilder->build($probabilities);

        return $sentences;
    }
1153034
public static function sentences($count = 3)
    {
        $sentences = array();
        for ($i = 0; $i < $count; $i++) {
            $sentences[] = static::sentence();
        }

        return $sentences;
    }
651500
public static function generate($options = array()) {
    $sentencesToDisplay = isset($options['sentences']) ? $options['sentences'] : rand(1, 10);
    $author = isset($options['author']) ? $options['author'] : array_rand(self::$authors, 1);
    $path = __DIR__ . '/data/' . $author . '.txt';
    if (file_exists($path)) {
      $text = file_get_contents($path);
      $chars_in_text = strlen($text);
      $length = 10000;
      $start = rand(0, $chars_in_text - $length);
      $subsection = substr($text, $start, $length);
      return self::getSentences($subsection, $sentencesToDisplay);
    }
  }
207298
function tokenize_sentence(sentence) {
  var tokenized = tokenizer.tokenize(sentence);
  logger.info("tokenize_sentence: " + tokenized);
  return(tokenized);
}
208281
public static function cutForSearch($sentence, $options = array())
    {
        $defaults = array(
            'mode'=>'default'
        );

        $options = array_merge($defaults, $options);

        $seg_list = array();

        $cut_seg_list = Jieba::cut($sentence);

        foreach ($cut_seg_list as $w) {
            $len = mb_strlen($w, 'UTF-8');

            if ($len>2) {
                for ($i=0; $i<($len-1); $i++) {
                    $gram2 = mb_substr($w, $i, 2, 'UTF-8');

                    if (isset(self::$FREQ[$gram2])) {
                        array_push($seg_list, $gram2);
                    }
                }
            }

            if (mb_strlen($w, 'UTF-8')>3) {
                for ($i=0; $i<($len-2); $i++) {
                    $gram3 = mb_substr($w, $i, 3, 'UTF-8');

                    if (isset(self::$FREQ[$gram3])) {
                        array_push($seg_list, $gram3);
                    }
                }
            }

            array_push($seg_list, $w);
        }

        return $seg_list;
    }
1328325
private function punctuate(&$sentence)
    {
        $count = count($sentence);
        $sentence[$count - 1] = $sentence[$count - 1].'.';

        if ($count < 4) {
            return $sentence;
        }

        $commas = $this->numberOfCommas($count);

        for ($i = 1; $i <= $commas; ++$i) {
            $index = (int) round($i * $count / ($commas + 1));

            if ($index < ($count - 1) && $index > 0) {
                $sentence[$index] = $sentence[$index].',';
            }
        }

        return null;
    }
372714
public static function tokenize($sentence, $options = array("HMM" => true))
    {
        $seg_list = self::cut($sentence, false, array("HMM" => $options["HMM"]));
        $tokenize_list = [];
        $start = 0;
        $end = 0;
        foreach ($seg_list as $seg) {
            $end = $start+mb_strlen($seg, 'UTF-8');
            $tokenize = [
                'word' => $seg,
                'start' => $start,
                'end' => $end
            ];
            $start = $end;
            array_push($tokenize_list, $tokenize);
        }
        return $tokenize_list;
    }
254154
public INDArray loadSingleSentence(String sentence) {
        List<String> tokens = tokenizeSentence(sentence);
        if(format == Format.CNN1D || format == Format.RNN){
            int[] featuresShape = new int[] {1, wordVectorSize, Math.min(maxSentenceLength, tokens.size())};
            INDArray features = Nd4j.create(featuresShape, (format == Format.CNN1D ? 'c' : 'f'));
            INDArrayIndex[] indices = new INDArrayIndex[3];
            indices[0] = NDArrayIndex.point(0);
            for (int i = 0; i < featuresShape[2]; i++) {
                INDArray vector = getVector(tokens.get(i));
                indices[1] = NDArrayIndex.all();
                indices[2] = NDArrayIndex.point(i);
                features.put(indices, vector);
            }
            return features;
        } else {
            int[] featuresShape = new int[] {1, 1, 0, 0};
            if (sentencesAlongHeight) {
                featuresShape[2] = Math.min(maxSentenceLength, tokens.size());
                featuresShape[3] = wordVectorSize;
            } else {
                featuresShape[2] = wordVectorSize;
                featuresShape[3] = Math.min(maxSentenceLength, tokens.size());
            }

            INDArray features = Nd4j.create(featuresShape);
            int length = (sentencesAlongHeight ? featuresShape[2] : featuresShape[3]);
            INDArrayIndex[] indices = new INDArrayIndex[4];
            indices[0] = NDArrayIndex.point(0);
            indices[1] = NDArrayIndex.point(0);
            for (int i = 0; i < length; i++) {
                INDArray vector = getVector(tokens.get(i));

                if (sentencesAlongHeight) {
                    indices[2] = NDArrayIndex.point(i);
                    indices[3] = NDArrayIndex.all();
                } else {
                    indices[2] = NDArrayIndex.all();
                    indices[3] = NDArrayIndex.point(i);
                }

                features.put(indices, vector);
            }

            return features;
        }
    }
518380
def encode_sentence(sentence):
    """"""
    tree = etree.Element('sentence')
    encode_infons(tree, sentence.infons)
    etree.SubElement(tree, 'offset').text = str(sentence.offset)
    if sentence.text:
        etree.SubElement(tree, 'text').text = sentence.text
    for ann in sentence.annotations:
        tree.append(encode_annotation(ann))
    for rel in sentence.relations:
        tree.append(encode_relation(rel))
    return tree
504626
def split_long_sentence(sentence, words_per_line):
    
    words = sentence.split(' ')
    split_sentence = ''
    for i in range(len(words)):
        split_sentence = split_sentence + words[i]
        if (i+1) % words_per_line == 0:
            split_sentence = split_sentence + '\n'
        elif i != len(words) - 1:
            split_sentence = split_sentence + " "
    return split_sentence
696152
public static function cut($sentence, $options = array())
    {

        $defaults = array(
            'mode'=>'default'
        );

        $options = array_merge($defaults, $options);

        $seg_list = array();

        $re_cjk_pattern = '([\x{3040}-\x{309F}]+)|([\x{30A0}-\x{30FF}]+)|([\x{4E00}-\x{9FA5}]+)|([\x{AC00}-\x{D7AF}]+)';
        $re_skip_pattern = '([a-zA-Z0-9+#&=\._\r\n]+)';
        preg_match_all(
            '/('.$re_cjk_pattern.'|'.$re_skip_pattern.')/u',
            $sentence,
            $matches,
            PREG_PATTERN_ORDER
        );
        $blocks = $matches[0];

        foreach ($blocks as $blk) {

            if (preg_match('/'.$re_cjk_pattern.'/u', $blk)) {

                $words = self::__cut($blk);

                foreach ($words as $word) {
                    array_push($seg_list, $word);
                }

            } else {

                array_push($seg_list, $blk);

            }// end else (preg_match('/'.$re_han_pattern.'/u', $blk))


        }// end foreach ($blocks as $blk)

        return $seg_list;

    }
692722
public static function cut($sentence, $cut_all = false, $options = array("HMM" => true))
    {
        $defaults = array(
            'mode'=>'default'
        );

        $options = array_merge($defaults, $options);

        $seg_list = array();

        $re_han_pattern = '([\x{4E00}-\x{9FA5}]+)';
        $re_han_with_ascii_pattern = '([\x{4E00}-\x{9FA5}a-zA-Z0-9+#&=\._]+)';
        $re_kanjikana_pattern = '([\x{3040}-\x{309F}\x{4E00}-\x{9FA5}]+)';
        $re_katakana_pattern = '([\x{30A0}-\x{30FF}]+)';
        $re_hangul_pattern = '([\x{AC00}-\x{D7AF}]+)';
        $re_ascii_pattern = '([a-zA-Z0-9+#&=\._\r\n]+)';
        $re_skip_pattern = '(\s+)';
        if ($cut_all) {
            $re_skip_pattern = '([a-zA-Z0-9+#&=\._\r\n]+)';
        }
        $re_punctuation_pattern = '([\x{ff5e}\x{ff01}\x{ff08}\x{ff09}\x{300e}'.
                                    '\x{300c}\x{300d}\x{300f}\x{3001}\x{ff1a}\x{ff1b}'.
                                    '\x{ff0c}\x{ff1f}\x{3002}]+)';

        if (self::$cjk_all) {
            $filter_pattern = $re_kanjikana_pattern.
                            '|'.$re_katakana_pattern.
                            '|'.$re_hangul_pattern;
        } else {
            $filter_pattern = $re_han_with_ascii_pattern;
        }

        preg_match_all(
            '/('.$filter_pattern.'|'.$re_ascii_pattern.'|'.$re_punctuation_pattern.')/u',
            $sentence,
            $matches,
            PREG_PATTERN_ORDER
        );
        $blocks = $matches[0];

        foreach ($blocks as $blk) {
            if (mb_strlen($blk, 'UTF-8')==0) {
                continue;
            }
            if (self::$cjk_all) {
                // skip korean
                $filter_pattern = $re_kanjikana_pattern.'|'.$re_katakana_pattern;
            } else {
                $filter_pattern = $re_han_with_ascii_pattern;
            }

            if (preg_match('/'.$filter_pattern.'/u', $blk)) {
                if ($cut_all) {
                    $words = Jieba::__cutAll($blk);
                } else {
                    if ($options['HMM']) {
                        $words = Jieba::__cutDAG($blk);
                    } else {
                        $words = Jieba::__cutDAGNoHMM($blk);
                    }
                }

                foreach ($words as $word) {
                    array_push($seg_list, $word);
                }
            } elseif (preg_match('/'.$re_skip_pattern.'/u', $blk)) {
                preg_match_all(
                    '/('.$re_skip_pattern.')/u',
                    $blk,
                    $tmp,
                    PREG_PATTERN_ORDER
                );
                $tmp = $tmp[0];
                foreach ($tmp as $x) {
                    if (preg_match('/'.$re_skip_pattern.'/u', $x)) {
                        if (str_replace(' ', '', $x) != '') {
                            array_push($seg_list, $x);
                        }
                    } else {
                        if (!$cut_all) {
                            $xx_array = preg_split('//u', $x, -1, PREG_SPLIT_NO_EMPTY);
                            foreach ($xx_array as $xx) {
                                array_push($seg_list, $xx);
                            }
                        } else {
                            array_push($seg_list, $x);
                        }
                    }
                }
            } elseif (preg_match('/'.$re_punctuation_pattern.'/u', $blk)) {
                array_push($seg_list, $blk);
            }// end else (preg_match('/'.$re_han_pattern.'/u', $blk))
        }// end foreach ($blocks as $blk)

        return $seg_list;
    }
1359159
def parse(self, sentence):
        
        words = np.zeros((len(sentence) + 1, 1), np.int32)
        tags = np.zeros((len(sentence) + 1, 1), np.int32)
        words[0, 0] = ParserVocabulary.ROOT
        tags[0, 0] = ParserVocabulary.ROOT
        vocab = self._vocab

        for i, (word, tag) in enumerate(sentence):
            words[i + 1, 0], tags[i + 1, 0] = vocab.word2id(word.lower()), vocab.tag2id(tag)

        with mx.Context(mxnet_prefer_gpu()):
            outputs = self._parser.forward(words, tags)
        words = []
        for arc, rel, (word, tag) in zip(outputs[0][0], outputs[0][1], sentence):
            words.append(ConllWord(id=len(words) + 1, form=word, pos=tag, head=arc, relation=vocab.id2rel(rel)))
        return ConllSentence(words)