评估

@param corpus
@return
430364
public BinaryClassificationFMeasure evaluate(String corpus)
    {
        Instance[] instanceList = readInstance(corpus, model.featureMap);
        return evaluate(instanceList);
    }
840996
def ingest(self, corpus_file):
        '''  '''
        corpus = open(corpus_file)
        total_letters = 0
        total_words = 0
        shortest_word = 100
        for word in corpus.readlines():
            # clean word
            word = word.strip()
            word = re.sub(r'[\',\.\"]', '', word)
            total_letters += len(word)
            total_words += 1
            shortest_word = len(word) if len(word) < shortest_word else shortest_word

            # iterate through n letter groups, where 1 <= n <= 3
            n = self.chunk_size
            start = 0
            # >: C, Cys: t, yst: i
            self.links[self.initial].append(word[0:n])
            for position in range(n, len(word)):
                start = position - n if position - n >= 0 else 0
                base = word[start:position]
                if not base in self.links:
                    self.links[base] = []

                self.links[base].append(word[position])
            if not word[-n:] in self.links:
                self.links[word[-n:]] = []
            self.links[word[-n:]].append(self.terminal)

        self.average_word_length = total_letters / total_words
        self.shortest = shortest_word
34604
def generate_corpus(self, text):
        
        if isinstance(text, str):
            sentences = self.sentence_split(text)
        else:
            sentences = []
            for line in text:
                sentences += self.sentence_split(line)
        passing = filter(self.test_sentence_input, sentences)
        runs = map(self.word_split, passing)
        return runs
338630
static boolean makeCoreDictionary(String inPath, String outPath)
    {
        final DictionaryMaker dictionaryMaker = new DictionaryMaker();
        final TreeSet<String> labelSet = new TreeSet<String>();

        CorpusLoader.walk(inPath, new CorpusLoader.Handler()
        {
            @Override
            public void handle(Document document)
            {
                for (List<Word> sentence : document.getSimpleSentenceList(true))
                {
                    for (Word word : sentence)
                    {
                        if (shouldInclude(word))
                            dictionaryMaker.add(word);
                    }
                }
//                for (List<Word> sentence : document.getSimpleSentenceList(false))
//                {
//                    for (Word word : sentence)
//                    {
//                        if (shouldInclude(word))
//                            dictionaryMaker.add(word);
//                    }
//                }
            }

            /**
             * 是否应当计算这个词语
             * @param word
             * @return
             */
            boolean shouldInclude(Word word)
            {
                if ("m".equals(word.label) || "mq".equals(word.label) || "w".equals(word.label) || "t".equals(word.label))
                {
                    if (!TextUtility.isAllChinese(word.value)) return false;
                }
                else if ("nr".equals(word.label))
                {
                    return false;
                }

                return true;
            }
        });
        if (outPath != null)
        return dictionaryMaker.saveTxtTo(outPath);
        return false;
    }
140121
public CorpusConfig getCorpusConfigWithCache(String corpus)
  {
    CorpusConfig config = new CorpusConfig();
    if (corpusConfigCache != null)
    {
      config = corpusConfigCache.getIfPresent(corpus);
      if (config == null)
      {
        if (corpus.equals(DEFAULT_CONFIG))
        {
          config = Helper.getDefaultCorpusConfig();
        }
        else
        {
          config = Helper.getCorpusConfig(corpus);
        }

        corpusConfigCache.put(corpus, config);
      }
    }

    return config;
  }
539032
public static Map<String, ZipEntry> corporaInZipfile(ZipFile zip) throws IOException
  {
    Map<String, ZipEntry> result = new HashMap<>();
    
    for(ZipEntry e : getANNISEntry(zip, "corpus"))
    {
      String name = extractToplevelCorpusNames(zip.getInputStream(e));
      result.put(name, e);
    }
    
    return result;
  }
1334266
def validate(self, corpus):
        

        overflow_segments = {}

        for utterance in corpus.utterances.values():
            utt_segments = self.validate_utterance(utterance)

            if len(utt_segments) > 0:
                overflow_segments[utterance.idx] = utt_segments

        passed = len(overflow_segments) <= 0
        info = {
            'Label-List ID': self.label_list_idx,
            'Threshold': str(self.threshold)
        }

        return LabelOverflowValidationResult(passed, overflow_segments, self.name(), info)
780096
public double[][] classify(List corpus) throws Exception {
		Document[] documents = (Document[]) corpus.toArray(new Document[corpus
				.size()]);
		return classify(documents);
	}
1143871
def add_phrases(self, corpus):
		'''
		
		'''
		from gensim.models import Phrases

		assert isinstance(corpus, ParsedCorpus)
		self.phrases = [Phrases(CorpusAdapterForGensim.get_sentences(corpus), delimiter=' ')]

		for i in range(1, self.max_tokens_per_phrase):
			self.phrases.append(Phrases(self.phrases[-1][CorpusAdapterForGensim.get_sentences(corpus)]))

		return self
1074641
def word_freqs() -> List[Tuple[str, int]]:
    
    lines = list(get_corpus(_FILENAME))
    listword = []
    for line in lines:
        listindata = line.split("\t")
        listword.append((listindata[0], int(listindata[1])))

    return listword
1137388
def validate(self, corpus):
        
        invalid_utterances = {}

        for utterance in corpus.utterances.values():
            if self.label_list_idx in utterance.label_lists.keys():
                ll = utterance.label_lists[self.label_list_idx]

                if len(ll) < self.min_number_of_labels:
                    invalid_utterances[utterance.idx] = 'Only {} labels'.format(len(ll))
            else:
                invalid_utterances[utterance.idx] = 'No label-list {}'.format(self.label_list_idx)

        passed = len(invalid_utterances) <= 0
        info = {
            'Min. number of labels': str(self.min_number_of_labels),
            'Label-List ID': self.label_list_idx
        }

        return base.InvalidUtterancesResult(passed, invalid_utterances, name=self.name(), info=info)
721167
def validate(self, corpus):
        

        passed = True
        results = {}

        for validator in self.validators:
            sub_result = validator.validate(corpus)
            results[validator.name()] = sub_result

            if not sub_result.passed:
                passed = False

        return CombinedValidationResult(passed, results)
676206
private String getTopLevelCorpusFromTmpArea()
  {
    String sql = "SELECT name FROM " + tableInStagingArea("corpus")
      + " WHERE type='CORPUS'\n"
      + "AND pre = (SELECT min(pre) FROM " + tableInStagingArea("corpus")
      + ")\n"
      + "AND post = (SELECT max(post) FROM " + tableInStagingArea("corpus")
      + ")";

    return getJdbcTemplate().query(sql, new ResultSetExtractor<String>()
    {
      @Override
      public String extractData(ResultSet rs) throws SQLException,
        DataAccessException
      {
        if (rs.next())
        {
          return rs.getString("name");
        }
        else
        {
          return null;
        }
      }
    });
  }
607539
def corpusindex():
        """"""
        corpora = []
        for f in glob.glob(settings.ROOT + "corpora/*"):
            if os.path.isdir(f):
                corpora.append(os.path.basename(f))
        return corpora
1204825
@GET
  @Produces("text/plain")
  @Path("check")
  public String check(@QueryParam("q") String query, 
    @DefaultValue("") @QueryParam("corpora") String rawCorpusNames)
  {
    Subject user = SecurityUtils.getSubject();
    List<String> corpusNames = splitCorpusNamesFromRaw(rawCorpusNames);
    for (String c : corpusNames)
    {
      user.checkPermission("query:parse:" + c);
    }
    Collections.sort(corpusNames);
    
    List<Long> corpusIDs = queryDao.mapCorpusNamesToIds(
      corpusNames);
    
    queryDao.parseAQL(query, corpusIDs);
    return "ok";
  }
531039
def _from_dict(cls, _dict):
        """"""
        args = {}
        if 'corpora' in _dict:
            args['corpora'] = [
                Corpus._from_dict(x) for x in (_dict.get('corpora'))
            ]
        else:
            raise ValueError(
                'Required property \'corpora\' not present in Corpora JSON')
        return cls(**args)
268542
def validate(self, corpus):
        
        invalid_utterances = {}

        for utterance in corpus.utterances.values():
            duration = utterance.duration
            ll = utterance.label_lists[self.label_list_idx]

            # We count the characters of all labels
            transcription = ' '.join([l.value for l in ll])
            num_chars = len(transcription.replace(' ', ''))

            char_per_sec = num_chars / duration

            if char_per_sec > self.max_characters_per_second:
                invalid_utterances[utterance.idx] = char_per_sec

        passed = len(invalid_utterances) <= 0
        info = {
            'Threshold max. characters per second': str(self.max_characters_per_second),
            'Label-List ID': self.label_list_idx
        }

        return base.InvalidUtterancesResult(passed, invalid_utterances, name=self.name(), info=info)