Takes in a set of training pairs and predicates and tries to find
        a good set of blocking rules.
557658
def learn(self, matches, recall):
        '''
        
        '''
        compound_length = 2

        dupe_cover = Cover(self.blocker.predicates, matches)
        dupe_cover.dominators(cost=self.total_cover)
        dupe_cover.compound(compound_length)

        comparison_count = self.comparisons(dupe_cover, compound_length)

        dupe_cover.dominators(cost=comparison_count, comparison=True)

        coverable_dupes = set.union(*viewvalues(dupe_cover))
        uncoverable_dupes = [pair for i, pair in enumerate(matches)
                             if i not in coverable_dupes]

        epsilon = int((1.0 - recall) * len(matches))

        if len(uncoverable_dupes) > epsilon:
            logger.warning(OUT_OF_PREDICATES_WARNING)
            logger.debug(uncoverable_dupes)
            epsilon = 0
        else:
            epsilon -= len(uncoverable_dupes)

        for pred in dupe_cover:
            pred.count = comparison_count[pred]

        searcher = BranchBound(len(coverable_dupes) - epsilon, 2500)
        final_predicates = searcher.search(dupe_cover)

        logger.info('Final predicate set:')
        for predicate in final_predicates:
            logger.info(predicate)

        return final_predicates
413457
def train(self, sentences, nr_iter=5):
        
        self._make_tagdict(sentences)
        self.perceptron.classes = self.classes
        for iter_ in range(nr_iter):
            c = 0
            n = 0
            for sentence in sentences:
                prev, prev2 = self.START
                context = [t[0] for t in sentence]
                for i, (token, tag) in enumerate(sentence):
                    guess = self.tagdict.get(token)
                    if not guess:
                        feats = self._get_features(i, context, prev, prev2)
                        guess = self.perceptron.predict(feats)
                        self.perceptron.update(tag, guess, feats)
                    prev2 = prev
                    prev = guess
                    c += guess == tag
                    n += 1
            random.shuffle(sentences)
            log.debug('Iter %s: %s/%s=%s' % (iter_, c, n, (float(c) / n) * 100))
        self.perceptron.average_weights()
216663
def build_trainer(nn, ds, verbosity=1):
    """"""
    return pb.supervised.trainers.rprop.RPropMinusTrainer(nn, dataset=ds, batchlearning=True, verbose=bool(verbosity))
735304
def backtracking(a, L, bestsofar):
    
    w, j = max(L.items())
    while j != -1:
        yield j
        w, j = bestsofar[j]
527791
def kfold(dataset, problem, method, k=10):
    

    if k <= 1:
        raise ValueError("k argument must be at least 2")

    dataset = list(dataset)
    random.shuffle(dataset)

    trials = 0
    positive = 0
    for i in range(k):
        train = [x for j, x in enumerate(dataset) if j % k != i]
        test = [x for j, x in enumerate(dataset) if j % k == i]
        classifier = method(train, problem)
        for data in test:
            trials += 1
            result = classifier.classify(data)
            if result is not None and result[0] == problem.target(data):
                positive += 1

    return float(positive) / float(trials)
1023326
public function setupRules()
    {
        $this->rules->addMany([
            'keyword' => new Rule(new WordMatcher([
                'auto', 'break', 'case', 'const', 'continue', 'default', 'do', 'else', 'enum', 'extern', 'for', 'goto',
                'if', 'register', 'return', 'sizeof', 'static', 'struct', 'switch', 'typedef', 'union', 'volatile',
                'while',
            ])),

            'meta.newline' => new CloseRule(new RegexMatcher('/()\R/m'), [
                'factory' => new TokenFactory(TerminatorToken::class),
                'context' => ['!operator.escape', '*string', '*call.preprocessor'],
                'closes'  => ['string.double', 'preprocessor'],
            ]),

            'preprocessor' => new OpenRule(new RegexMatcher('/^(#)/m'), [
                'context' => Validator::everywhere(),
                'priority' => -1
            ]),

            'call' => [
                'preprocessor' => new Rule(new RegexMatcher('/^#\s*(\w+)\b/m'), [
                    'context' => ['preprocessor']
                ]),
                new Rule(new RegexMatcher('/([a-z_]\w*)\s*\(/i'), ['priority' => -1]),
            ],

            'operator.escape' => new Rule(new RegexMatcher('/(\\\(?:\R|.|[0-7]{3}|x\x{2}))/s'), [
                'context' => Validator::everywhere()
            ]),

            'keyword.format' => new Rule(
                new RegexMatcher('/(%[diuoxXfFeEgGaAcspn%][-+#0]?(?:[0-9]+|\*)?(?:\.(?:[0-9]+|\*))?)/'), [
                    'context' => ['string']
                ]
            ),

            'string' => array_merge([
                new Rule(new RegexMatcher('/(<.*?>)/'), [
                    'context' => ['preprocessor']
                ])
            ], CommonFeatures::strings(['single' => '\'', 'double' => '"'], [
                'context' => ['!operator.escape', '!comment', '!string'],
            ])),

            'symbol.type' => [
                new Rule(new RegexMatcher('/(\w+)(?:\s+|\s*\*\s*)\w+\s*[=();,]/'), ['name' => 'universal']),
                new Rule(new WordMatcher(['int', 'float', 'double', 'char', 'void', 'long', 'short', 'signed', 'unsigned']), ['name' => 'builtin'])
            ],

            'comment' => [
                new Rule(new CommentMatcher(['//'], [['/*', '*/']]), ['priority' => 2])
            ],

            'variable.property' => new Rule(new RegexMatcher('/(?=(?:\w|\)|\])\s*(?:->|\.)\s*([a-z_]\w*))/'), ['priority' => 0]),

            'number' => new Rule(new RegexMatcher('/\b(-?(?:0x[\da-f]+|\d*\.?\d+(?:e[+-]?\d+)?)[ful]*)\b/i')),
            'operator' => [
                'punctuation' => new Rule(new RegexMatcher('/([;,])/'), ['priority' => 0]),
                new Rule(new RegexMatcher('/([*&])/'), ['priority' => 0]),
                new Rule(new RegexMatcher('/([!+\-\/*&|^<>=]{1,2}=?)/'), ['priority' => 0])
            ],

            'language.c' => [
                new Rule(new RegexMatcher('/^#define\s+\w+(.*?)(?>[^\\\]\r\n|[^\\\\\r]\n|\Z)/sim'), [
                    'factory' => new TokenFactory(LanguageToken::class),
                    'inject'  => $this,
                    'context' => ['preprocessor'],
                    'priority' => 10
                ]),
            ]
        ]);
    }
808917
private static Set<Integer> find(final Predicate<Instruction> predicate, final List<Instruction> instructions) {
        final Set<Integer> positions = new HashSet<>();

        for (int i = 0; i < instructions.size(); i++) {
            final Instruction instruction = instructions.get(i);
            if (predicate.test(instruction)) {
                positions.add(i);
            }
        }

        return positions;
    }
1235568
def _supervised_evaluation_error_checking(targets, predictions):
    
    _raise_error_if_not_sarray(targets, "targets")
    _raise_error_if_not_sarray(predictions, "predictions")
    if (len(targets) != len(predictions)):
        raise _ToolkitError(
         "Input SArrays 'targets' and 'predictions' must be of the same length.")
905964
def _get_predicate_guards(self, state, state_training_data):
        
        # choose the sink transition.

        # First option: Just the maximum transition
        # sink = max(state_training_data, key=lambda x: len(x[1]))[0]

        # Second option: Heuristics based on RE filters properties
        max_size_trans = max(state_training_data, key=lambda x: len(x[1]))
        max_size_trans_l = [x for x in state_training_data if
                            len(x[1]) == len(max_size_trans[1])]

        target_states = [t[0] for t in max_size_trans_l]
        if len(max_size_trans_l) == 1:
            sink = max_size_trans[0]
        elif '' in target_states:
            sink = ''
        elif state in target_states:
            sink = state
        else:
            sink = random.choice(target_states)
        # End of sink selection

        transitions = []
        known_symbols = []
        for (t, data) in state_training_data:
            if t == sink:
                continue
            pred = SetPredicate(data)
            transitions.append((t, pred))
            known_symbols += data
        transitions.append(
            (sink, SetPredicate(set(self.alphabet) - set(known_symbols))))
        return transitions
1166188
def meet(self, featuresets):
        """"""
        concepts = (f.concept for f in featuresets)
        meet = self.lattice.meet(concepts)
        return self._featuresets[meet.index]
756101
def getSrcBlocks(self, url, dataset="", block=""):
        
        if block:
            params={'block_name':block, 'open_for_writing':0}
        elif dataset:
            params={'dataset':dataset, 'open_for_writing':0}
        else:
            m = 'DBSMigration: Invalid input.  Either block or dataset name has to be provided'
            e = 'DBSMigrate/getSrcBlocks: Invalid input.  Either block or dataset name has to be provided'
            dbsExceptionHandler('dbsException-invalid-input2', m, self.logger.exception, e )

        return cjson.decode(self.callDBSService(url, 'blocks', params, {}))
474394
private Set<Predicate> copyPredicatesSet() {
    Set<Predicate> predicatesCopy = Sets.newLinkedHashSet();
    for (Predicate predicate : this.predicates) {
      Predicate copyPredicate = new Predicate();
      copyPredicate.setField(predicate.getField());
      copyPredicate.setOperator(predicate.getOperator());
      copyPredicate.getValues().addAll(predicate.getValues());
      predicatesCopy.add(copyPredicate);
    }
    return predicatesCopy;
  }
541106
def _rule_block(self):
        

        name = self._get_token(self.RE_NAME)
        self._expect_token('{')

        # consume additional options if available
        options = []
        while self._lookahead_token() != '}':
            options.append(self._rule_option())

        self._expect_token('}')
        return [name, options]
830647
private void attemptMatch(BasicBlock basicBlock, BasicBlock.InstructionIterator instructionIterator)
            throws DataflowAnalysisException {
        work(new State(basicBlock, instructionIterator, pattern.getFirst()));
    }
1332837
function optimizeRules( optimizedRules, rules ) {
		var elementsRules = optimizedRules.elements,
			genericRules = optimizedRules.generic,
			i, l, rule, element, priority;

		for ( i = 0, l = rules.length; i < l; ++i ) {
			// Shallow copy. Do not modify original rule.
			rule = copy( rules[ i ] );
			priority = rule.classes === true || rule.styles === true || rule.attributes === true;
			optimizeRule( rule );

			// E.g. "*(xxx)[xxx]" - it's a generic rule that
			// validates properties only.
			// Or '$1': { match: function() {...} }
			if ( rule.elements === true || rule.elements === null ) {
				// Add priority rules at the beginning.
				genericRules[ priority ? 'unshift' : 'push' ]( rule );
			}
			// If elements list was explicitly defined,
			// add this rule for every defined element.
			else {
				// We don't need elements validator for this kind of rule.
				var elements = rule.elements;
				delete rule.elements;

				for ( element in elements ) {
					if ( !elementsRules[ element ] )
						elementsRules[ element ] = [ rule ];
					else
						elementsRules[ element ][ priority ? 'unshift' : 'push' ]( rule );
				}
			}
		}
	}
1275433
def each(&block)
      Enumerator.new do |y|
        @rules.each do |key, node|
          y << entry_to_rule(node, key)
        end
      end.each(&block)
    end
408411
def _train_and_eval_dataset_v1(problem_name, data_dir):
  """"""
  assert not tf.executing_eagerly(), "tf.eager mode must be turned off."
  problem = t2t_problems.problem(problem_name)
  train_dataset = problem.dataset(tf.estimator.ModeKeys.TRAIN, data_dir)
  train_dataset = train_dataset.map(_select_features)
  eval_dataset = problem.dataset(tf.estimator.ModeKeys.EVAL, data_dir)
  eval_dataset = eval_dataset.map(_select_features)
  hparams = problem.get_hparams()
  # We take a few training examples to guess the shapes.
  input_shapes, target_shapes = [], []
  example_tensor = train_dataset.make_one_shot_iterator().get_next()
  sess = tf.Session()
  example1 = sess.run(example_tensor)
  example2 = sess.run(example_tensor)
  example3 = sess.run(example_tensor)
  # We use "inputs" as input except for purely auto-regressive tasks like
  # language models where "targets" are used as input_key.
  input_key = "inputs" if "inputs" in example1 else "targets"
  supervised_keys = ([input_key], ["targets"])
  for example in [example1, example2, example3]:
    input_shapes.append(list(example[input_key].shape))
    target_shapes.append(list(example["targets"].shape))
  input_vocab_size = hparams.vocab_size[input_key]
  target_vocab_size = hparams.vocab_size["targets"]
  input_info = _make_info(input_shapes, input_vocab_size)
  target_info = _make_info(target_shapes, target_vocab_size)
  info = {input_key: input_info, "targets": target_info}
  return train_dataset, eval_dataset, info, supervised_keys