Build and return a frequency distribution over xpath occurrences.
338870
def get_xpath_frequencydistribution(paths):
    
    # "html/body/div/div/text" -> [ "html", "body", "div", "div", "text" ]
    splitpaths = [p.rsplit('/', 1) for p in paths]

    # get list of "parentpaths" by right-stripping off the last xpath-node,
    # effectively getting the parent path
    # thanks to eugene-eeo for optimization
    parentpaths = [p[0] for p in splitpaths]

    # build frequency distribution
    parentpaths_counter = Counter(parentpaths)
    return parentpaths_counter.most_common()
68579
def serialize(self, occur=None):
        
        fmt = self.ser_format.get(self.name, SchemaNode._default_format)
        return fmt(self, occur) % (escape(self.text) +
                                   self.serialize_children())
4085
def _case_format(self, occur):
        """""" 
        if self.occur == 1:
            self.attr["nma:implicit"] = "true"
        ccnt = len(self.rng_children())
        if ccnt == 0: return "<empty/>%s"
        if ccnt == 1 or not self.interleave:
            return self.start_tag("group") + "%s" + self.end_tag("group")
        return (self.start_tag("interleave") + "%s" +
                self.end_tag("interleave"))
925086
def countRandomBitFrequencies(numTerms = 100000, percentSparsity = 0.01):
  """"""
  # Accumulate counts by inplace-adding sparse matrices
  counts = SparseMatrix()
  size = 128*128
  counts.resize(1, size)

  # Pre-allocate buffer sparse matrix
  sparseBitmap = SparseMatrix()
  sparseBitmap.resize(1, size)

  random.seed(42)

  # Accumulate counts for each bit for each word
  numWords=0
  for term in xrange(numTerms):
    bitmap = random.sample(xrange(size), int(size*percentSparsity))
    bitmap.sort()

    sparseBitmap.setRowFromSparse(0, bitmap, [1]*len(bitmap))
    counts += sparseBitmap
    numWords += 1

  # Compute normalized version of counts as a separate matrix
  frequencies = SparseMatrix()
  frequencies.resize(1, size)
  frequencies.copy(counts)
  frequencies.divide(float(numWords))

  # Wrap up by printing some statistics and then saving the normalized version
  printFrequencyStatistics(counts, frequencies, numWords, size)

  frequencyFilename = "bit_frequencies_random.pkl"
  print "Saving frequency matrix in",frequencyFilename
  with open(frequencyFilename, "wb") as frequencyPickleFile:
    pickle.dump(frequencies, frequencyPickleFile)

  return counts
151516
private static Map<GroupHit, Spans> collectSpansForOccurences(
      Set<GroupHit> occurences, Set<String> prefixes, String field,
      IndexSearcher searcher, LeafReaderContext lrc) throws IOException {
    Map<GroupHit, Spans> list = new HashMap<>();
    IndexReader reader = searcher.getIndexReader();
    final float boost = 0;
    for (GroupHit hit : occurences) {
      MtasSpanQuery queryHit = createQueryFromGroupHit(prefixes, field, hit);
      if (queryHit != null) {
        MtasSpanQuery queryHitRewritten = queryHit.rewrite(reader);
        SpanWeight weight = queryHitRewritten.createWeight(searcher, false,
            boost);
        Spans spans = weight.getSpans(lrc, SpanWeight.Postings.POSITIONS);
        if (spans != null) {
          list.put(hit, spans);
        }
      }
    }
    return list;
  }
723378
def xpath(node, query, namespaces={}):
    """"""
    if namespaces and 'None' not in namespaces:
        return node.xpath(query, namespaces=namespaces)
    return node.xpath(query)
484454
def rspr(self, times=1, **kwargs):
        

        spr = SPR(self.copy())
        for _ in range(times):
            spr.rspr(**kwargs)
        return spr.tree
451668
def freq_from_iterators(cls, iterators):
        
        return {
            set(it): f for f, it in cls.FREQUENCIES.items()}[set(iterators)]
1076920
def set_frequency(self, host, sem=None, interval=None):
        """"""
        # single sem or global sem
        sem = sem or self.sem
        interval = self.interval if interval is None else interval
        frequency = Frequency(sem, interval, host)
        frequencies = {host: frequency}
        self.update_frequency(frequencies)
        return frequency
109060
def make_xpath_ranges(html, phrase):
    '''

    '''
    if not html:
        return []
    if not isinstance(phrase, unicode):
        try:
            phrase = phrase.decode('utf8')
        except:
            logger.info('failed %r.decode("utf8")', exc_info=True)
            return []

    phrase_re = re.compile(
        phrase, flags=re.UNICODE | re.IGNORECASE | re.MULTILINE)
    spans = []
    for match in phrase_re.finditer(html, overlapped=False):
        spans.append(match.span())  # a list of tuple(start, end) char indexes

    # now run fancy aligner magic to get xpath info and format them as
    # XPathRange per above
    try:
        xpath_ranges = list(char_offsets_to_xpaths(html, spans))
    except:
        logger.info('failed to get xpaths', exc_info=True)
        return []
    ranges = []
    for xpath_range in filter(None, xpath_ranges):
        ranges.append(dict(
            start=dict(node=xpath_range.start_xpath,
                       idx=xpath_range.start_offset),
            end=dict(node=xpath_range.end_xpath,
                     idx=xpath_range.end_offset)))

    return ranges
433411
def construct(self, data):
		
		occurrences = {}
		main_occurrences = {}

		# occurrences of devel packages
		for pkg in data["data"]["dependencies"]:
			package = pkg["package"]
			for item in pkg["dependencies"]:
				dep = item["name"]
				if package != ".":
					deps = map(lambda l: "%s/%s" % (package, l), item["location"])
				else:
					deps = item["location"]
				if dep not in occurrences:
					occurrences[dep] = deps
				else:
					occurrences[dep] = occurrences[dep] + deps

		self.occurrences = occurrences

		# occurrences of main packages
		for main in data["data"]["main"]:
			filename = main["filename"]
			for dep in main["dependencies"]:
				if dep not in main_occurrences:
					main_occurrences[dep] = [filename]
				else:
					main_occurrences[dep].append(filename)

		self.main_occurrences = main_occurrences
	
		# test directories
		self.test_directories = sorted(map(lambda l: l["test"], data["data"]["tests"]))

		# provided devel packages
		self.provided_packages = sorted(data["data"]["packages"])

		# imported paths in devel packages
		imported_packages = []
		imported_native_packages = []
		for path in occurrences:
			try:
				self.ipparser.parse(path)
			except ValueError:
				continue

			if self.ipparser.isNative():
				imported_native_packages.append(path)
			else:
				imported_packages.append(path)

		self.imported_packages = sorted(imported_packages)
		self.imported_native_packages = sorted(imported_native_packages)

		# main packages
		self.main_packages = map(lambda l: l["filename"], data["data"]["main"])
1050680
def _extract_all(node, name, limit_count=None):
    """"""

    seq = []

    for i in range(0, len(node.getElementsByTagName(name))):
        if len(seq) == limit_count:
            break

        seq.append(_extract(node, name, i))

    return seq
640602
XPath getCountMatchPattern(XPathContext support, int contextNode)
          throws javax.xml.transform.TransformerException
  {

    XPath countMatchPattern = m_countMatchPattern;
    DTM dtm = support.getDTM(contextNode);
    if (null == countMatchPattern)
    {
      switch (dtm.getNodeType(contextNode))
      {
      case DTM.ELEMENT_NODE :
        MyPrefixResolver resolver;

        if (dtm.getNamespaceURI(contextNode) == null) {
             resolver =  new MyPrefixResolver(dtm.getNode(contextNode), dtm,contextNode, false);
        } else {
            resolver = new MyPrefixResolver(dtm.getNode(contextNode), dtm,contextNode, true);
        }

        countMatchPattern = new XPath(dtm.getNodeName(contextNode), this, resolver,
                                      XPath.MATCH, support.getErrorListener());
        break;

      case DTM.ATTRIBUTE_NODE :

        // countMatchPattern = m_stylesheet.createMatchPattern("@"+contextNode.getNodeName(), this);
        countMatchPattern = new XPath("@" + dtm.getNodeName(contextNode), this,
                                      this, XPath.MATCH, support.getErrorListener());
        break;
      case DTM.CDATA_SECTION_NODE :
      case DTM.TEXT_NODE :

        // countMatchPattern = m_stylesheet.createMatchPattern("text()", this);
        countMatchPattern = new XPath("text()", this, this, XPath.MATCH, support.getErrorListener());
        break;
      case DTM.COMMENT_NODE :

        // countMatchPattern = m_stylesheet.createMatchPattern("comment()", this);
        countMatchPattern = new XPath("comment()", this, this, XPath.MATCH, support.getErrorListener());
        break;
      case DTM.DOCUMENT_NODE :

        // countMatchPattern = m_stylesheet.createMatchPattern("/", this);
        countMatchPattern = new XPath("/", this, this, XPath.MATCH, support.getErrorListener());
        break;
      case DTM.PROCESSING_INSTRUCTION_NODE :

        // countMatchPattern = m_stylesheet.createMatchPattern("pi("+contextNode.getNodeName()+")", this);
        countMatchPattern = new XPath("pi(" + dtm.getNodeName(contextNode)
                                      + ")", this, this, XPath.MATCH, support.getErrorListener());
        break;
      default :
        countMatchPattern = null;
      }
    }

    return countMatchPattern;
  }
817341
def ensure_frequencies(self, frequencies):
        """"""
        if not frequencies:
            return {}
        if not isinstance(frequencies, dict):
            raise ValueError("frequencies should be dict")
        frequencies = {
            host: Frequency.ensure_frequency(frequencies[host]) for host in frequencies
        }
        return frequencies
1226229
def xpath_pick_one(self, xpaths):
		
		for xpathi, xpath in enumerate(xpaths):
			try:
				return self.xpath(xpath, [1, 1])[0]
			except UnexpectedContentException as e:
				if xpathi == len(xpaths) - 1:
					raise
1292573
private long getFrequentItemsets(PrintStream out, List<ItemSet> list) {
        long n = 0;
        if (root.children != null) {
            for (int i = 0; i < root.children.length; i++) {
                Node child = root.children[i];
                if (child != null && child.support >= minSupport) {
                    int[] itemset = {child.id};
                    n += getFrequentItemsets(out, list, itemset, i, child);
                }
            }
        }
        
        return n;
    }
680114
def schematron(self, fn=None, outfn=None, ext='.sch'):
        """"""
        from .xslt import XSLT
        from . import PATH, XML, etree
        fn = fn or self.fn
        if os.path.splitext(fn)[-1].lower()==ext:
            return fn
        elif os.path.splitext(fn)[-1].lower()!='.rng':
            fn = Schema(fn=fn).trang(ext='.rng')
        rng2sch = XSLT(fn=os.path.join(PATH, 'xslts', 'rng2sch.xslt'))
        rng = XML(fn=fn)
        outfn = outfn or os.path.splitext(fn)[0]+ext
        sch = XML(fn=outfn, root=rng2sch.saxon9(rng.root).getroot())
        sch.write()
        return sch.fn