Generates WikipediaArticles from GCS that are part of shard shard_id.
494345
def _wiki_articles(shard_id, wikis_dir=None):
  """"""
  if not wikis_dir:
    wikis_dir = WIKI_CONTENT_DIR
  with tf.Graph().as_default():
    dataset = tf.data.TFRecordDataset(
        cc_utils.readahead(
            os.path.join(wikis_dir, WIKI_CONTENT_FILE % shard_id)),
        buffer_size=16 * 1000 * 1000)

    def _parse_example(ex_ser):
      """Parse serialized Example containing Wikipedia article content."""
      features = {
          "url": tf.VarLenFeature(tf.string),
          "title": tf.VarLenFeature(tf.string),
          "section_titles": tf.VarLenFeature(tf.string),
          "section_texts": tf.VarLenFeature(tf.string),
      }
      ex = tf.parse_single_example(ex_ser, features)
      for k in ex.keys():
        ex[k] = ex[k].values
      ex["url"] = ex["url"][0]
      ex["title"] = ex["title"][0]
      return ex

    dataset = dataset.map(_parse_example, num_parallel_calls=32)
    dataset = dataset.prefetch(100)
    record_it = dataset.make_one_shot_iterator().get_next()

    with tf.Session() as sess:
      while True:
        try:
          ex = sess.run(record_it)
        except tf.errors.OutOfRangeError:
          break

        sections = [
            WikipediaSection(title=text_encoder.to_unicode(title),
                             text=text_encoder.to_unicode(text))
            for title, text in zip(ex["section_titles"], ex["section_texts"])
        ]
        yield WikipediaArticle(
            url=text_encoder.to_unicode(ex["url"]),
            title=text_encoder.to_unicode(ex["title"]),
            sections=sections)
91211
def to_json(self):
    """"""
    result = {
        "mapper_handler_spec": self.handler_spec,
        "mapper_input_reader": self.input_reader_spec,
        "mapper_params": self.params,
        "mapper_shard_count": self.shard_count
    }
    if self.output_writer_spec:
      result["mapper_output_writer"] = self.output_writer_spec
    return result
268438
def finalize(self, ctx, shard_state):
    """"""
    filenames = []
    for filehandle in self._filehandles:
      filenames.append(filehandle.name)
      filehandle.close()
    shard_state.writer_state = {"shard_filenames": filenames}
1311196
def gendocs():
    ""
    helppage = path("docs/references-cli-usage.rst")
    content = [
        ".. automatically generated using 'paver gendocs'.",
        "",
        ".. contents::",
        "    :local:",
        "",
        ".. note::",
        "",
        "    The help output presented here applies to version ``%s`` of the tools."
            % sh("pyroadmin --version", capture=True).split()[1],
        "",
    ]

    for tool in sorted(project.entry_points["console_scripts"]):
        tool, _ = tool.split(None, 1)
        content.extend([
            ".. _cli-usage-%s:" % tool,
            "",
            tool,
            '^' * len(tool),
            "",
            "::",
            "",
        ])
        help_opt = "--help-fields --config-dir /tmp" if tool == "rtcontrol" else "--help"
        help_txt = sh("%s -q %s" % (tool, help_opt), capture=True, ignore_error=True).splitlines()
        content.extend('    ' + i for i in help_txt
                       if ' on Python ' not in i and 'Copyright (c) 200' not in i
                       and 'see the full documentation' not in i
                       and '    https://pyrocore.readthedocs.io/' not in i)
        content.extend([
            "",
        ])

    content = [line.rstrip() for line in content if all(
        i not in line for i in (", Copyright (c) ", "Total time: ", "Configuration file '/tmp/")
    )]
    content = [line for line, succ in zip(content, content[1:] + ['']) if line or succ] # filter twin empty lines
    helppage.write_lines(content)
1061976
def create_new(cls, mapreduce_id, shard_number):
    
    shard_id = cls.shard_id_from_number(mapreduce_id, shard_number)
    state = cls(key_name=shard_id,
                mapreduce_id=mapreduce_id)
    return state
466003
def generateDirectoryNodeDocuments(self):
        '''
        
        '''
        all_dirs = []
        for d in self.dirs:
            d.findNestedDirectories(all_dirs)

        for d in all_dirs:
            self.generateDirectoryNodeRST(d)
60088
def cdbs(client, event, channel, nick, rest):
	
	db_name, shard = shlex.split(rest)
	return sharding.create_db_in_shard(db_name, shard, client=get_client())
1293592
def extract_references_from_wets(wet_files, metadata_dir, out_dir,
                                 tmp_dir=None):
  """"""
  # Setup output files
  shard_files = make_ref_shard_files(out_dir)

  num_refs = 0
  for i, wet_file in enumerate(wet_files):
    num_refs_in_wet = 0
    tf.logging.info("Processing file %d", i)

    # Read metadata file
    metadata_fname = os.path.join(
        metadata_dir, os.path.basename(wet_file)) + cc_utils.METADTA_SUFFIX
    with tf.gfile.Open(cc_utils.readahead(metadata_fname)) as f:
      wet_metadata = json.loads(f.read())

    if not wet_metadata:
      # No references in this WET file
      continue

    if wet_file.startswith("http"):
      # download
      if not tmp_dir:
        tmp_dir = tempfile.gettempdir()
      record_gen = cc_utils.wet_records_from_url(wet_file, tmp_dir)
    else:
      # local
      record_gen = cc_utils.wet_records_from_file_obj(
          cc_utils.gzip_memfile(wet_file), take_ownership=True)

    for wet_record in record_gen:
      shard_ids = wet_metadata.get(wet_record.url)
      if not shard_ids:
        # URL not in dataset
        continue

      # Serialize and write out
      ex = _make_example_from_record(wet_record)
      ex_str = ex.SerializeToString()
      for shard_id in shard_ids:
        shard_files[shard_id].write(ex_str)
      num_refs += 1
      num_refs_in_wet += 1

    tf.logging.info("Wrote out %d references for this WET", num_refs_in_wet)

  tf.logging.info("Wrote out %d references total", num_refs)

  # Cleanup
  for shard_file in shard_files:
    shard_file.close()
1059440
def build_related_articles(related_articles):
    
    article_list = []

    for related_article in related_articles:
        article = ea.RelatedArticle()
        if related_article.get('xlink_href'):
            article.xlink_href = related_article.get('xlink_href')
        if related_article.get('related_article_type'):
            article.related_article_type = related_article.get('related_article_type')
        if related_article.get('ext_link_type'):
            article.ext_link_type = related_article.get('ext_link_type')

        # Append it to our list
        article_list.append(article)

    return article_list
503296
def generate_answers
      the_answers = {}
      dashboard = only_host_with_role(@hosts, 'dashboard')
      master = only_host_with_role(@hosts, 'master')
      @hosts.each do |h|
        the_answers[h.name] = host_answers(h, master, dashboard, @options)
        if the_answers[h.name] && h[:custom_answers]
          the_answers[h.name] = the_answers[h.name].merge(h[:custom_answers])
        end
        h[:answers] = the_answers[h.name]
      end
      return the_answers
    end
1103962
def iter_doc_filepaths(self, **kwargs):
        
        for shard in self._shards:
            for doc_id, blob in shard.iter_doc_filepaths(**kwargs):
                yield doc_id, blob
1343171
def _encode_wiki_sections(sections, vocab):
  """"""
  ids = []
  section_boundaries = []
  for i, section in enumerate(sections):
    if i > 0:
      # Skip including article title
      ids.extend(vocab.encode(_format_title(_normalize_text(section.title))))
    ids.extend(vocab.encode(_normalize_text(section.text)))
    section_boundaries.append(len(ids))

  return ids, section_boundaries
498350
private function InitArchiveArticles()
    {
        $sql = Access::SqlBuilder();
        $tblArticle = Article::Schema()->Table();
        $tblCategory = Category::Schema()->Table();
        $join = $sql->Join($tblCategory);
        $joinCond = $sql->Equals($tblArticle->Field('Category'), $tblCategory->Field('ID'));
        $orderBy = $sql->OrderList($sql->OrderDesc($tblArticle->Field('Created')));
        $where = $this->PublishedCondition()->
                And_($sql->Equals($tblCategory->Field('Archive'), $sql->Value($this->archive->GetID())));
        $this->articles = Article::Schema()->Fetch(false, $where, $orderBy, null, 
                $this->ArticleOffset(), $this->ArticleCount(), $join, Sql\JoinType::Inner(), $joinCond);
        $this->articlesTotal = Article::Schema()->Count(false, $where, null, $join, Sql\JoinType::Inner(), $joinCond);
    }
47535
def shard_query_generator(self, query):
    ''''''
    shard_query = query.copy()

    for shard in self._stores:
      # yield all items matching within this shard
      cursor = shard.query(shard_query)
      for item in cursor:
        yield item

      # update query with results of first query
      shard_query.offset = max(shard_query.offset - cursor.skipped, 0)
      if shard_query.limit:
        shard_query.limit = max(shard_query.limit - cursor.returned, 0)

        if shard_query.limit <= 0:
          break
1006812
def _generate_content(self):
        """"""
        # concat into a dict
        dic = self.config
        dic['mode'] = self.language
        if self.theme:
            dic['theme'] = self.theme
        # dumps with json
        return json.dumps(dic, indent=8, separators=(',', ': '))
659984
def wiki_create(self, title, body, other_names=None):
        
        params = {
            'wiki_page[title]': title,
            'wiki_page[body]': body,
            'wiki_page[other_names]': other_names
            }
        return self._get('wiki_pages.json', params, method='POST', auth=True)
1272952
def pages

			result_map = map_query_to_results

			query_result["query"]["pages"].each do |key, value|

				page_title = value["title"]
				original_query = find_result_map_match_for_title(result_map, page_title)
				@page_hash[original_query] = MediaWiki::Page.new(value)

			end

			@page_hash
		end