Gets the index of news crawl files from commoncrawl.org and returns an array of names
        :return:
331655
def __get_remote_index(self):
        
        # cleanup
        subprocess.getoutput("rm tmpaws.txt")
        # get the remote info
        cmd = "aws s3 ls --recursive s3://commoncrawl/crawl-data/CC-NEWS/ --no-sign-request > tmpaws.txt && " \
              "awk '{ print $4 }' tmpaws.txt && " \
              "rm tmpaws.txt"
        self.__logger.info('executing: %s', cmd)
        stdout_data = subprocess.getoutput(cmd)
        print(stdout_data)

        lines = stdout_data.splitlines()
        return lines
618133
def GetIndex(self):
        '''
        '''
        if self.index is None:
            return None

        if self.index:
            res, ui = chmlib.chm_resolve_object(self.file, self.index)
            if (res != chmlib.CHM_RESOLVE_SUCCESS):
                return None

        size, text = chmlib.chm_retrieve_object(self.file, ui, 0l, ui.length)
        if (size == 0):
            sys.stderr.write('GetIndex: file size = 0\n')
            return None
        return text
927922
def _read_indexlist(self, name):
        """"""
        setattr(self, '_' + name, [self._timeline[int(i)] for i in
                                   self.db.lrange('site:{0}'.format(name), 0,
                                                  -1)])
1184053
def list_search_indexes(self):
        
        if not self.yz_wm_index:
            raise NotImplementedError("Search 2.0 administration is not "
                                      "supported for this version")

        url = self.search_index_path()

        # Run the request...
        status, headers, body = self._request('GET', url)

        if status == 200:
            json_data = json.loads(bytes_to_str(body))
            # Return a list of dictionaries
            return json_data
        else:
            raise RiakError('Error getting Search 2.0 index.')
354556
def get_index_from_filename(self, filename):
        
        filenames = [d.filename for d in self.data]
        return filenames.index(filename)
1369969
def GetArchiveInfo(self):
        '''
        '''

        self.searchable = extra.is_searchable(self.file)
        self.lcid = None

        result, ui = chmlib.chm_resolve_object(self.file, '/#SYSTEM')
        if (result != chmlib.CHM_RESOLVE_SUCCESS):
            sys.stderr.write('GetArchiveInfo: #SYSTEM does not exist\n')
            return 0

        size, text = chmlib.chm_retrieve_object(self.file, ui, 4l, ui.length)
        if (size == 0):
            sys.stderr.write('GetArchiveInfo: file size = 0\n')
            return 0

        buff = array.array('B', text)

        index = 0
        while (index < size):
            cursor = buff[index] + (buff[index+1] * 256)

            if (cursor == 0):
                index += 2
                cursor = buff[index] + (buff[index+1] * 256)
                index += 2
                self.topics = '/' + text[index:index+cursor-1]
            elif (cursor == 1):
                index += 2
                cursor = buff[index] + (buff[index+1] * 256)
                index += 2
                self.index = '/' + text[index:index+cursor-1]
            elif (cursor == 2):
                index += 2
                cursor = buff[index] + (buff[index+1] * 256)
                index += 2
                self.home = '/' + text[index:index+cursor-1]
            elif (cursor == 3):
                index += 2
                cursor = buff[index] + (buff[index+1] * 256)
                index += 2
                self.title = text[index:index+cursor-1]
            elif (cursor == 4):
                index += 2
                cursor = buff[index] + (buff[index+1] * 256)
                index += 2
                self.lcid = buff[index] + (buff[index+1] * 256)
            elif (cursor == 6):
                index += 2
                cursor = buff[index] + (buff[index+1] * 256)
                index += 2
                tmp = text[index:index+cursor-1]
                if not self.topics:
                    tmp1 = '/' + tmp + '.hhc'
                    tmp2 = '/' + tmp + '.hhk'
                    res1, ui1 = chmlib.chm_resolve_object(self.file, tmp1)
                    res2, ui2 = chmlib.chm_resolve_object(self.file, tmp2)
                    if not self.topics and res1 == chmlib.CHM_RESOLVE_SUCCESS:
                        self.topics = '/' + tmp + '.hhc'
                    if not self.index and res2 == chmlib.CHM_RESOLVE_SUCCESS:
                        self.index = '/' + tmp + '.hhk'
            elif (cursor == 16):
                index += 2
                cursor = buff[index] + (buff[index+1] * 256)
                index += 2
                self.encoding = text[index:index+cursor-1]
            else:
                index += 2
                cursor = buff[index] + (buff[index+1] * 256)
                index += 2
            index += cursor

        self.GetWindowsInfo()

        if not self.lcid:
            self.lcid = extra.get_lcid(self.file)

        return 1
1164798
def get_urlpatterns(self):
        
        return [
            path(_('topics/'), self.latest_topics_feed(), name='latest_topics'),
            path(
                _('forum/<str:forum_slug>-<int:forum_pk>/topics/'),
                self.latest_topics_feed(),
                name='forum_latest_topics',
            ),
            path(
                _('forum/<str:forum_slug>-<int:forum_pk>/topics/all/'),
                self.latest_topics_feed(),
                {'descendants': True},
                name='forum_latest_topics_with_descendants',
            ),
        ]
1267594
def get_all_thread_ids(self):
        
        json = self._get_json(self._url.thread_list())
        return [thread['no'] for page in json for thread in page['threads']]
220076
def terms_iterator_for_elasticsearch(fo: IO, index_name: str):
    """"""

    species_list = config["bel_resources"].get("species_list", [])

    fo.seek(0)  # Seek back to beginning of file
    with gzip.open(fo, "rt") as f:
        for line in f:
            term = json.loads(line)
            # skip if not term record (e.g. is a metadata record)
            if "term" not in term:
                continue
            term = term["term"]

            # Filter species if enabled in config
            species_id = term.get("species_id", None)
            if species_list and species_id and species_id not in species_list:
                continue

            all_term_ids = set()
            for term_id in [term["id"]] + term.get("alt_ids", []):
                all_term_ids.add(term_id)
                all_term_ids.add(lowercase_term_id(term_id))

            term["alt_ids"] = copy.copy(list(all_term_ids))

            yield {
                "_op_type": "index",
                "_index": index_name,
                "_type": "term",
                "_id": term["id"],
                "_source": copy.deepcopy(term),
            }
1043782
def get_all_file_report_pages(self, query):
        
        responses = []
        next_page, response = self.get_hashes_from_search(self, query)
        responses.append(_return_response_and_status_code(response))
        while next_page:
            next_page, response = self.get_hashes_from_search(query, next_page)
            responses.append(_return_response_and_status_code(response))
        return dict(results=responses)
1335985
def index_resources(self):
        
        if not self.__index_resources:
            self.__index_resources = IndexResources(self.__connection)
        return self.__index_resources
281521
private static List<Integer> getUsedIndexes() {
        File folder = getMotdFileOutputStreamProvider().getMOTDFolder();
        List<Integer> res = new ArrayList<>();

        for (File file : folder.listFiles()) {
            if (file.isFile() && file.getName().matches(getRegexToMatchSerializedMOTs())) {
                res.add(getIndexFromFilename(file.getName()));
            }
        }

        return res;
    }
795798
def write_urls_index(app, exc):
    '''
    
    '''
    inventory = os.path.join(app.builder.outdir, 'objects.inv')
    objects = sphinx.ext.intersphinx.fetch_inventory(app, DOCS_URL, inventory)

    with open(os.path.join(app.builder.outdir, 'shorturls.json'), 'w') as f:
        json.dump(objects, f)
181015
def crawl_links(self, seed_url=None):
        
        if seed_url is not None:
            self.seed_url = seed_url

        if self.seed_url is None:
            sys.stderr.write('Crawling requires a seed URL.\n')
            return []

        prev_part_num = utils.get_num_part_files()
        crawled_links = set()
        uncrawled_links = OrderedSet()

        uncrawled_links.add(self.seed_url)
        try:
            while uncrawled_links:
                # Check limit on number of links and pages to crawl
                if self.limit_reached(len(crawled_links)):
                    break
                url = uncrawled_links.pop(last=False)

                # Remove protocol, fragments, etc. to get unique URLs
                unique_url = utils.remove_protocol(utils.clean_url(url))
                if unique_url not in crawled_links:
                    raw_resp = utils.get_raw_resp(url)
                    if raw_resp is None:
                        if not self.args['quiet']:
                            sys.stderr.write('Failed to parse {0}.\n'.format(url))
                        continue

                    resp = lh.fromstring(raw_resp)
                    if self.page_crawled(resp):
                        continue

                    crawled_links.add(unique_url)
                    new_links = self.get_new_links(url, resp)
                    uncrawled_links.update(new_links)
                    if not self.args['quiet']:
                        print('Crawled {0} (#{1}).'.format(url, len(crawled_links)))

                    # Write page response to PART.html file
                    utils.write_part_file(self.args, url, raw_resp, resp, len(crawled_links))
        except (KeyboardInterrupt, EOFError):
            pass

        curr_part_num = utils.get_num_part_files()
        return utils.get_part_filenames(curr_part_num, prev_part_num)
1007872
function getIndexDetails(client, namespace, done) {
  var tasks = {
    client: attach.bind(null, client),
    namespace: attach.bind(null, namespace),
    getIndexes: ['client', 'namespace', getIndexes],
    getIndexStats: ['client', 'namespace', getIndexStats],
    getIndexSizes: ['client', 'namespace', getIndexSizes],
    indexes: [
      'getIndexes',
      'getIndexStats',
      'getIndexSizes',
      combineStatsAndIndexes
    ]
  };

  async.auto(tasks, function(err, results) {
    if (err) {
      // report error
      return done(err);
    }
    // all info was collected in indexes
    return done(null, results.indexes);
  });
}
320756
def _index_fs(self):
        

        indexed_objects = self._return_deque()

        directory = self.job_args.get('directory')
        if directory:
            indexed_objects = self._return_deque(
                deque=indexed_objects,
                item=self._drectory_local_files(
                    directory=directory
                )
            )

        object_names = self.job_args.get('object')
        if object_names:
            indexed_objects = self._return_deque(
                deque=indexed_objects,
                item=self._named_local_files(
                    object_names=object_names
                )
            )

        return indexed_objects
1187634
def crawl(self, urls, name='crawl', api='analyze', **kwargs):
        
        # If multiple seed URLs are specified, join with whitespace.
        if isinstance(urls, list):
            urls = ' '.join(urls)
        url = self.endpoint('crawl')
        process_url = self.endpoint(api)
        params = {
            'token': self._token,
            'seeds': urls,
            'name': name,
            'apiUrl': process_url,
        }

        # Add any additional named parameters as accepted by Crawlbot
        params['maxToCrawl'] = 10
        params.update(kwargs)

        self._get(url, params=params)

        return Job(self._token, name, self._version)