These are the file sets that belong to this work, but not necessarily
 in order.
 Arbitrarily maxed at 10 thousand; had to specify rows due to solr's default of 10
335663
def file_set_ids
        @file_set_ids ||= begin
                            ActiveFedora::SolrService.query("{!field f=has_model_ssim}FileSet",
                                                            rows: 10_000,
                                                            fl: ActiveFedora.id_field,
                                                            fq: "{!join from=ordered_targets_ssim to=id}id:\"#{id}/list_source\"")
                                                     .flat_map { |x| x.fetch(ActiveFedora.id_field, []) }
                          end
      end
1032754
def _stream_search(self, query):
        """"""
        for doc in self.solr.search(query, rows=100000000):
            if self.unique_key != "_id":
                doc["_id"] = doc.pop(self.unique_key)
            yield doc
734150
@Override
  protected void bulkLoad(List<RdKNNEntry> entries) {
    super.bulkLoad(entries);

    // adjust all knn distances
    ArrayModifiableDBIDs ids = DBIDUtil.newArray(entries.size());
    for(RdKNNEntry entry : entries) {
      DBID id = ((RdKNNLeafEntry) entry).getDBID();
      ids.add(id);
    }
    ids.sort();
    List<? extends KNNList> knnLists = knnQuery.getKNNForBulkDBIDs(ids, settings.k_max);
    adjustKNNDistance(getRootEntry(), ids, knnLists);

    // test
    doExtraIntegrityChecks();
  }
484479
public Long getFileSizeAvg() {
        Long fileSizeAvg = 0L;
        if (items.size() != 0) {
            for (TableStat item : items) {
                if (item.getEndTime().equals(item.getStartTime())) {
                    fileSizeAvg += item.getFileSize();
                } else {
                    fileSizeAvg += item.getFileSize() * 1000
                                   / (item.getEndTime().getTime() - item.getStartTime().getTime());
                }
            }
            fileSizeAvg = fileSizeAvg / items.size();
        }
        return fileSizeAvg;
    }
708578
public function exportIndex(
        Manager $manager,
        $filename,
        $types,
        $chunkSize,
        OutputInterface $output,
        $maxLinesInFile = 300000
    ) {

        $search = new Search();
        $search->addQuery(new MatchAllQuery());
        $search->setSize($chunkSize);
        $search->addSort(new FieldSort('_doc'));

        $queryParameters = [
                '_source' => true,
                'scroll' => '10m',
            ];

        $searchResults = $manager->search($types, $search->toArray(), $queryParameters);

        $results = new RawIterator(
            $searchResults,
            $manager,
            [
                'duration' => $queryParameters['scroll'],
                '_scroll_id' => $searchResults['_scroll_id'],
            ]
        );

        $progress = new ProgressBar($output, $results->count());
        $progress->setRedrawFrequency(100);
        $progress->start();

        $counter = $fileCounter = 0;
        $count = $this->getFileCount($results->count(), $maxLinesInFile, $fileCounter);

        $date = date(\DateTime::ISO8601);
        $metadata = [
            'count' => $count,
            'date' => $date,
        ];

        $filename = str_replace('.json', '', $filename);
        $writer = $this->getWriter($this->getFilePath($filename.'.json'), $metadata);

        $file = [];
        foreach ($results as $data) {
            if ($counter >= $maxLinesInFile) {
                $writer->finalize();
                $writer = null;
                $fileCounter++;
                $count = $this->getFileCount($results->count(), $maxLinesInFile, $fileCounter);
                $metadata = [
                    'count' => $count,
                    'date' => $date,
                ];
                $writer = $this->getWriter($this->getFilePath($filename."_".$fileCounter.".json"), $metadata);
                $counter = 0;
            }

            $doc = array_intersect_key($data, array_flip(['_id', '_type', '_source']));
            $writer->push($doc);
            $file[] = $doc;
            $progress->advance();
            $counter++;
        }

        $writer->finalize();
        $progress->finish();
        $output->writeln('');
    }
321692
def __get_managed_files_rpm(self):
        '''
        
        '''
        dirs = set()
        links = set()
        files = set()

        for line in salt.utils.stringutils.to_str(self._syscall("rpm", None, None, '-qlav')[0]).split(os.linesep):
            line = line.strip()
            if not line:
                continue
            line = line.replace("\t", " ").split(" ")
            if line[0][0] == "d":
                dirs.add(line[-1])
            elif line[0][0] == "l":
                links.add(line[-1])
            elif line[0][0] == "-":
                files.add(line[-1])

        return sorted(files), sorted(dirs), sorted(links)
1132961
def sequence_rendering
      renderings = []
      if solr_document.rendering_ids.present?
        solr_document.rendering_ids.each do |file_set_id|
          renderings << manifest_helper.build_rendering(file_set_id)
        end
      end
      renderings.flatten
    end
647928
def items(self, match="*", count=1000):
        
        for field, value in self._client.hscan_iter(
          self.key_prefix, match=match, count=count):
            yield self._decode(field), self._loads(value)
681457
Index getIndexForColumns(OrderedIntHashSet set) {

        int   maxMatchCount = 0;
        Index selected      = null;

        if (set.isEmpty()) {
            return null;
        }

        for (int i = 0, count = indexList.length; i < count; i++) {
            Index currentindex = getIndex(i);
            int[] indexcols    = currentindex.getColumns();
            int   matchCount   = set.getOrderedMatchCount(indexcols);

            if (matchCount == 0) {
                continue;
            }

            if (matchCount == indexcols.length) {
                return currentindex;
            }

            if (matchCount > maxMatchCount) {
                maxMatchCount = matchCount;
                selected      = currentindex;
            }
        }

        if (selected != null) {
            return selected;
        }

        switch (tableType) {

            case TableBase.SYSTEM_SUBQUERY :
            case TableBase.SYSTEM_TABLE :
            case TableBase.VIEW_TABLE :
            case TableBase.TEMP_TABLE : {
                selected = createIndexForColumns(set.toArray());
            }
        }

        return selected;
    }
1005054
public SolrDocumentList runQuery(String q, List<String> fieldList,
			int results) throws SolrServerException {
		SolrQuery query = new SolrQuery();
		query.setQuery(escapeQuery(q));
		query.setRows(results);
		query.setFields(fieldList.toArray(new String[1]));
		QueryResponse rsp = server.query(query, METHOD.POST);
		return rsp.getResults();
	}
1130779
def exec(self, **kwargs):
        

        params = self.solr_params()
        logging.info("PARAMS="+str(params))
        results = self.solr.search(**params)
        n_docs = len(results.docs)
        logging.info("Docs found: {}".format(results.hits))

        if self.iterate:
            docs = results.docs
            start = n_docs
            while n_docs >= self.rows:
                logging.info("Iterating; start={}".format(start))
                next_results = self.solr.search(start=start, **params)
                next_docs = next_results.docs
                n_docs = len(next_docs)
                docs += next_docs
                start += self.rows
            results.docs = docs

        fcs = results.facets

        payload = {
            'facet_counts': translate_facet_field(fcs, self.invert_subject_object),
            'pagination': {},
            'numFound': results.hits
        }

        include_raw=self.include_raw
        if include_raw:
            # note: this is not JSON serializable, do not send via REST
            payload['raw'] = results

        # TODO - check if truncated

        logging.info("COMPACT={} INV={}".format(self.use_compact_associations, self.invert_subject_object))
        if self.use_compact_associations:
            payload['compact_associations'] = self.translate_docs_compact(results.docs, field_mapping=self.field_mapping,
                                                                     slim=self.slim, invert_subject_object=self.invert_subject_object,
                                                                     map_identifiers=self.map_identifiers, **kwargs)
        else:
            payload['associations'] = self.translate_docs(results.docs, field_mapping=self.field_mapping, map_identifiers=self.map_identifiers, **kwargs)

        if 'facet_pivot' in fcs:
            payload['facet_pivot'] = fcs['facet_pivot']
        if 'facets' in results.raw_response:
            payload['facets'] = results.raw_response['facets']

        # For solr, we implement this by finding all facets
        # TODO: no need to do 2nd query, see https://wiki.apache.org/solr/SimpleFacetParameters#Parameters
        fetch_objects=self.fetch_objects
        if fetch_objects:
            core_object_field = M.OBJECT
            if self.slim is not None and len(self.slim)>0:
                core_object_field = M.OBJECT_CLOSURE
            object_field = map_field(core_object_field, self.field_mapping)
            if self.invert_subject_object:
                object_field = map_field(M.SUBJECT, self.field_mapping)
            oq_params = params.copy()
            oq_params['fl'] = []
            oq_params['facet.field'] = [object_field]
            oq_params['facet.limit'] = -1
            oq_params['rows'] = 0
            oq_params['facet.mincount'] = 1
            oq_results = self.solr.search(**oq_params)
            ff = oq_results.facets['facet_fields']
            ofl = ff.get(object_field)
            # solr returns facets counts as list, every 2nd element is number, we don't need the numbers here
            payload['objects'] = ofl[0::2]

        fetch_subjects=self.fetch_subjects
        if fetch_subjects:
            core_subject_field = M.SUBJECT
            if self.slim is not None and len(self.slim)>0:
                core_subject_field = M.SUBJECT_CLOSURE
            subject_field = map_field(core_subject_field, self.field_mapping)
            if self.invert_subject_object:
                subject_field = map_field(M.SUBJECT, self.field_mapping)
            oq_params = params.copy()
            oq_params['fl'] = []
            oq_params['facet.field'] = [subject_field]
            oq_params['facet.limit'] = self.max_rows
            oq_params['rows'] = 0
            oq_params['facet.mincount'] = 1
            oq_results = self.solr.search(**oq_params)
            ff = oq_results.facets['facet_fields']
            ofl = ff.get(subject_field)
            # solr returns facets counts as list, every 2nd element is number, we don't need the numbers here
            payload['subjects'] = ofl[0::2]
            if len(payload['subjects']) == self.max_rows:
                payload['is_truncated'] = True

        if self.slim is not None and len(self.slim)>0:
            if 'objects' in payload:
                payload['objects'] = [x for x in payload['objects'] if x in self.slim]
            if 'associations' in payload:
                for a in payload['associations']:
                    a['slim'] = [x for x in a['object_closure'] if x in self.slim]
                    del a['object_closure']

        return payload
935940
def items(self, limit=10000):
        
        _limit = 500
        if 'ids' in self.kwargs:
            col = self.kwargs.get('query', {}).get('collection', {}).get('eq', None)
            if col is None:
                raise SatSearchError('Collection required when searching by id')
            return self.items_by_id(self.kwargs['ids'], col)

        items = []
        found = self.found()
        if found > limit:
            logger.warning('There are more items found (%s) than the limit (%s) provided.' % (found, limit))
        maxitems = min(found, limit)
        kwargs = {
            'page': 1,
            'limit': min(_limit, maxitems)
        }
        kwargs.update(self.kwargs)
        while len(items) < maxitems:
            items += [Item(i) for i in self.query(**kwargs)['features']]
            kwargs['page'] += 1

        # retrieve collections
        collections = []
        for c in set([item.properties['collection'] for item in items if 'collection' in item.properties]):
            collections.append(self.collection(c))
            #del collections[c]['links']

        # merge collections into items
        #_items = []
        #for item in items:
        #    import pdb; pdb.set_trace()
        #    if 'collection' in item['properties']:
        #        item = dict_merge(item, collections[item['properties']['collection']])
        #    _items.append(Item(item))

        return Items(items, collections=collections, search=self.kwargs)
575594
def max(self):
        
        if len(self._data) == 0:
            return 600
        return next(iter(reversed(sorted(self._data.keys()))))
1328274
private static List<Tuple2<Book,ReadableInstant>> getSitemapBooks(
		final ServletContext servletContext,
		HttpServletRequest req,
		HttpServletResponse resp
	) throws ServletException, IOException {
		SemanticCMS semanticCMS = SemanticCMS.getInstance(servletContext);
		HtmlRenderer htmlRenderer = HtmlRenderer.getInstance(servletContext);
		final SortedSet<View> views = htmlRenderer.getViews();
		List<Book> books;
		{
			// Filter published and accessible only
			Collection<Book> values = semanticCMS.getPublishedBooks().values();
			books = new ArrayList<>(values.size());
			for(Book book : values) if(book.isAccessible()) books.add(book);
		}
		int numBooks = books.size();
		if(
			numBooks > 1
			&& CountConcurrencyFilter.useConcurrentSubrequests(req)
		) {
			// Concurrent implementation
			final HttpServletRequest threadSafeReq = new UnmodifiableCopyHttpServletRequest(req);
			final HttpServletResponse threadSafeResp = new UnmodifiableCopyHttpServletResponse(resp);
			final TempFileContext tempFileContext = ServletTempFileContext.getTempFileContext(req);
			List<Book> booksWithSiteMapUrl;
			{
				List<Callable<Boolean>> tasks = new ArrayList<>(numBooks);
				{
					for(final Book book : books) {
						tasks.add(
							new Callable<Boolean>() {
								@Override
								public Boolean call() throws ServletException, IOException {
									HttpServletRequest subrequest = new HttpServletSubRequest(threadSafeReq);
									HttpServletResponse subresponse = new HttpServletSubResponse(threadSafeResp, tempFileContext);
									if(logger.isLoggable(Level.FINE)) logger.log(
										Level.FINE,
										"called, subrequest={0}, book={1}",
										new Object[] {
											subrequest,
											book
										}
									);
									return hasSiteMapUrl(
										servletContext,
										subrequest,
										subresponse,
										views,
										book,
										book.getContentRoot()
										//new HashSet<PageRef>()
									);
								}
							}
						);
					}
				}
				List<Boolean> results;
				try {
					results = semanticCMS.getExecutors().getPerProcessor().callAll(tasks);
				} catch(InterruptedException e) {
					throw new ServletException(e);
				} catch(ExecutionException e) {
					Throwable cause = e.getCause();
					if(cause instanceof RuntimeException) throw (RuntimeException)cause;
					if(cause instanceof ServletException) throw (ServletException)cause;
					if(cause instanceof IOException) throw (IOException)cause;
					throw new ServletException(cause);
				}
				// Now find the last modified with concurrency
				booksWithSiteMapUrl = new ArrayList<>(numBooks);
				{
					int i = 0;
					for(Book book : books) {
						if(results.get(i++)) booksWithSiteMapUrl.add(book);
					}
					assert i == numBooks;
				}
			}
			int booksWithSiteMapUrlSize = booksWithSiteMapUrl.size();
			if(booksWithSiteMapUrlSize > 0) {
				if(booksWithSiteMapUrlSize > 1) {
					// Concurrent implementation
					List<Callable<ReadableInstant>> lastModifiedTasks = new ArrayList<>(booksWithSiteMapUrlSize);
					{
						for(final Book book : booksWithSiteMapUrl) {
							lastModifiedTasks.add(
								new Callable<ReadableInstant>() {
									@Override
									public ReadableInstant call() throws ServletException, IOException {
										HttpServletRequest subrequest = new HttpServletSubRequest(threadSafeReq);
										HttpServletResponse subresponse = new HttpServletSubResponse(threadSafeResp, tempFileContext);
										if(logger.isLoggable(Level.FINE)) logger.log(
											Level.FINE,
											"called, subrequest={0}, book={1}",
											new Object[] {
												subrequest,
												book
											}
										);
										return SiteMapServlet.getLastModified(
											servletContext,
											subrequest,
											subresponse,
											views,
											book
										);
									}
								}
							);
						}
					}
					List<ReadableInstant> lastModifieds;
					try {
						lastModifieds = semanticCMS.getExecutors().getPerProcessor().callAll(lastModifiedTasks);
					} catch(InterruptedException e) {
						throw new ServletException(e);
					} catch(ExecutionException e) {
						Throwable cause = e.getCause();
						if(cause instanceof RuntimeException) throw (RuntimeException)cause;
						if(cause instanceof ServletException) throw (ServletException)cause;
						if(cause instanceof IOException) throw (IOException)cause;
						throw new ServletException(cause);
					}
					List<Tuple2<Book,ReadableInstant>> sitemapBooks = new ArrayList<>(booksWithSiteMapUrlSize);
					for(int i = 0; i < booksWithSiteMapUrlSize; i++) {
						sitemapBooks.add(
							new Tuple2<>(
								booksWithSiteMapUrl.get(i),
								lastModifieds.get(i)
							)
						);
					}
					return sitemapBooks;
				} else {
					// Single implementation
					Book book = booksWithSiteMapUrl.get(0);
					return Collections.singletonList(
						new Tuple2<>(
							book,
							SiteMapServlet.getLastModified(
								servletContext,
								req,
								resp,
								views,
								book
							)
						)
					);
				}
			} else {
				return Collections.emptyList();
			}
		} else {
			// Sequential implementation
			List<Tuple2<Book,ReadableInstant>> sitemapBooks = new ArrayList<>(numBooks);
			for(Book book : books) {
				if(
					hasSiteMapUrl(
						servletContext,
						req,
						resp,
						views,
						book,
						book.getContentRoot()
					)
				) {
					sitemapBooks.add(
						new Tuple2<>(
							book,
							SiteMapServlet.getLastModified(
								servletContext,
								req,
								resp,
								views,
								book
							)
						)
					);
				}
			}
			return sitemapBooks;
		}
	}
910331
def to_ary
      lines.take(1000).map { |l| { :number => l.filtered_number, :file => l.filtered_file, :method => l.filtered_method, :source => l.source } }
    end
78754
def build_dataset(filename, sequence_length, alphabet_size, max_lines=-1):
    """"""
    with open(filename) as f:
        content = f.readlines()
    content = content[:max_lines]
    content = [line for line in content if len(line) > 2]   # Remove blank lines
    seqs = np.zeros((sequence_length, len(content), alphabet_size))
    for ix, line in enumerate(content):
        padded_line = (line + " " * sequence_length)[:sequence_length]
        seqs[:, ix, :] = string_to_one_hot(padded_line, alphabet_size)
    return seqs
561873
@Override
	public HashSet<Long> loadIncludes(FhirContext theContext, EntityManager theEntityManager, Collection<Long> theMatches, Set<Include> theRevIncludes,
												 boolean theReverseMode, DateRangeParam theLastUpdated, String theSearchIdOrDescription) {
		if (theMatches.size() == 0) {
			return new HashSet<>();
		}
		if (theRevIncludes == null || theRevIncludes.isEmpty()) {
			return new HashSet<>();
		}
		String searchFieldName = theReverseMode ? "myTargetResourcePid" : "mySourceResourcePid";

		Collection<Long> nextRoundMatches = theMatches;
		HashSet<Long> allAdded = new HashSet<>();
		HashSet<Long> original = new HashSet<>(theMatches);
		ArrayList<Include> includes = new ArrayList<>(theRevIncludes);

		int roundCounts = 0;
		StopWatch w = new StopWatch();

		boolean addedSomeThisRound;
		do {
			roundCounts++;

			HashSet<Long> pidsToInclude = new HashSet<>();

			for (Iterator<Include> iter = includes.iterator(); iter.hasNext(); ) {
				Include nextInclude = iter.next();
				if (nextInclude.isRecurse() == false) {
					iter.remove();
				}

				boolean matchAll = "*".equals(nextInclude.getValue());
				if (matchAll) {
					String sql;
					sql = "SELECT r FROM ResourceLink r WHERE r." + searchFieldName + " IN (:target_pids) ";
					List<Collection<Long>> partitions = partition(nextRoundMatches, MAXIMUM_PAGE_SIZE);
					for (Collection<Long> nextPartition : partitions) {
						TypedQuery<ResourceLink> q = theEntityManager.createQuery(sql, ResourceLink.class);
						q.setParameter("target_pids", nextPartition);
						List<ResourceLink> results = q.getResultList();
						for (ResourceLink resourceLink : results) {
							if (theReverseMode) {
								pidsToInclude.add(resourceLink.getSourceResourcePid());
							} else {
								pidsToInclude.add(resourceLink.getTargetResourcePid());
							}
						}
					}
				} else {

					List<String> paths;
					RuntimeSearchParam param;
					String resType = nextInclude.getParamType();
					if (isBlank(resType)) {
						continue;
					}
					RuntimeResourceDefinition def = theContext.getResourceDefinition(resType);
					if (def == null) {
						ourLog.warn("Unknown resource type in include/revinclude=" + nextInclude.getValue());
						continue;
					}

					String paramName = nextInclude.getParamName();
					if (isNotBlank(paramName)) {
						param = mySearchParamRegistry.getSearchParamByName(def, paramName);
					} else {
						param = null;
					}
					if (param == null) {
						ourLog.warn("Unknown param name in include/revinclude=" + nextInclude.getValue());
						continue;
					}

					paths = param.getPathsSplit();

					String targetResourceType = defaultString(nextInclude.getParamTargetType(), null);
					for (String nextPath : paths) {
						String sql;

						boolean haveTargetTypesDefinedByParam = param.getTargets() != null && param.getTargets().isEmpty() == false;
						if (targetResourceType != null) {
							sql = "SELECT r FROM ResourceLink r WHERE r.mySourcePath = :src_path AND r." + searchFieldName + " IN (:target_pids) AND r.myTargetResourceType = :target_resource_type";
						} else if (haveTargetTypesDefinedByParam) {
							sql = "SELECT r FROM ResourceLink r WHERE r.mySourcePath = :src_path AND r." + searchFieldName + " IN (:target_pids) AND r.myTargetResourceType in (:target_resource_types)";
						} else {
							sql = "SELECT r FROM ResourceLink r WHERE r.mySourcePath = :src_path AND r." + searchFieldName + " IN (:target_pids)";
						}

						List<Collection<Long>> partitions = partition(nextRoundMatches, MAXIMUM_PAGE_SIZE);
						for (Collection<Long> nextPartition : partitions) {
							TypedQuery<ResourceLink> q = theEntityManager.createQuery(sql, ResourceLink.class);
							q.setParameter("src_path", nextPath);
							q.setParameter("target_pids", nextPartition);
							if (targetResourceType != null) {
								q.setParameter("target_resource_type", targetResourceType);
							} else if (haveTargetTypesDefinedByParam) {
								q.setParameter("target_resource_types", param.getTargets());
							}
							List<ResourceLink> results = q.getResultList();
							for (ResourceLink resourceLink : results) {
								if (theReverseMode) {
									Long pid = resourceLink.getSourceResourcePid();
									if (pid != null) {
										pidsToInclude.add(pid);
									}
								} else {
									Long pid = resourceLink.getTargetResourcePid();
									if (pid != null) {
										pidsToInclude.add(pid);
									}
								}
							}
						}
					}
				}
			}

			if (theReverseMode) {
				if (theLastUpdated != null && (theLastUpdated.getLowerBoundAsInstant() != null || theLastUpdated.getUpperBoundAsInstant() != null)) {
					pidsToInclude = new HashSet<>(filterResourceIdsByLastUpdated(theEntityManager, theLastUpdated, pidsToInclude));
				}
			}
			for (Long next : pidsToInclude) {
				if (original.contains(next) == false && allAdded.contains(next) == false) {
					theMatches.add(next);
				}
			}

			addedSomeThisRound = allAdded.addAll(pidsToInclude);
			nextRoundMatches = pidsToInclude;
		} while (includes.size() > 0 && nextRoundMatches.size() > 0 && addedSomeThisRound);

		ourLog.info("Loaded {} {} in {} rounds and {} ms for search {}", allAdded.size(), theReverseMode ? "_revincludes" : "_includes", roundCounts, w.getMillisAndRestart(), theSearchIdOrDescription);

		return allAdded;
	}