If we're going to keep segmented reports, we need to know which archives are
for segments. This info is only in the numeric tables, so we must query them.
493647
private function findSegmentArchives($numericTables)
    {
        if (!is_null($this->segmentArchiveIds) || empty($numericTables)) {
            return;
        }

        foreach ($numericTables as $table) {
            $tableDate = ArchiveTableCreator::getDateFromTableName($table);

            $maxIdArchive = Db::fetchOne("SELECT MAX(idarchive) FROM $table");

            $sql = "SELECT idarchive FROM $table
                     WHERE name != 'done'
                       AND name LIKE 'done_%.%'
                       AND idarchive >= ?
                       AND idarchive < ?";

            if (is_null($this->segmentArchiveIds)) {
                $this->segmentArchiveIds = array();
            }

            $this->segmentArchiveIds[$tableDate] = array();
            foreach (Db::segmentedFetchAll($sql, 0, $maxIdArchive, self::$selectSegmentSize) as $row) {
                $this->segmentArchiveIds[$tableDate][] = $row['idarchive'];
            }
        }
    }
533952
public boolean
findRecord(Record r, int section) {
	return (sections[section] != null && sections[section].contains(r));
}
268142
private int estimateSegmentsBasedOnSize() {
        // the idea is that if values are huge, operations on them (and simply ser/deser)
        // could take long time, so we want more segment to minimize probablity that
        // two or more concurrent write ops will go to the same segment, and then all but one of
        // these threads will wait for long time.
        int segmentsForEntries = estimateSegmentsForEntries(entries());
        double averageValueSize = averageValueSize();
        return averageValueSize >= 1000000
                ? segmentsForEntries * 16
                : averageValueSize >= 100000
                ? segmentsForEntries * 8
                : averageValueSize >= 10000
                ? segmentsForEntries * 4
                : averageValueSize >= 1000
                ? segmentsForEntries * 2
                : segmentsForEntries;
    }
404420
private void listArchive(Archive archive,
                               RelativeDirectory subdirectory,
                               Set<JavaFileObject.Kind> fileKinds,
                               boolean recurse,
                               ListBuffer<JavaFileObject> resultList) {
        // Get the files directly in the subdir
        List<String> files = archive.getFiles(subdirectory);
        if (files != null) {
            for (; !files.isEmpty(); files = files.tail) {
                String file = files.head;
                if (isValidFile(file, fileKinds)) {
                    resultList.append(archive.getFileObject(subdirectory, file));
                }
            }
        }
        if (recurse) {
            for (RelativeDirectory s: archive.getSubdirectories()) {
                if (subdirectory.contains(s)) {
                    // Because the archive map is a flat list of directories,
                    // the enclosing loop will pick up all child subdirectories.
                    // Therefore, there is no need to recurse deeper.
                    listArchive(archive, s, fileKinds, false, resultList);
                }
            }
        }
    }
720535
private SegmentScoreEntry computeSegmentScore(String segment) {
		if(this.stopList.contains(segment) )
			return SegmentScoreEntry.SCORE_ZERO;
		CompostIndexEntry closestEntry = compostIndex.getEntry(segment);
		double indexSimilarity = 0.0;
		
		if(closestEntry == null) {
			if(this.opt.getSegmentSimilarityThreshold() == 1)
				// do not compare similarity of this segment to the index
				return SegmentScoreEntry.SCORE_ZERO;
					
			// Find an entry by similarity
			Iterator<CompostIndexEntry> it = compostIndex.closedEntryCandidateIterator(segment);
			int entryLength = segment.length();
			double dist = 0;
			CompostIndexEntry entry;
			while(it.hasNext()) {
				entry = it.next();
				dist = distance.computeNormalized(segment, entry.getText());
				if(Math.abs(entry.getText().length() - entryLength) <= 3 
						&& dist >= this.opt.getSegmentSimilarityThreshold()) {
					indexSimilarity = dist;
					closestEntry = entry;
				}
			}
			if(closestEntry == null) {
				// could not find any close entry in the compost index
				return SegmentScoreEntry.SCORE_ZERO;
			}
		} else {
			indexSimilarity = 1f;
		}
		int inCorpus = 0;
		int inDico = closestEntry.isInDico() || closestEntry.isInNeoClassicalPrefix() ? 1 : 0;

		// retrieves all sw terms that have the same lemma
		Collection<Term> corpusTerms = swtLemmaIndex.getTerms(segment);
		double wr = 0f;
		for(Iterator<Term> it = corpusTerms.iterator(); it.hasNext();)
			wr+=Math.pow(10, it.next().getSpecificity());
		
		double dataCorpus;
		if(closestEntry.isInCorpus() && !corpusTerms.isEmpty()) {
			dataCorpus = wr / getMaxSpec();
			inCorpus = 1;
		} else {
			dataCorpus = 0;
			inCorpus = closestEntry.isInNeoClassicalPrefix() ? 1 : 0;
		}
		double score = this.opt.getAlpha() * indexSimilarity + this.opt.getBeta() * inDico + this.opt.getGamma() * inCorpus + this.opt.getDelta() * dataCorpus;
		if(logger.isTraceEnabled()) {
			logger.trace("Score for {} is {} [alpha: {} beta: {} gamma: {} delta: {}]", 
					segment, 
					score,
					indexSimilarity,
					inDico,
					inCorpus,
					dataCorpus);
		}
		return new SegmentScoreEntry(segment, findSegmentLemma(segment), score, closestEntry);
	}
34653
private ArrayList<Segment10> tableSegments(TableEntry10 table)
  {
    ArrayList<Segment10> tableSegments = new ArrayList<>();
    
    for (Segment10 segment : _segments) {
      if (Arrays.equals(segment.key(), table.key())) {
        tableSegments.add(segment);
      }
    }
    
    Collections.sort(tableSegments,
                     (x,y)->Long.signum(y.sequence() - x.sequence()));
    
    return tableSegments;
  }
907675
protected Collection<JournalSegment<E>> loadSegments() {
    // Ensure log directories are created.
    directory.mkdirs();

    TreeMap<Long, JournalSegment<E>> segments = new TreeMap<>();

    // Iterate through all files in the log directory.
    for (File file : directory.listFiles(File::isFile)) {

      // If the file looks like a segment file, attempt to load the segment.
      if (JournalSegmentFile.isSegmentFile(name, file)) {
        JournalSegmentFile segmentFile = new JournalSegmentFile(file);
        ByteBuffer buffer = ByteBuffer.allocate(JournalSegmentDescriptor.BYTES);
        try (FileChannel channel = openChannel(file)) {
          channel.read(buffer);
          buffer.flip();
        } catch (IOException e) {
          throw new StorageException(e);
        }

        JournalSegmentDescriptor descriptor = new JournalSegmentDescriptor(buffer);

        // Load the segment.
        JournalSegment<E> segment = loadSegment(descriptor.id());

        // Add the segment to the segments list.
        log.debug("Found segment: {} ({})", segment.descriptor().id(), segmentFile.file().getName());
        segments.put(segment.index(), segment);
      }
    }

    // Verify that all the segments in the log align with one another.
    JournalSegment<E> previousSegment = null;
    boolean corrupted = false;
    Iterator<Map.Entry<Long, JournalSegment<E>>> iterator = segments.entrySet().iterator();
    while (iterator.hasNext()) {
      JournalSegment<E> segment = iterator.next().getValue();
      if (previousSegment != null && previousSegment.lastIndex() != segment.index() - 1) {
        log.warn("Journal is inconsistent. {} is not aligned with prior segment {}", segment.file().file(), previousSegment.file().file());
        corrupted = true;
      }
      if (corrupted) {
        segment.close();
        segment.delete();
        iterator.remove();
      }
      previousSegment = segment;
    }

    return segments.values();
  }
1254680
def is_eligible(self, segment):
        
        if not segment or not segment.sampled:
            return False

        return segment.get_total_subsegments_size() > self.streaming_threshold
1163993
function(segment) {
                return (this.start.eql(segment.start) && this.end.eql(segment.end)) ||
                    (this.start.eql(segment.end) && this.end.eql(segment.start));
              }
1210951
def segment_list
      segments = ['Segments']

      job_steps.each do |job_step|
        if job_step.crawl_segment.present?
          segment = job_step.crawl_segment
          segments.push(segment.segment_desc)
        end
      end

      segments.push('')
    end
1128209
protected void submitSegmentIndexBuffer() {
        if(_sibEnabled && !_sib.isDirty()) {
            _sib.setSegmentId(_segment.getSegmentId());
            _sib.setSegmentLastForcedTime(_segment.getLastForcedTime());
            _segmentManager.submit(_sib);
        } else {
            _segmentManager.remove(_sib);
        }
    }
850746
public function requireDefaultRecords()
    {
        $class = get_class($this->owner);
        
        $records = DataList::create($class)->where('URLSegment IS NULL');
        
        if ($records->exists()) {
            
            foreach ($records as $record) {
                $record->write();
            }
            
            DB::alteration_message(sprintf('Updated %s records without URL segments', $class), 'changed');
        }
    }
329830
private JournalSegment<E> loadSegment(long segmentId) {
    File segmentFile = JournalSegmentFile.createSegmentFile(name, directory, segmentId);
    ByteBuffer buffer = ByteBuffer.allocate(JournalSegmentDescriptor.BYTES);
    try (FileChannel channel = openChannel(segmentFile)) {
      channel.read(buffer);
      buffer.flip();
      JournalSegmentDescriptor descriptor = new JournalSegmentDescriptor(buffer);
      JournalSegment<E> segment = newSegment(new JournalSegmentFile(segmentFile), descriptor);
      log.debug("Loaded disk segment: {} ({})", descriptor.id(), segmentFile.getName());
      return segment;
    } catch (IOException e) {
      throw new StorageException(e);
    }
  }
396945
public boolean
findRecord(Record r) {
	for (int i = Section.ANSWER; i <= Section.ADDITIONAL; i++)
		if (sections[i] != null && sections[i].contains(r))
			return true;
	return false;
}
394937
private void ensureCapacity(int segmentIndex) {
        while(segmentIndex >= segments.length) {
            segments = Arrays.copyOf(segments, segments.length * 3 / 2);
        }

        long numSegmentsPopulated = length >> log2OfSegmentSize;

        for(long i=numSegmentsPopulated; i <= segmentIndex; i++) {
            segments[(int)i] = memoryPool.getSegment();
            length += 1 << log2OfSegmentSize;
        }
    }
327933
def _allocate_specified_segment(self, session, seg_id, source):
        
        try:
            with session.begin(subtransactions=True):
                alloc = (session.query(self.model).filter_by(
                    segmentation_id=seg_id).first())
                if alloc:
                    if alloc.allocated:
                        # Segment already allocated
                        return
                    else:
                        # Segment not allocated
                        count = (session.query(self.model).
                                 filter_by(allocated=False,
                                           segmentation_id=seg_id).update(
                                               {"allocated": True}))
                        if count:
                            return alloc

                # Segment to create or already allocated
                alloc = self.model(segmentation_id=seg_id,
                                   allocated=True, source=source)
                session.add(alloc)

        except db_exc.DBDuplicateEntry:
            # Segment already allocated (insert failure)
            alloc = None

        return alloc
398942
@Override
    public Collection<V> values() {
        try {
            lockAllSegments();
            Collection<V> collection = new ArrayList<>();
            for (LinkedHashMapSegment<K, V> seg : segments) {
                collection.addAll(seg.values());
            }
            return collection;
        } finally {
            unlockAllSegments();
        }
    }