Gets the number of partitions to be used for an initial hash-table.
192856
private int getPartitioningFanOutNoEstimates() {
		return Math.max(11, findSmallerPrime((int) Math.min(buildRowCount * avgRecordLen / (10 * segmentSize),
				MAX_NUM_PARTITIONS)));
	}
276016
private List<Short> selectInitializedPartitions(int clusterPartitions, List<Short> partitions) {
    List<Short> initializedPartitions = new ArrayList<>();
    SessionState state = sessionState();

    for (short partition : partitions) {
      PartitionState ps = state.get(partition);
      if (ps != null) {
        if (MathUtils.lessThanUnsigned(ps.getStartSeqno(), ps.getEndSeqno())) {
          initializedPartitions.add(partition);
        } else {
          LOGGER.debug("Skipping partition {}, because startSeqno({}) >= endSeqno({})",
              partition, ps.getStartSeqno(), ps.getEndSeqno());
        }
      } else {
        LOGGER.debug("Skipping partition {}, because its state is null", partition);
      }
    }

    if (initializedPartitions.size() > clusterPartitions) {
      throw new IllegalStateException("Session State has " + initializedPartitions
          + " partitions while the cluster has " + clusterPartitions + "!");
    }
    return initializedPartitions;
  }
87778
@Override
    public Table<AllocationStatus, Integer, Long> getAllocationStatistics() {
        Table<AllocationStatus, Integer, Long> table = HashBasedTable.create();
        table.put(AllocationStatus.HOST, 0, zeroUseCounter.get());
        for (Integer deviceId : configuration.getAvailableDevices()) {
            table.put(AllocationStatus.DEVICE, deviceId, getAllocatedDeviceMemory(deviceId));
        }
        return table;
    }
1350524
def paused_partitions(self):
        """"""
        return set(partition for partition in self.assignment
                   if self.is_paused(partition))
398363
@Override
    public long getAllocatedHostObjects() {
        AtomicLong counter = new AtomicLong(0);
        for (Long threadId : zeroAllocations.keySet()) {
            counter.addAndGet(zeroAllocations.get(threadId).size());
        }
        return counter.get();
    }
861816
private static List<Integer> repartitionStreamlet(int incomingInteger, int numPartitions) {
    List<Integer> partitions;

    if (incomingInteger >= 0 && incomingInteger < 25) {
      partitions = Arrays.asList(0, 1);
    } else if (incomingInteger > 26 && incomingInteger < 50) {
      partitions = Arrays.asList(2, 3);
    } else if (incomingInteger > 50 && incomingInteger < 75) {
      partitions = Arrays.asList(4, 5);
    } else if (incomingInteger > 76 && incomingInteger <= 100) {
      partitions = Arrays.asList(6, 7);
    } else {
      partitions = Arrays.asList(ThreadLocalRandom.current().nextInt(0, 8));
    }

    String logMessage = String.format("Sending value %d to partitions: %s",
        incomingInteger,
        StreamletUtils.intListAsString(partitions));

    LOG.info(logMessage);

    return partitions;
  }
526599
def partitions(self):
        
        if not self._partitions:
            self._partitions = [Partition(d, self)
                                for d in self.device.children]
        return self._partitions
566911
public int getCrossZonePartitionStoreMoves() {
        int xzonePartitionStoreMoves = 0;
        for (RebalanceTaskInfo info : batchPlan) {
            Node donorNode = finalCluster.getNodeById(info.getDonorId());
            Node stealerNode = finalCluster.getNodeById(info.getStealerId());

            if(donorNode.getZoneId() != stealerNode.getZoneId()) {
                xzonePartitionStoreMoves += info.getPartitionStoreMoves();
            }
        }

        return xzonePartitionStoreMoves;
    }
410250
private static Set<Integer> getPartitionIdsForHosts(AbstractTopology topology, Set<Integer> hostIds) {
        Set<Integer> partitionIds = new HashSet<>();
        for (int hostId : hostIds) {
            partitionIds.addAll(topology.getPartitionIdList(hostId));
        }
        return partitionIds;
    }
349229
def _FindPartition(self, key):
        
        hash_value = self.hash_generator.ComputeHash(key)
        return self._LowerBoundSearch(self.partitions, hash_value)
343868
int getNumOccupiedMemorySegments() {
		// either the number of memory segments, or one for spilling
		final int numPartitionBuffers = this.partitionBuffers != null ?
				this.partitionBuffers.length
				: this.buildSideWriteBuffer.getNumOccupiedMemorySegments();
		return numPartitionBuffers + bucketArea.buckets.length + bucketArea.numOverflowSegments;
	}
535013
private Completable initFromBeginningToNow() {
    return initWithCallback(new Action1<PartitionAndSeqno>() {
      @Override
      public void call(PartitionAndSeqno partitionAndSeqno) {
        short partition = partitionAndSeqno.partition();
        long seqno = partitionAndSeqno.seqno();
        PartitionState partitionState = sessionState().get(partition);
        partitionState.setEndSeqno(seqno);
        sessionState().set(partition, partitionState);
      }
    });
  }
752244
def partitionBy(self, numPartitions, partitionFunc=portable_hash):
        
        return self.transform(lambda rdd: rdd.partitionBy(numPartitions, partitionFunc))
39909
protected byte[][] determineNewLargestElements(int numberOfPartitions) throws IOException {
        int elementsPerPart = determineElementsPerPart(numberOfPartitions);
        byte[][] keysToInsert = new byte[numberOfPartitions][];
        long offset;
        for (int i = 0; i < numberOfPartitions; i++) {
            if (i == numberOfPartitions - 1) {
                // Handling of last partition
                offset = sourceFile.getFilledUpFromContentStart() - sourceFile.getElementSize();
            } else {
                // Handling of all other partitions
                offset = ((i + 1) * elementsPerPart - 1) * sourceFile.getElementSize();
            }

            ByteBuffer keyBuffer = ByteBuffer.allocate(gp.getKeySize());
            sourceFile.read(offset, keyBuffer);
            keyBuffer.position(0);

            keysToInsert[i] = keyBuffer.array();
        }
        return keysToInsert;
    }
511894
public java.util.List<String> getPartitions() {
        if (partitions == null) {
            partitions = new com.amazonaws.internal.SdkInternalList<String>();
        }
        return partitions;
    }
716936
private boolean resizeHashTable() throws IOException {
		final int newNumBuckets = 2*this.numBuckets;
		final int bucketsPerSegment = this.bucketsPerSegmentMask + 1;
		final int newNumSegments = (newNumBuckets + (bucketsPerSegment-1)) / bucketsPerSegment;
		final int additionalSegments = newNumSegments-this.buckets.length;
		final int numPartitions = this.partitions.size();
		
		if (this.availableMemory.size() < additionalSegments) {
			for (int i = 0; i < numPartitions; i++) {
				compactPartition(i);
				if(this.availableMemory.size() >= additionalSegments) {
					break;
				}
			}
		}
		
		if (this.availableMemory.size() < additionalSegments || this.closed) {
			return false;
		}
		else {
			this.isResizing = true;
			// allocate new buckets
			final int startOffset = (this.numBuckets * HASH_BUCKET_SIZE) % this.segmentSize;
			final int oldNumBuckets = this.numBuckets;
			final int oldNumSegments = this.buckets.length;
			MemorySegment[] mergedBuckets = new MemorySegment[newNumSegments];
			System.arraycopy(this.buckets, 0, mergedBuckets, 0, this.buckets.length);
			this.buckets = mergedBuckets;
			this.numBuckets = newNumBuckets;
			// initialize all new buckets
			boolean oldSegment = (startOffset != 0);
			final int startSegment = oldSegment ? (oldNumSegments-1) : oldNumSegments;
			for (int i = startSegment, bucket = oldNumBuckets; i < newNumSegments && bucket < this.numBuckets; i++) {
				MemorySegment seg;
				int bucketOffset;
				if(oldSegment) { // the first couple of new buckets may be located on an old segment
					seg = this.buckets[i];
					for (int k = (oldNumBuckets % bucketsPerSegment) ; k < bucketsPerSegment && bucket < this.numBuckets; k++, bucket++) {
						bucketOffset = k * HASH_BUCKET_SIZE;	
						// initialize the header fields
						seg.put(bucketOffset + HEADER_PARTITION_OFFSET, assignPartition(bucket, (byte)numPartitions));
						seg.putInt(bucketOffset + HEADER_COUNT_OFFSET, 0);
						seg.putLong(bucketOffset + HEADER_FORWARD_OFFSET, BUCKET_FORWARD_POINTER_NOT_SET);
					}
				} else {
					seg = getNextBuffer();
					// go over all buckets in the segment
					for (int k = 0; k < bucketsPerSegment && bucket < this.numBuckets; k++, bucket++) {
						bucketOffset = k * HASH_BUCKET_SIZE;
						// initialize the header fields
						seg.put(bucketOffset + HEADER_PARTITION_OFFSET, assignPartition(bucket, (byte)numPartitions));
						seg.putInt(bucketOffset + HEADER_COUNT_OFFSET, 0);
						seg.putLong(bucketOffset + HEADER_FORWARD_OFFSET, BUCKET_FORWARD_POINTER_NOT_SET);
					}
				}				
				this.buckets[i] = seg;
				oldSegment = false; // we write on at most one old segment
			}
			int hashOffset;
			int hash;
			int pointerOffset;
			long pointer;
			IntArrayList hashList = new IntArrayList(NUM_ENTRIES_PER_BUCKET);
			LongArrayList pointerList = new LongArrayList(NUM_ENTRIES_PER_BUCKET);
			IntArrayList overflowHashes = new IntArrayList(64);
			LongArrayList overflowPointers = new LongArrayList(64);
			
			// go over all buckets and split them between old and new buckets
			for (int i = 0; i < numPartitions; i++) {
				InMemoryPartition<T> partition = this.partitions.get(i);
				final MemorySegment[] overflowSegments = partition.overflowSegments;
				
				int posHashCode;
				for (int j = 0, bucket = i; j < this.buckets.length && bucket < oldNumBuckets; j++) {
					MemorySegment segment = this.buckets[j];
					// go over all buckets in the segment belonging to the partition
					for (int k = bucket % bucketsPerSegment; k < bucketsPerSegment && bucket < oldNumBuckets; k += numPartitions, bucket += numPartitions) {
						int bucketOffset = k * HASH_BUCKET_SIZE;
						if((int)segment.get(bucketOffset + HEADER_PARTITION_OFFSET) != i) {
							throw new IOException("Accessed wrong bucket! wanted: " + i + " got: " + segment.get(bucketOffset + HEADER_PARTITION_OFFSET));
						}
						// loop over all segments that are involved in the bucket (original bucket plus overflow buckets)
						int countInSegment = segment.getInt(bucketOffset + HEADER_COUNT_OFFSET);
						int numInSegment = 0;
						pointerOffset = bucketOffset + BUCKET_POINTER_START_OFFSET;
						hashOffset = bucketOffset + BUCKET_HEADER_LENGTH;
						while (true) {
							while (numInSegment < countInSegment) {
								hash = segment.getInt(hashOffset);
								if((hash % this.numBuckets) != bucket && (hash % this.numBuckets) != (bucket+oldNumBuckets)) {
									throw new IOException("wanted: " + bucket + " or " + (bucket + oldNumBuckets) + " got: " + hash%this.numBuckets);
								}
								pointer = segment.getLong(pointerOffset);
								hashList.add(hash);
								pointerList.add(pointer);
								pointerOffset += POINTER_LEN;
								hashOffset += HASH_CODE_LEN;
								numInSegment++;
							}
							// this segment is done. check if there is another chained bucket
							final long forwardPointer = segment.getLong(bucketOffset + HEADER_FORWARD_OFFSET);
							if (forwardPointer == BUCKET_FORWARD_POINTER_NOT_SET) {
								break;
							}
							final int overflowSegNum = (int) (forwardPointer >>> 32);
							segment = overflowSegments[overflowSegNum];
							bucketOffset = (int) forwardPointer;
							countInSegment = segment.getInt(bucketOffset + HEADER_COUNT_OFFSET);
							pointerOffset = bucketOffset + BUCKET_POINTER_START_OFFSET;
							hashOffset = bucketOffset + BUCKET_HEADER_LENGTH;
							numInSegment = 0;
						}
						segment = this.buckets[j];
						bucketOffset = k * HASH_BUCKET_SIZE;
						// reset bucket for re-insertion
						segment.putInt(bucketOffset + HEADER_COUNT_OFFSET, 0);
						segment.putLong(bucketOffset + HEADER_FORWARD_OFFSET, BUCKET_FORWARD_POINTER_NOT_SET);
						// refill table
						if(hashList.size() != pointerList.size()) {
							throw new IOException("Pointer and hash counts do not match. hashes: " + hashList.size() + " pointer: " + pointerList.size());
						}
						int newSegmentIndex = (bucket + oldNumBuckets) / bucketsPerSegment;
						MemorySegment newSegment = this.buckets[newSegmentIndex];
						
						// we need to avoid overflows in the first run
						int oldBucketCount = 0;
						int newBucketCount = 0;
						while (!hashList.isEmpty()) {
							hash = hashList.removeLast();
							pointer = pointerList.removeLong(pointerList.size()-1);
							posHashCode = hash % this.numBuckets;
							if (posHashCode == bucket && oldBucketCount < NUM_ENTRIES_PER_BUCKET) {
								bucketOffset = (bucket % bucketsPerSegment) * HASH_BUCKET_SIZE;
								insertBucketEntryFromStart(segment, bucketOffset, hash, pointer, partition.getPartitionNumber());
								oldBucketCount++;
							}
							else if (posHashCode == (bucket + oldNumBuckets) && newBucketCount < NUM_ENTRIES_PER_BUCKET) {
								bucketOffset = ((bucket + oldNumBuckets) % bucketsPerSegment) * HASH_BUCKET_SIZE;
								insertBucketEntryFromStart(newSegment, bucketOffset, hash, pointer, partition.getPartitionNumber());
								newBucketCount++;
							}
							else if (posHashCode == (bucket + oldNumBuckets) || posHashCode == bucket) {
								overflowHashes.add(hash);
								overflowPointers.add(pointer);
							}
							else {
								throw new IOException("Accessed wrong bucket. Target: " + bucket + " or " + (bucket + oldNumBuckets) + " Hit: " + posHashCode);
							}
						}
						hashList.clear();
						pointerList.clear();
					}
				}
				// reset partition's overflow buckets and reclaim their memory
				this.availableMemory.addAll(partition.resetOverflowBuckets());
				// clear overflow lists
				int bucketArrayPos;
				int bucketInSegmentPos;
				MemorySegment bucket;
				while(!overflowHashes.isEmpty()) {
					hash = overflowHashes.removeLast();
					pointer = overflowPointers.removeLong(overflowPointers.size()-1);
					posHashCode = hash % this.numBuckets; 
					bucketArrayPos = posHashCode >>> this.bucketsPerSegmentBits;
					bucketInSegmentPos = (posHashCode & this.bucketsPerSegmentMask) << NUM_INTRA_BUCKET_BITS;
					bucket = this.buckets[bucketArrayPos];
					insertBucketEntryFromStart(bucket, bucketInSegmentPos, hash, pointer, partition.getPartitionNumber());
				}
				overflowHashes.clear();
				overflowPointers.clear();
			}
			this.isResizing = false;
			return true;
		}
	}
284149
public List<Partition> getPartitionsFromDataset() throws IOException{
    try (AutoReturnableObject<IMetaStoreClient> client = getClientPool().getClient()) {
      List<Partition> partitions =
          HiveUtils.getPartitions(client.get(), getTable(), Optional.<String>absent());
      return sortPartitions(partitions);
    }
  }