Prepare this slice for reading. If this the first slice in the shard, then the shardStartTime
will be used. However if the is not the first slice use the last time read (+1, since that
timestamp was already read) as the starting point.
541125
@Override
  public void beginSlice() {
    log.fine("Beginning slice in shard: " + shardLogQuery.getStartTimeUsec() + "-"
        + shardLogQuery.getEndTimeUsec());
    LogService logService = LogServiceFactory.getLogService();

    if (lastOffset != null) {
      shardLogQuery.offset(lastOffset);
    }
    logIterator = logService.fetch(shardLogQuery).iterator();
  }
339522
def get_shard_map(self, force_refresh=False):
        
        now = datetime.utcnow()
        if force_refresh is True or \
                        self.shard_map is None or \
                        (now - self.last_refresh).total_seconds() > self.refresh_ttl:
            self.last_refresh = now
            self.refresh_shard_map()
        return self.shard_map
1017977
def get_shard_num_by_key(self, key):
        
        key_id = self._get_key_id_from_key(key)
        return self.get_shard_num_by_key_id(key_id)
879436
def get_shard_names(self):
        
        results = []
        for shard_num in range(0, self.num_shards()):
            shard_name = self.get_shard_name(shard_num)
            results.append(shard_name)

        return results
620279
def _init_scratch(self):
        """"""
        scratch = np.zeros((self._num_shards, self._shard_size),
                           dtype=np.complex64)
        scratch_handle = mem_manager.SharedMemManager.create_array(
            scratch.view(dtype=np.float32))
        self._shared_mem_dict['scratch_handle'] = scratch_handle
1202395
def readiterinit(d):
    

    # set requested time range based on given parameters
    starttime_mjd = d['starttime_mjd']
    timeskip = d['inttime']*d['nskip']
    starttime = qa.getvalue(qa.convert(qa.time(qa.quantity(starttime_mjd+timeskip/(24.*60*60),'d'),form=['ymd'], prec=9)[0], 's'))[0]
    stoptime = qa.getvalue(qa.convert(qa.time(qa.quantity(starttime_mjd+(timeskip+(d['nints']+1)*d['inttime'])/(24.*60*60), 'd'), form=['ymd'], prec=9)[0], 's'))[0]  # nints+1 to be avoid buffer running out and stalling iteration
    logger.debug('Time of first integration:', qa.time(qa.quantity(starttime_mjd,'d'),form=['ymd'],prec=9)[0])
    logger.info('Reading times %s to %s in %d iterations' % (qa.time(qa.quantity(starttime_mjd+timeskip/(24.*60*60),'d'),form=['hms'], prec=9)[0], qa.time(qa.quantity(starttime_mjd+(timeskip+(d['nints']+1)*d['inttime'])/(24.*60*60), 'd'), form=['hms'], prec=9)[0], d['nthread']))

    # read data into data structure
    ms.open(d['filename'])
    if len(d['spwlist']) == 1:
        ms.selectinit(datadescid=d['spwlist'][0])
    else:
        ms.selectinit(datadescid=0, reset=True)    # reset includes spw in iteration over time
    selection = {'time': [starttime, stoptime], 'uvdist': [1., 1e10], 'antenna1': d['ants'], 'antenna2': d['ants']}    # exclude auto-corrs
    ms.select(items = selection)
    ms.selectpolarization(d['pols'])
    ms.iterinit(['TIME'], 0, d['iterint']*d['nbl']*d['nspw']*d['npol'], adddefaultsortcolumns=False)
    iterstatus = ms.iterorigin()
68003
def _wait(starting_time, first_timestamp, timestamp):
        
        target_time = starting_time + (timestamp - first_timestamp)
        time.sleep(max(target_time - time.time(), 0))
1184010
private boolean isValidShardDate(String shardDate) {
        try {
            // If the format is invalid, a ParseException is thrown.
            Utils.dateFromString(shardDate);
            return true;
        } catch (IllegalArgumentException ex) {
            return false;
        }
    }
971353
public int getPartition(Shard key, IntermediateForm value, int numPartitions) {
    int partition = map.get(key).intValue();
    if (partition < numPartitions) {
      return partition;
    } else {
      return numPartitions - 1;
    }
  }
153443
@Override
    public void processMessage() {
        // protection check, we definitely don't want double spending here
        INDArray syn0 = storage.getArray(WordVectorStorage.SYN_0);
        INDArray syn1 = storage.getArray(WordVectorStorage.SYN_1);
        INDArray syn1Neg = storage.getArray(WordVectorStorage.SYN_1_NEGATIVE);
        INDArray expTable = storage.getArray(WordVectorStorage.EXP_TABLE);
        if (syn0 == null) {
            log.info("sI_{} is starting initialization...", transport.getShardIndex());

            // we initialize only syn0/syn1/syn1neg and expTable
            // negTable will be initalized at driver level and will be shared via message
            Nd4j.getRandom().setSeed(seed * (shardIndex + 1));

            if (voidConfiguration.getExecutionMode() == ExecutionMode.AVERAGING) {
                // each shard has full own copy
                columnsPerShard = vectorLength;
            } else if (voidConfiguration.getExecutionMode() == ExecutionMode.SHARDED) {
                // each shard will have only part of the data
                if (voidConfiguration.getNumberOfShards() - 1 == shardIndex) {
                    int modulo = vectorLength % voidConfiguration.getNumberOfShards();
                    if (modulo != 0) {
                        columnsPerShard += modulo;
                        log.info("Got inequal split. using higher number of elements: {}", columnsPerShard);
                    }
                }
            }

            int[] shardShape = new int[] {numWords, columnsPerShard};

            syn0 = Nd4j.rand(shardShape, 'c').subi(0.5).divi(vectorLength);

            if (useHs)
                syn1 = Nd4j.create(shardShape, 'c');

            if (useNeg)
                syn1Neg = Nd4j.create(shardShape, 'c');

            // we handle full exp table here
            expTable = initExpTable(100000);


            storage.setArray(WordVectorStorage.SYN_0, syn0);

            if (useHs)
                storage.setArray(WordVectorStorage.SYN_1, syn1);
            if (useNeg)
                storage.setArray(WordVectorStorage.SYN_1_NEGATIVE, syn1Neg);

            storage.setArray(WordVectorStorage.EXP_TABLE, expTable);

            InitializationAggregation ia = new InitializationAggregation((short) voidConfiguration.getNumberOfShards(),
                            transport.getShardIndex());
            ia.setOriginatorId(this.originatorId);
            transport.sendMessage(ia);
        }
    }
1299671
public void beforeRecover() {
		if (data.getPeriod() > 0) {			
			final long now = System.currentTimeMillis();
			long startTime = data.getStartTime();
			while(startTime <= now) {
				startTime += data.getPeriod();
				data.setStartTime(startTime);				
			}			
			if (logger.isDebugEnabled()) {
				logger.debug("Task with id "
						+ data.getTaskID() + " start time reset to " + data.getStartTime());
			}
		}
	}
1130694
public void advanceLastDiscoveredShardOfStream(String stream, String shardId) {
		String lastSeenShardIdOfStream = this.subscribedStreamsToLastDiscoveredShardIds.get(stream);

		// the update is valid only if the given shard id is greater
		// than the previous last seen shard id of the stream
		if (lastSeenShardIdOfStream == null) {
			// if not previously set, simply put as the last seen shard id
			this.subscribedStreamsToLastDiscoveredShardIds.put(stream, shardId);
		} else if (shouldAdvanceLastDiscoveredShardId(shardId, lastSeenShardIdOfStream)) {
			this.subscribedStreamsToLastDiscoveredShardIds.put(stream, shardId);
		}
	}
1262247
def generate_shard_args(outfiles, num_examples):
  """"""
  num_shards = len(outfiles)
  num_examples_per_shard = num_examples // num_shards
  start_idxs = [i * num_examples_per_shard for i in range(num_shards)]
  end_idxs = list(start_idxs)
  end_idxs.pop(0)
  end_idxs.append(num_examples)
  return zip(start_idxs, end_idxs, outfiles)
1193726
private void initHashArea()
    {
        /* Let's use small hash area of size 4, and one spill; don't
         * want too big (need to clear up room), nor too small (only
         * collisions)
         */
        mAttrHashSize = mAttrSpillEnd = 4;
        if (mAttrMap == null || mAttrMap.length < mAttrHashSize) {
            mAttrMap = new int[mAttrHashSize+1];
        }
        mAttrMap[0] =  mAttrMap[1] = mAttrMap[2] = mAttrMap[3] = 0;
        allocBuffers();
    }
764302
public boolean verifyTimestampImprint() throws NoSuchAlgorithmException {
        if (timeStampToken == null)
            return false;
        MessageImprint imprint = timeStampToken.getTimeStampInfo().toASN1Structure().getMessageImprint();
        byte[] md = MessageDigest.getInstance("SHA-1").digest(digest);
        byte[] imphashed = imprint.getHashedMessage();
        boolean res = Arrays.equals(md, imphashed);
        return res;
    }
1349488
def is_in_shard(self, s):
    
    return self.compute_shard(s, self._nshards) == self._shard
1228067
function _sha1() {
  var sha = forge.md.sha1.create();
  var num = arguments.length;
  for (var i = 0; i < num; ++i) {
    sha.update(arguments[i]);
  }
  return sha.digest();
}