Copes scratch shards to state shards.
324377
def _copy_scratch_to_state(args: Dict[str, Any]):
    """"""
    np.copyto(_state_shard(args), _scratch_shard(args))
781738
def put(self, key, value):
    ''''''
    self.shardDatastore(key).put(key, value)
1224259
private void realSubscribe(Subscriber subs) {
        AtomicInteger doneCount = new AtomicInteger(shards.size());
        ChangeReceiver receiver = subs.getReceiver();
        ArrayList<Subscriber> subsList = new ArrayList();
        shards.forEach( shard -> {
            Subscriber shardSubs = new Subscriber(subs.getFilter(), change -> {
                if (change.getType() == ChangeMessage.QUERYDONE) {
                    int count = doneCount.decrementAndGet();
                    if (count == 0) {
                        receiver.receive(change);
                    }
                } else {
                    receiver.receive(change);
                }
            });
            subsList.add(shardSubs);
            shard.subscribe(shardSubs);
        });
        subsMap.put(subs,subsList);
    }
625708
def heap_item(clock, record, shard):
    """"""
    # Primary ordering is by event creation time.
    # However, creation time is *approximate* and has whole-second resolution.
    # This means two events in the same shard within one second can't be ordered.
    ordering = record["meta"]["created_at"]
    # From testing, SequenceNumber isn't a guaranteed ordering either.  However,
    # it is guaranteed to be unique within a shard.  This will be tie-breaker
    # for multiple records within the same shard, within the same second.
    second_ordering = int(record["meta"]["sequence_number"])
    # It's possible though unlikely, that sequence numbers will collide across
    # multiple shards, within the same second.  The final tie-breaker is
    # a monotonically increasing integer from the buffer.
    total_ordering = (ordering, second_ordering, clock())
    return total_ordering, record, shard
389786
def _single_qubit_accumulate_into_scratch(args: Dict[str, Any]):
    """"""
    index = args['indices'][0]
    shard_num = args['shard_num']
    half_turns = args['half_turns']
    num_shard_qubits = args['num_shard_qubits']
    scratch = _scratch_shard(args)

    # ExpZ = exp(-i pi Z half_turns / 2).
    if index >= num_shard_qubits:
        # Acts on prefix qubits.
        sign = 1 - 2 * _kth_bit(shard_num, index - num_shard_qubits)
        scratch -= half_turns * sign
    else:
        # Acts on shard qubits.
        scratch -= half_turns * _pm_vects(args)[index]
1309436
def _add(self, shard_uri, name):
        """"""
        return self.router_command("addShard", (shard_uri, {"name": name}), is_eval=False)
558244
@Override
  public Iterator<E> shardIterator(int shardId, int numShards) {
    if (shardId >= entries.length) {
      return null;
    }
    if (shardId >= numShards) {
      throw new IllegalArgumentException(
          "Shard id must be less than total shards, shardId: " + shardId
              + ", numShards: " + numShards);
    }
    return new SetIterator(shardId, numShards);
  }
657063
def cleanup(self):
        """"""
        for _id, shard in self._shards.items():
            if shard.get('isServer', False):
                Servers().remove(shard['_id'])
            if shard.get('isReplicaSet', False):
                ReplicaSets().remove(shard['_id'])

        for mongos in self._routers:
            Servers().remove(mongos)

        for config_id in self._configsvrs:
            self.configdb_singleton.remove(config_id)

        self._configsvrs = []
        self._routers = []
        self._shards = {}
1104374
private final void updateShardInfo() {
    final long expirationTime = 1000;
    // don't update too much
    if (shardKey == null || (shardInfoUpdated
        && Objects.requireNonNull(shardInfoUpdate, "update expire time not set") < System.currentTimeMillis())) {
      return;
    }
    // and only if necessary
    if (!shardInfoUpdated || !finished || shardInfoNeedUpdate) {
      // reset
      shardInfoError = false;
      // get list of relevant stages
      Set<Integer> stagesList = new HashSet<>();
      for (Integer stage : shardStageKeys.keySet()) {
        if (!shardStageStatus.containsKey(stage) || !shardStageStatus.get(stage).finished) {
          stagesList.add(stage);
        }
      }
      // loop for this list over shards
      for (Entry<String, ShardStatus> entry : shards.entrySet()) {
        ShardStatus shardStatus = entry.getValue();
        SolrClient solrClient = null;
        StageStatus stageStatus;
        // then loop over stages
        for (Integer stage : stagesList) {
          // get shardStage
          if (!shardStageStatus.containsKey(stage)) {
            stageStatus = new StageStatus(stage);
            shardStageStatus.put(stage, stageStatus);
          } else {
            stageStatus = shardStageStatus.get(stage);
          }
          if (shardStatus.finishedStages.contains(stage)) {
            stageStatus.add(true, shardStatus.numberDocumentsFoundStage.get(stage));
          } else {
            // create request
            ModifiableSolrParams solrParams = new ModifiableSolrParams();
            solrParams.add(CommonParams.QT, shardStatus.mtasHandler);
            solrParams.add(MtasRequestHandler.PARAM_ACTION, MtasRequestHandler.ACTION_STATUS);
            solrParams.add(MtasRequestHandler.PARAM_KEY, shardStageKeys.get(stage));

            try {
              // set solrClient
              solrClient = new HttpSolrClient.Builder(shardStatus.location).build();
              // get response
              QueryResponse response = solrClient.query(solrParams);
              // check for response
              if (response.getResponse().findRecursive(MtasSolrComponentStatus.NAME) != null) {
                shardStatus.numberDocumentsFoundStage.put(stage,
                    getLong(response.getResponse(), MtasSolrComponentStatus.NAME, NAME_STATUS_DOCUMENT_NUMBER_FOUND));
                shardStatus.timeStage.put(stage,
                    getInteger(response.getResponse(), MtasSolrComponentStatus.NAME, NAME_TIME_TOTAL));
                stageStatus.add(
                    shardStatus.setFinishedStage(stage,
                        getBoolean(response.getResponse(), MtasSolrComponentStatus.NAME, NAME_FINISHED)),
                    shardStatus.numberDocumentsFoundStage.get(stage));
                shardInfoError = shardStatus.setErrorStage(stage,
                    getString(response.getResponse(), MtasSolrComponentStatus.NAME, NAME_ERROR)) || shardInfoError;
                shardStatus.setAbortStage(stage,
                    getString(response.getResponse(), MtasSolrComponentStatus.NAME, NAME_ABORT));
                if (stage.equals(currentStage)) {
                  shardStatus.stage = stage;
                  shardStatus.stageNumberDocumentsFinished = getLong(response.getResponse(),
                      MtasSolrComponentStatus.NAME, NAME_STATUS_DOCUMENT_NUMBER_FINISHED);
                  shardStatus.stageSubNumberDocumentsFinished = getLongMap(response.getResponse(),
                      MtasSolrComponentStatus.NAME, NAME_STATUS_DOCUMENT_SUB_NUMBER_FINISHED);
                  shardStatus.stageSubNumberDocumentsTotal = getLong(response.getResponse(),
                      MtasSolrComponentStatus.NAME, NAME_STATUS_DOCUMENT_SUB_NUMBER_TOTAL);
                  shardStatus.stageSubNumberDocumentsFinishedTotal = getLong(response.getResponse(),
                      MtasSolrComponentStatus.NAME, NAME_STATUS_DOCUMENT_SUB_NUMBER_FINISHED_TOTAL);
                  shardStatus.stageNumberSegmentsFinished = getInteger(response.getResponse(),
                      MtasSolrComponentStatus.NAME, NAME_STATUS_SEGMENT_SUB_NUMBER_FINISHED_TOTAL);
                  shardStatus.stageSubNumberSegmentsFinished = getIntegerMap(response.getResponse(),
                      MtasSolrComponentStatus.NAME, NAME_STATUS_SEGMENT_SUB_NUMBER_FINISHED);
                  shardStatus.stageSubNumberSegmentsTotal = getInteger(response.getResponse(),
                      MtasSolrComponentStatus.NAME, NAME_STATUS_SEGMENT_SUB_NUMBER_TOTAL);
                  shardStatus.stageSubNumberSegmentsFinishedTotal = getInteger(response.getResponse(),
                      MtasSolrComponentStatus.NAME, NAME_STATUS_SEGMENT_SUB_NUMBER_FINISHED_TOTAL);
                }
              } else if (!finished && currentStage.equals(stage) && !stageStatus.checked) {
                stageStatus.finished = false;
              }
            } catch (SolrServerException | IOException e) {
              shardInfoError = shardInfoError || shardStatus.setErrorStage(stage, e.getMessage());
            } finally {
              if (solrClient != null) {
                try {
                  solrClient.close();
                } catch (IOException e) {
                  shardInfoError = shardInfoError || shardStatus.setErrorStage(stage, e.getMessage());
                }
              }
            }
          }
        }
      }
      if (!shardInfoError) {
        for (StageStatus item : shardStageStatus.values()) {
          item.checked = true;
        }
        shardInfoUpdated = true;
        shardInfoUpdate = System.currentTimeMillis() + expirationTime;
      } else {
        shardInfoUpdated = false;
        shardInfoUpdate = null;
      }
    }
  }
512755
def _update_descriptor(self, state, allocation):
        ''''''
        if state.descriptor.shard is None or state.descriptor.shard == 'lobby':
            own_shard = state.agent.get_shard_id()
            state.descriptor.shard = own_shard
        state.descriptor.resources = allocation.alloc
        f = fiber.succeed(state.descriptor)
        f.add_callback(state.agent.save_document)
        f.add_callback(self._store_descriptor)
        return f
658931
def _compute(self, feed_dict, shard):
        

        try:
            descriptor, enq = self._tfrun(self._tf_expr[shard], feed_dict=feed_dict)
            self._inputs_waiting.decrement(shard)

        except Exception as e:
            montblanc.log.exception("Compute Exception")
            raise
1218462
def with_all(continue_on_error = false)
      cluster.shards.map do |shard|
        begin
          with_shard(shard) {
            yield
          }
        rescue Exception => err
          unless continue_on_error
            raise err
          end
          err
        end
      end
    end
635376
def shard(items, num_shards):
  """"""
  sharded = []
  num_per_shard = len(items) // num_shards
  start = 0
  for _ in range(num_shards):
    sharded.append(items[start:start + num_per_shard])
    start += num_per_shard

  remainder = len(items) % num_shards
  start = len(items) - remainder
  for i in range(remainder):
    sharded[i].append(items[start + i])

  assert sum([len(fs) for fs in sharded]) == len(items)
  return sharded
989718
def _two_qubit_accumulate_into_scratch(args: Dict[str, Any]):
    """"""
    index0, index1 = args['indices']
    half_turns = args['half_turns']
    scratch = _scratch_shard(args)

    projector = _one_projector(args, index0) * _one_projector(args, index1)
    # Exp11 = exp(-i pi |11><11| half_turns), but we accumulate phases as
    # pi / 2.
    scratch += 2 * half_turns * projector
434059
def latencies(self):
        
        return [(shard_id, shard.ws.latency) for shard_id, shard in self.shards.items()]
229434
def make_scratch_dirs(file_mapping, dry_run=True):
        """"""
        scratch_dirs = {}
        for value in file_mapping.values():
            scratch_dirname = os.path.dirname(value)
            scratch_dirs[scratch_dirname] = True
        for scratch_dirname in scratch_dirs:
            if dry_run:
                print("mkdir -f %s" % (scratch_dirname))
            else:
                try:
                    os.makedirs(scratch_dirname)
                except OSError:
                    pass
1300590
function deps(s, p) {
        if (!shards.map[p] || !shards.map[p].ParentShardId) return;
        shards.deps[s] = lib.toFlags("add", shards.deps[s], shards.map[p].ParentShardId);
        deps(s, shards.map[p].ParentShardId);
    }