__record_progress: save progress to respective restoration file
            Args: None
            Returns: None
966782
def __record_progress(self, next_step=None):
        
        config.SUSHI_BAR_CLIENT.report_progress(
            self.get_status(), self.get_status().value/Status.DONE.value)
        if next_step:
            now = time.time()
            config.SUSHI_BAR_CLIENT.report_stage(self.get_status(), now - self.timestamp)
            self.timestamp = now
            self.status = next_step
        with open(self.get_restore_path(Status.LAST), 'wb') as handle, open(self.get_restore_path(), 'wb') as step_handle:
            pickle.dump(self, handle)
            pickle.dump(self, step_handle)
1315096
def progress_inc(self, step=1, new_status=None):
    
    self._value += step
    self.progress(self._value, new_status=new_status)
411070
def progress(self):
        
        return Progress(
            done=len(self._get_all_set_properties()),
            base=len(worker_mapping()),
        )
994405
def _handle_progress(self, total, progress_callback):  # pylint: disable=no-self-use
    """"""
    current = 0
    while True:
      current += yield
      try:
        progress_callback(current, total)
      except Exception:  # pylint: disable=broad-except
        _LOG.exception('Progress callback raised an exception. %s',
                       progress_callback)
        continue
772793
def progress(self, p):
        '''
        
        '''
        self.task_stack[-1] = self.task_stack[-1]._replace(progress=p)
        self.progress_report()
387519
protected void publish(final LogRecord record) {
    if(record instanceof ProgressLogRecord) {
      ProgressLogRecord preg = (ProgressLogRecord) record;
      Progress prog = preg.getProgress();
      JProgressBar pbar = getOrCreateProgressBar(prog);
      updateProgressBar(prog, pbar);
      if(prog.isComplete()) {
        removeProgressBar(prog, pbar);
      }
      if(prog.isComplete() || prog instanceof StepProgress) {
        publishTextRecord(record);
      }
    }
    else {
      publishTextRecord(record);
    }
  }
1276984
def save!(&block)
      fail StandardError, 'Cannot re-save a flow' if id
      self.id = SecureRandom.uuid
      record_id = redis_id + ':definition'
      self.created_at = Time.now.to_f

      # Actually save the record:
      redis.set record_id, serialize

      # Transition into the first phase:
      add_transition to: current_phase, timestamp: Time.now.utc.to_f

      # Add our id to the list of active flows:
      redis.sadd 'distribot.flows.active', id
      redis.incr('distribot.flows.running')

      # Announce our arrival to the rest of the system:
      Distribot.publish! 'distribot.flow.created', flow_id: id

      wait_for_flow_to_finish(block) if block_given?
      self
    end
237854
def read_oplog_progress(self):
        

        if self.oplog_checkpoint is None:
            return None

        # Check for empty file
        try:
            if os.stat(self.oplog_checkpoint).st_size == 0:
                LOG.info("MongoConnector: Empty oplog progress file.")
                return None
        except OSError:
            return None

        with open(self.oplog_checkpoint, "r") as progress_file:
            try:
                data = json.load(progress_file)
            except ValueError:
                LOG.exception(
                    'Cannot read oplog progress file "%s". '
                    "It may be corrupt after Mongo Connector was shut down"
                    "uncleanly. You can try to recover from a backup file "
                    '(may be called "%s.backup") or create a new progress file '
                    "starting at the current moment in time by running "
                    "mongo-connector --no-dump <other options>. "
                    "You may also be trying to read an oplog progress file "
                    "created with the old format for sharded clusters. "
                    "See https://github.com/10gen-labs/mongo-connector/wiki"
                    "/Oplog-Progress-File for complete documentation."
                    % (self.oplog_checkpoint, self.oplog_checkpoint)
                )
                return
            # data format:
            # [name, timestamp] = replica set
            # [[name, timestamp], [name, timestamp], ...] = sharded cluster
            if not isinstance(data[0], list):
                data = [data]
            with self.oplog_progress:
                self.oplog_progress.dict = dict(
                    (name, util.long_to_bson_ts(timestamp)) for name, timestamp in data
                )
1361967
def record(self):
        # type: () -> bytes
        '''
        
        '''
        if not self._initialized:
            raise pycdlibexception.PyCdlibInternalError('RR record not yet initialized!')

        return b'RR' + struct.pack('=BBB', RRRRRecord.length(), SU_ENTRY_VERSION, self.rr_flags)
571391
def commit(self):
        
        assert self.record
        result = self.files_written, self.dirs_created
        self._init_record()
        return result
1230322
def _finish_progress(self):
        

        if self._show_progressbar:
            if self._progressbar is None:
                self._initialize_progressbar()
            if self._progressbar is not None:
                self._progressbar.finish()

        if self._progress_callback is not None:
            self._progress_callback(100.0)
381692
def set_uploading(self, files_uploaded):
        
        self.files_uploaded = files_uploaded
        self.__record_progress(Status.UPLOADING_FILES)
1257762
def _reconstruct_record(self, record_object):
        
        '''  '''
        
        record_details = {}
        current_details = record_details
        for key, value in self.model.keyMap.items():
            record_key = key[1:]
            if record_key:
                record_value = getattr(record_object, record_key, None)
                if record_value != None:
                    record_segments = record_key.split('.')
                    for i in range(len(record_segments)):
                        segment = record_segments[i]
                        if i + 1 < len(record_segments):
                            if segment not in record_details.keys():
                                current_details[segment] = {}
                            current_details = current_details[segment]
                        else:
                            if isinstance(record_value, bytes):
                                current_details[segment] = pickle.loads(record_value)
                            else:
                                current_details[segment] = record_value
                    current_details = record_details
                    
        return record_details
1063583
def _update_resume_for_completed(self):
        # type: (Descriptor) -> None
        
        if not self.is_resumable:
            return
        with self._meta_lock:
            self._resume_mgr.add_or_update_record(
                self.final_path, self._ase, self._chunk_size,
                self._next_integrity_chunk, True, None,
            )
698259
def processed_records(self, records):
        
        for record in records:
            self.logger.debug('Marking as processed: %s', record)
            record.processed = True
        db.session.commit()
625921
def write_oplog_progress(self):
        

        if self.oplog_checkpoint is None:
            return None

        with self.oplog_progress as oplog_prog:
            oplog_dict = oplog_prog.get_dict()
        items = [[name, util.bson_ts_to_long(oplog_dict[name])] for name in oplog_dict]
        if not items:
            return

        # write to temp file
        backup_file = self.oplog_checkpoint + ".backup"
        os.rename(self.oplog_checkpoint, backup_file)

        # for each of the threads write to file
        with open(self.oplog_checkpoint, "w") as dest:
            if len(items) == 1:
                # Write 1-dimensional array, as in previous versions.
                json_str = json.dumps(items[0])
            else:
                # Write a 2d array to support sharded clusters.
                json_str = json.dumps(items)
            try:
                dest.write(json_str)
            except IOError:
                # Basically wipe the file, copy from backup
                dest.truncate()
                with open(backup_file, "r") as backup:
                    shutil.copyfile(backup, dest)

        os.remove(backup_file)
1282740
def update_progress(total, done)
      percentage = total.zero? ? 100 : ((done.to_f / total) * 100).to_i
      progress(percentage)
    end