sync last recorded export to actual

    Use --dryrun to check status.
338673
def sync(config, group, accounts=(), dryrun=False, region=None):
    
    config = validate.callback(config)
    destination = config.get('destination')
    client = boto3.Session().client('s3')

    for account in config.get('accounts', ()):
        if accounts and account['name'] not in accounts:
            continue

        session = get_session(account['role'], region)
        account_id = session.client('sts').get_caller_identity()['Account']
        prefix = destination.get('prefix', '').rstrip('/') + '/%s' % account_id
        prefix = "%s/%s" % (prefix, group)

        exports = get_exports(client, destination['bucket'], prefix + "/")

        role = account.pop('role')
        if isinstance(role, six.string_types):
            account['account_id'] = role.split(':')[4]
        else:
            account['account_id'] = role[-1].split(':')[4]
        account.pop('groups')

        if exports:
            last_export = exports.pop()
            account['export'] = last_export
        else:
            account['export'] = 'missing'
            last_export = None
        try:
            tag_set = client.get_object_tagging(
                Bucket=destination['bucket'], Key=prefix).get('TagSet', [])
        except ClientError:
            tag_set = []

        tags = {t['Key']: t['Value'] for t in tag_set}
        tagged_last_export = None

        if 'LastExport' in tags:
            le = parse(tags['LastExport'])
            tagged_last_export = (le.year, le.month, le.day)
            account['sync'] = tagged_last_export
        else:
            account['sync'] = account['export'] != 'missing' and 'sync' or 'missing'

        if last_export is None:
            continue

        if tagged_last_export == last_export or account['export'] == 'missing':
            continue

        if dryrun:
            continue

        client.put_object(
            Bucket=destination['bucket'],
            Key=prefix,
            Body=json.dumps({}),
            ACL="bucket-owner-full-control",
            ServerSideEncryption="AES256")

        export_time = datetime.now().replace(tzinfo=tzlocal()).astimezone(tzutc())
        export_time = export_time.replace(
            year=last_export[0], month=last_export[1], day=last_export[2],
            minute=0, second=0, microsecond=0, hour=0)
        client.put_object_tagging(
            Bucket=destination['bucket'], Key=prefix,
            Tagging={
                'TagSet': [{
                    'Key': 'LastExport',
                    'Value': export_time.isoformat()}]})

    accounts_report = []
    for a in config.get('accounts'):
        if accounts and a['name'] not in accounts:
            continue
        if isinstance(a['sync'], tuple):
            a['sync'] = "%s/%s/%s" % (a['sync'])
        if isinstance(a['export'], tuple):
            a['export'] = "%s/%s/%s" % (a['export'])
        accounts_report.append(a)

    accounts_report.sort(key=operator.itemgetter('export'), reverse=True)
    print(tabulate(accounts_report, headers='keys'))
925075
def replay_sync(self, live=False):
        ''
        self.cursorRowIndex = 0
        CommandLog.currentReplay = self
        with Progress(total=len(self.rows)) as prog:
            while self.cursorRowIndex < len(self.rows):
                if CommandLog.currentReplay is None:
                    status('replay canceled')
                    return

                vd().statuses.clear()
                try:
                    if self.replayOne(self.cursorRow):
                        self.cancel()
                        return
                except Exception as e:
                    self.cancel()
                    exceptionCaught(e)
                    status('replay canceled')
                    return

                self.cursorRowIndex += 1
                prog.addProgress(1)

                sync(1 if live else 0)  # expect this thread also if playing live
                while not self.delay():
                    pass

        status('replay complete')
        CommandLog.currentReplay = None
219217
def fast_sync_snapshot_compress( snapshot_dir, export_path ):
    

    snapshot_dir = os.path.abspath(snapshot_dir)
    export_path = os.path.abspath(export_path)
    if os.path.exists(export_path):
        return {'error': 'Snapshot path exists: {}'.format(export_path)}

    old_dir = os.getcwd()
    
    count_ref = [0]

    def print_progress(tarinfo):
        count_ref[0] += 1
        if count_ref[0] % 100 == 0:
            log.debug("{} files compressed...".format(count_ref[0]))

        return tarinfo

    try:
        os.chdir(snapshot_dir)
        with tarfile.TarFile.bz2open(export_path, "w") as f:
            f.add(".", filter=print_progress)

    except:
        os.chdir(old_dir)
        raise
    
    finally:
        os.chdir(old_dir)

    return {'status': True}
70637
def touch_last_worklog(self, issue):
        
        worklogs = self.get_worklog(issue)
        if worklogs:
            last_worklog = worklogs[-1]
            last_worklog.update()
            return True
669911
def down_by_time(*filters, remote_dir=DEFAULT_REMOTE_DIR, local_dir=".", count=1):
    """"""
    files = command.list_files(*filters, remote_dir=remote_dir)
    most_recent = sorted(files, key=lambda f: f.datetime)
    to_sync = most_recent[-count:]
    _notify_sync(Direction.down, to_sync)
    down_by_files(to_sync[::-1], local_dir=local_dir)
1033785
def is_running(self):
        
        self.__update_status()
        return self.status == Status.UP or self.status == Status.DECOMMISSIONED
471519
def update_last_backup_meta_data(self):
        """"""
        self.meta['last_backup']['time'] = get_time_string_for_float(self.last_backup_time)
        self.meta['last_backup']['file_system_path'] = self._tmp_storage_path
        self.meta['last_backup']['marked_dirty'] = self.state_machine_model.state_machine.marked_dirty
76644
def check(self):
        '''  '''
        if self.lastrun + self.interval < time.time():
            return True
        else:
            return False
115129
def resume(self):
        
        if self._status != TransferState.PAUSED:
            self._running.set()
            self._status = TransferState.RUNNING
        else:
            raise SbgError('Can not pause. Download not in PAUSED state.')
861006
def check_sync(self):
        """"""
        # If refresh interval is not specified, we should refresh every time.
        expiration = utcnow()
        refresh_td = current_app.config.get('GITHUB_REFRESH_TIMEDELTA')
        if refresh_td:
            expiration -= refresh_td
        last_sync = parse_timestamp(self.account.extra_data['last_sync'])
        return last_sync < expiration
763310
def _print_if_needed(self):
        """"""
        if self._in_txn or self.print_frequency is None:
            return
        elif self.last_export is not None and \
             self.last_export + self.print_frequency > time.time():
            return

        self.export()
229835
def status(config):
    """"""
    with open(config) as fh:
        config = yaml.safe_load(fh.read())
    jsonschema.validate(config, CONFIG_SCHEMA)
    last_index = get_incremental_starts(config, None)
    accounts = {}
    for (a, region), last in last_index.items():
        accounts.setdefault(a, {})[region] = last
    print(yaml.safe_dump(accounts, default_flow_style=False))
797854
def _refresh_synced(self):
        
        if self.path.startswith('http'):
            juicer.utils.Log.log_debug("%s is not synced" % self.path)
            self.is_synced = False
        else:
            juicer.utils.Log.log_debug("%s is synced" % self.path)
            self.is_synced = True
1358197
def sync(self, force=None):
        """"""

        try:
            if force:
                sd = force
            else:
                sd = self.sync_dir()

            if sd == self.SYNC_DIR.FILE_TO_RECORD:

                if force and not self.exists():
                    return None

                self.fs_to_record()

            elif sd == self.SYNC_DIR.RECORD_TO_FILE:
                self.record_to_fs()

            else:
                return None

            self._dataset.config.sync[self.file_const][sd] = time.time()
            return sd
        except Exception as e:
            self._bundle.rollback()
            self._bundle.error("Failed to sync '{}': {}".format(self.file_const, e))
            raise
686099
def start_check (aggregate, out):
    """"""
    # check in background
    t = threading.Thread(target=director.check_urls, args=(aggregate,))
    t.start()
    # time to wait for new data
    sleep_seconds = 2
    # current running time
    run_seconds = 0
    while not aggregate.is_finished():
        yield out.get_data()
        time.sleep(sleep_seconds)
        run_seconds += sleep_seconds
        if run_seconds > MAX_REQUEST_SECONDS:
            director.abort(aggregate)
            break
    yield out.get_data()
498014
def status(self):
        
        return {
            self.original: {
                'formatted_filename': self.out_location,
                'state': self.state,
                'messages': '\n\t'.join(self.messages),
            }
        }
403110
def _sync(form, saltenv=None, extmod_whitelist=None, extmod_blacklist=None):
    '''
    
    '''
    if saltenv is None:
        saltenv = _get_top_file_envs()
    if isinstance(saltenv, six.string_types):
        saltenv = saltenv.split(',')
    ret, touched = salt.utils.extmods.sync(__opts__, form, saltenv=saltenv, extmod_whitelist=extmod_whitelist,
                                           extmod_blacklist=extmod_blacklist)
    # Dest mod_dir is touched? trigger reload if requested
    if touched:
        mod_file = os.path.join(__opts__['cachedir'], 'module_refresh')
        with salt.utils.files.fopen(mod_file, 'a'):
            pass
    if form == 'grains' and \
       __opts__.get('grains_cache') and \
       os.path.isfile(os.path.join(__opts__['cachedir'], 'grains.cache.p')):
        try:
            os.remove(os.path.join(__opts__['cachedir'], 'grains.cache.p'))
        except OSError:
            log.error('Could not remove grains cache!')
    return ret