// Backup writes all application data into an export stream
491185
func (dfs *DistributedFilesystem) Backup(data BackupInfo, w io.Writer) error {

	backupLogger := plog.WithFields(log.Fields{
		"backupversion": data.BackupVersion,
		"timestamp":     data.Timestamp,
	})

	progress := NewProgressCounter(300)
	progress.Log = func() { plog.Infof("Written %v bytes to archive for backup", progress.Total) }

	tarOut := tar.NewWriter(io.MultiWriter(w, progress))

	// write the backup metadata
	if err := dfs.writeBackupMetadata(data, tarOut); err != nil {
		plog.WithError(err).Error("Unable to write metadata for backup")
		return err
	}

	var images []string

	baseImageLogger := backupLogger.WithField("total", len(data.BaseImages))
	baseImageLogger.Info("Preparing docker images for backup")

	// download the base images
	for i, image := range data.BaseImages {
		baseImageLogger = baseImageLogger.WithFields(log.Fields{
			"image":          image,
			"numbercomplete": i + 1,
		})

		if _, err := dfs.docker.FindImage(image); docker.IsImageNotFound(err) {
			if err := dfs.docker.PullImage(image); docker.IsImageNotFound(err) {
				baseImageLogger.Warn("Could not pull base image for backup, skipping")
				continue
			} else if err != nil {
				baseImageLogger.WithError(err).Error("Could not pull image for backup")
				return err
			}
		} else if err != nil {
			baseImageLogger.WithError(err).Error("Could not find image for backup")
			return err
		}

		baseImageLogger.Info("Prepared Docker image for backup")

		images = append(images, image)
	}

	numberOfSnapshots := len(data.Snapshots)

	backupLogger.WithField("total", numberOfSnapshots).Info("Preparing snapshots for backup")

	// export the snapshots
	for i, snapshot := range data.Snapshots {
		vol, info, err := dfs.getSnapshotVolumeAndInfo(snapshot)
		if err != nil {
			return err
		}

		// load the images from this snapshot
		tenantLogger := backupLogger.WithField("tenant", info.TenantID)
		tenantLogger.Info("Preparing images for tenant")

		r, err := vol.ReadMetadata(info.Label, ImagesMetadataFile)
		if err != nil {
			tenantLogger.WithError(err).Error("Could not receive images metadata for tenant")
			return err
		}

		var imgs []string
		if err := importJSON(r, &imgs); err != nil {
			tenantLogger.WithError(err).Error("Could not interpret images metadata for tenant")
			return err
		}

		timer := time.NewTimer(0)
		for _, img := range imgs {
			tenantImageLogger := tenantLogger.WithField("image", img)

			timer.Reset(dfs.timeout)
			if err := dfs.reg.PullImage(timer.C, img); err != nil {
				tenantImageLogger.WithError(err).Error("Could not pull image from registry")
				return err
			}

			image, err := dfs.reg.ImagePath(img)
			if err != nil {
				tenantImageLogger.WithError(err).Error("Could not get the image path from registry")
				return err
			}

			plog.WithField("image", image).Info("Prepared Docker image for backup")

			images = append(images, image)
		}

		timer.Stop()

		snapshotLogger := backupLogger.WithField("snapshot", snapshot)

		// dump the snapshot into the backup
		prefix := path.Join(SnapshotsMetadataDir, info.TenantID, info.Label)
		snapReader, errchan := dfs.snapshotSavePipe(vol, info.Label, data.SnapshotExcludes[snapshot])
		if err := rewriteTar(prefix, tarOut, snapReader); err != nil {
			// be a good citizen and clean up any running threads
			<-errchan
			snapshotLogger.WithError(err).Error("Could not write snapshot to backup")
			return err
		} else if err := <-errchan; err != nil {
			snapshotLogger.WithError(err).Error("Could not export snapshot for backup")
			return err
		}

		snapshotLogger.WithFields(log.Fields{
			"numbercomplete": i + 1,
			"total":          numberOfSnapshots,
		}).Info("Exported snapshot to backup")
	}

	// dump the images from all the snapshots into the backup
	imageReader, errchan := dfs.dockerSavePipe(images...)
	imageLogger := backupLogger.WithField("images", images)
	imageLogger.Info("Starting export of images to backup")
	if err := rewriteTar(DockerImagesFile, tarOut, imageReader); err != nil {
		// be a good citizen and clean up any running threads
		<-errchan
		imageLogger.WithError(err).Error("Could not write images to backup")
		return err
	} else if err := <-errchan; err != nil {
		imageLogger.WithError(err).Error("Could not export images for backup")
		return err
	}
	tarOut.Close()

	imageLogger.Info("Exported images to backup")

	return nil
}
452983
func (v *RsyncVolume) Export(label, parent string, writer io.Writer, excludes []string) error {
	if len(excludes) > 0 {
		glog.Warning("rsync backups do not support excluding directories")
	}
	v.Lock()
	defer v.Unlock()
	if label = strings.TrimSpace(label); label == "" {
		glog.Errorf("%s: label cannot be empty", volume.DriverTypeRsync)
		return ErrRsyncInvalidLabel
	}
	label = v.rawSnapshotLabel(label)
	tarfile := tar.NewWriter(writer)
	defer tarfile.Close()
	// Set the driver type
	header := &tar.Header{Name: fmt.Sprintf("%s-driver", label), Size: int64(len([]byte(v.Driver().DriverType())))}
	if err := tarfile.WriteHeader(header); err != nil {
		glog.Errorf("Could not export driver type header: %s", err)
		return err
	}
	if _, err := fmt.Fprint(tarfile, v.Driver().DriverType()); err != nil {
		glog.Errorf("Could not export driver type: %s", err)
		return err
	}
	// write metadata
	mdpath := filepath.Join(v.driver.MetadataDir(), label)
	if err := volume.ExportDirectory(tarfile, mdpath, fmt.Sprintf("%s-metadata", label)); err != nil {
		return err
	}
	// write volume
	volpath := v.snapshotPath(label)
	if err := volume.ExportDirectory(tarfile, volpath, fmt.Sprintf("%s-volume", label)); err != nil {
		return err
	}
	return nil
}
1017181
func (c *ItemConsume) Backup2(t1 LexItem) {
	c.items[1] = t1
	c.peekCount = 2
}
360284
func (e *etc) Dump() (Application, error) {
	appl := Application{}
	err := e.values.DoAllDeep(func(ks []string, v string) error {
		if len(ks) == 1 {
			// Continue on root element.
			return nil
		}
		path := strings.Join(ks[1:], "/")
		appl[path] = v
		return nil
	})
	if err != nil {
		return nil, err
	}
	return appl, nil
}
1184099
func (be *XtrabackupEngine) ExecuteBackup(ctx context.Context, cnf *Mycnf, mysqld MysqlDaemon, logger logutil.Logger, bh backupstorage.BackupHandle, backupConcurrency int, hookExtraEnv map[string]string) (bool, error) {

	if *xtrabackupUser == "" {
		return false, vterrors.New(vtrpc.Code_INVALID_ARGUMENT, "xtrabackupUser must be specified.")
	}
	// use a mysql connection to detect flavor at runtime
	conn, err := mysqld.GetDbaConnection()
	if conn != nil && err == nil {
		defer conn.Close()
	}

	if err != nil {
		return false, vterrors.Wrap(err, "unable to obtain a connection to the database")
	}
	pos, err := conn.MasterPosition()
	if err != nil {
		return false, vterrors.Wrap(err, "unable to obtain master position")
	}
	flavor := pos.GTIDSet.Flavor()
	logger.Infof("Detected MySQL flavor: %v", flavor)

	backupProgram := path.Join(*xtrabackupEnginePath, xtrabackupBinaryName)

	flagsToExec := []string{"--defaults-file=" + cnf.path,
		"--backup",
		"--socket=" + cnf.SocketFile,
		"--slave-info",
		"--user=" + *xtrabackupUser,
		"--target-dir=" + cnf.TmpDir,
	}
	if *xtrabackupStreamMode != "" {
		flagsToExec = append(flagsToExec, "--stream="+*xtrabackupStreamMode)
	}

	if *xtrabackupBackupFlags != "" {
		flagsToExec = append(flagsToExec, strings.Fields(*xtrabackupBackupFlags)...)
	}

	backupFileName := be.backupFileName()

	wc, err := bh.AddFile(ctx, backupFileName, 0)
	if err != nil {
		return false, vterrors.Wrapf(err, "cannot create backup file %v", backupFileName)
	}
	closeFile := func(wc io.WriteCloser, fileName string) {
		if closeErr := wc.Close(); err == nil {
			err = closeErr
		} else if closeErr != nil {
			// since we already have an error just log this
			logger.Errorf("error closing file %v: %v", fileName, err)
		}
	}
	defer closeFile(wc, backupFileName)

	backupCmd := exec.Command(backupProgram, flagsToExec...)
	backupOut, _ := backupCmd.StdoutPipe()
	backupErr, _ := backupCmd.StderrPipe()
	dst := bufio.NewWriterSize(wc, writerBufferSize)
	writer := io.MultiWriter(dst)

	// Create the gzip compression pipe, if necessary.
	var gzip *pgzip.Writer
	if *backupStorageCompress {
		gzip, err = pgzip.NewWriterLevel(writer, pgzip.BestSpeed)
		if err != nil {
			return false, vterrors.Wrap(err, "cannot create gziper")
		}
		gzip.SetConcurrency(*backupCompressBlockSize, *backupCompressBlocks)
		writer = gzip
	}

	if err = backupCmd.Start(); err != nil {
		return false, vterrors.Wrap(err, "unable to start backup")
	}

	// Copy from the stream output to destination file (optional gzip)
	_, err = io.Copy(writer, backupOut)
	if err != nil {
		return false, vterrors.Wrap(err, "cannot copy output from xtrabackup command")
	}

	// Close gzip to flush it, after that all data is sent to writer.
	if gzip != nil {
		if err = gzip.Close(); err != nil {
			return false, vterrors.Wrap(err, "cannot close gzip")
		}
	}

	// Flush the buffer to finish writing on destination.
	if err = dst.Flush(); err != nil {
		return false, vterrors.Wrapf(err, "cannot flush destination: %v", backupFileName)
	}

	stderrOutput, err := ioutil.ReadAll(backupErr)
	if err != nil {
		return false, vterrors.Wrap(err, "backup failed while reading command output")
	}
	err = backupCmd.Wait()
	output := string(stderrOutput)
	logger.Infof("Xtrabackup backup command output: %v", output)
	if err != nil {
		return false, vterrors.Wrap(err, "xtrabackup failed with error")
	}

	replicationPosition, rerr := findReplicationPosition(output, flavor, logger)
	if rerr != nil {
		return false, vterrors.Wrap(rerr, "backup failed trying to find replication position")
	}
	// open the MANIFEST
	mwc, err := bh.AddFile(ctx, backupManifest, 0)
	if err != nil {
		return false, vterrors.Wrapf(err, "cannot add %v to backup", backupManifest)
	}
	defer closeFile(mwc, backupManifest)

	// JSON-encode and write the MANIFEST
	bm := &xtraBackupManifest{
		FileName:     backupFileName,
		BackupMethod: xtrabackupBackupMethod,
		Position:     replicationPosition,
		SkipCompress: !*backupStorageCompress,
		Params:       *xtrabackupBackupFlags,
	}

	data, err := json.MarshalIndent(bm, "", "  ")
	if err != nil {
		return false, vterrors.Wrapf(err, "cannot JSON encode %v", backupManifest)
	}
	if _, err := mwc.Write([]byte(data)); err != nil {
		return false, vterrors.Wrapf(err, "cannot write %v", backupManifest)
	}

	return true, nil
}
687210
func (dao *ControlPlaneDao) Backup(backupRequest model.BackupRequest, filename *string) (err error) {
	ctx := datastore.Get()
	if len(backupRequest.Username) > 0 {
		ctx.SetUser(backupRequest.Username)
	}
	// synchronize the dfs
	dfslocker := dao.facade.DFSLock(ctx)
	dfslocker.Lock("backup")
	defer dfslocker.Unlock()

	if backupRequest.Dirpath == "" {
		backupRequest.Dirpath = dao.backupsPath
	}
	// CC-2421: Check for space before doing backup
	est := model.BackupEstimate{}
	err = dao.facade.EstimateBackup(ctx, backupRequest, &est)
	if err != nil {
		log.WithError(err).Error("Could not estimate backup size")
		return
	} else if !est.AllowBackup {
		estlog := log.WithFields(logrus.Fields{
			"estimatedsize":  est.EstimatedString,
			"destination":    est.BackupPath,
			"availablespace": est.AvailableString,
		})
		if backupRequest.Force {
			estlog.Warning("Backup not recommended, but proceeding because '-force' was specified")
		} else {
			message := fmt.Sprintf("Could not take backup - insufficient space on %s (%s estimated backup size, %s available)", est.BackupPath, est.EstimatedString, est.AvailableString)
			err := errors.New(message)
			estlog.WithError(err).Error("Could not take backup")
			return err
		}
	}

	// set the progress of the backup file
	*filename = time.Now().UTC().Format("backup-2006-01-02-150405.tgz")
	backupfilename := filepath.Join(backupRequest.Dirpath, *filename)

	inprogress.SetProgress(backupfilename, "backup")
	defer func() {
		if err != nil {
			log.WithError(err).Error("Backup failed with error")
			os.Remove(backupfilename)
		}
		inprogress.SetError(err)
	}()
	// create the file and write
	fh, err := os.Create(backupfilename)
	if err != nil {
		log.WithError(err).WithField("backupfilename", backupfilename).Error("Could not create backup file")
		return
	}
	defer fh.Close()
	w := gzip.NewWriter(fh)
	// CC-2292: Limit concurrency of backup gzipping
	// This setting will cause the writer to process up to 2 100KB blocks
	// at a time before the writer blocks. The default was 16 250KB blocks.
	// Smaller blocks will allow other goroutines to get time more frequently.
	w.SetConcurrency(100000, 2)
	defer w.Close()
	err = dao.facade.Backup(ctx, w, backupRequest.Excludes, backupRequest.SnapshotSpacePercent, backupfilename)
	return
}
937175
func (fbh *FileBackupHandle) EndBackup(ctx context.Context) error {
	if fbh.readOnly {
		return fmt.Errorf("EndBackup cannot be called on read-only backup")
	}
	return nil
}
1341658
func (s *Store) backupDB() error {
	if os.Geteuid() != 0 {
		return ErrDBUpdateNeedsRoot
	}
	backupsDir := filepath.Join(s.dir, "db-backups")
	return backup.CreateBackup(s.dbDir(), backupsDir, backupsNumber)
}
1341834
func Dump(c *cli.Context) {
	hostsfile := AlwaysLoadHostFile(c)
	jsonbytes, err := hostsfile.Hosts.Dump()
	if err != nil {
		MaybeError(c, err.Error())
	}
	fmt.Println(fmt.Sprintf("%s", jsonbytes))
}
1300153
func (bs *S3BackupStorage) StartBackup(ctx context.Context, dir, name string) (backupstorage.BackupHandle, error) {
	log.Infof("StartBackup: [s3] dir: %v, name: %v, bucket: %v", dir, name, *bucket)
	c, err := bs.client()
	if err != nil {
		return nil, err
	}

	return &S3BackupHandle{
		client:   c,
		bs:       bs,
		dir:      dir,
		name:     name,
		readOnly: false,
	}, nil
}
423509
func (c *Container) Export(outfile *os.File) error {
	dc, err := getDockerClient()
	if err != nil {
		return err
	}
	return dc.ExportContainer(dockerclient.ExportContainerOptions{c.ID, outfile})
}
73323
func (be *BuiltinBackupEngine) ExecuteBackup(ctx context.Context, cnf *Mycnf, mysqld MysqlDaemon, logger logutil.Logger, bh backupstorage.BackupHandle, backupConcurrency int, hookExtraEnv map[string]string) (bool, error) {

	logger.Infof("Hook: %v, Compress: %v", *backupStorageHook, *backupStorageCompress)

	// Save initial state so we can restore.
	slaveStartRequired := false
	sourceIsMaster := false
	readOnly := true
	var replicationPosition mysql.Position
	semiSyncMaster, semiSyncSlave := mysqld.SemiSyncEnabled()

	// See if we need to restart replication after backup.
	logger.Infof("getting current replication status")
	slaveStatus, err := mysqld.SlaveStatus()
	switch err {
	case nil:
		slaveStartRequired = slaveStatus.SlaveRunning()
	case mysql.ErrNotSlave:
		// keep going if we're the master, might be a degenerate case
		sourceIsMaster = true
	default:
		return false, vterrors.Wrap(err, "can't get slave status")
	}

	// get the read-only flag
	readOnly, err = mysqld.IsReadOnly()
	if err != nil {
		return false, vterrors.Wrap(err, "can't get read-only status")
	}

	// get the replication position
	if sourceIsMaster {
		if !readOnly {
			logger.Infof("turning master read-only before backup")
			if err = mysqld.SetReadOnly(true); err != nil {
				return false, vterrors.Wrap(err, "can't set read-only status")
			}
		}
		replicationPosition, err = mysqld.MasterPosition()
		if err != nil {
			return false, vterrors.Wrap(err, "can't get master position")
		}
	} else {
		if err = mysqld.StopSlave(hookExtraEnv); err != nil {
			return false, vterrors.Wrapf(err, "can't stop slave")
		}
		var slaveStatus mysql.SlaveStatus
		slaveStatus, err = mysqld.SlaveStatus()
		if err != nil {
			return false, vterrors.Wrap(err, "can't get slave status")
		}
		replicationPosition = slaveStatus.Position
	}
	logger.Infof("using replication position: %v", replicationPosition)

	// shutdown mysqld
	err = mysqld.Shutdown(ctx, cnf, true)
	if err != nil {
		return false, vterrors.Wrap(err, "can't shutdown mysqld")
	}

	// Backup everything, capture the error.
	backupErr := be.backupFiles(ctx, cnf, mysqld, logger, bh, replicationPosition, backupConcurrency, hookExtraEnv)
	usable := backupErr == nil

	// Try to restart mysqld, use background context in case we timed out the original context
	err = mysqld.Start(context.Background(), cnf)
	if err != nil {
		return usable, vterrors.Wrap(err, "can't restart mysqld")
	}

	// Restore original mysqld state that we saved above.
	if semiSyncMaster || semiSyncSlave {
		// Only do this if one of them was on, since both being off could mean
		// the plugin isn't even loaded, and the server variables don't exist.
		logger.Infof("restoring semi-sync settings from before backup: master=%v, slave=%v",
			semiSyncMaster, semiSyncSlave)
		err := mysqld.SetSemiSyncEnabled(semiSyncMaster, semiSyncSlave)
		if err != nil {
			return usable, err
		}
	}
	if slaveStartRequired {
		logger.Infof("restarting mysql replication")
		if err := mysqld.StartSlave(hookExtraEnv); err != nil {
			return usable, vterrors.Wrap(err, "cannot restart slave")
		}

		// this should be quick, but we might as well just wait
		if err := WaitForSlaveStart(mysqld, slaveStartDeadline); err != nil {
			return usable, vterrors.Wrap(err, "slave is not restarting")
		}
	}

	// And set read-only mode
	logger.Infof("resetting mysqld read-only to %v", readOnly)
	if err := mysqld.SetReadOnly(readOnly); err != nil {
		return usable, err
	}

	return usable, backupErr
}
994128
func (bh *GCSBackupHandle) EndBackup(ctx context.Context) error {
	if bh.readOnly {
		return fmt.Errorf("EndBackup cannot be called on read-only backup")
	}
	return nil
}
573946
func (client *Client) Backup(ctx context.Context, tablet *topodatapb.Tablet, concurrency int, allowMaster bool) (logutil.EventStream, error) {
	cc, c, err := client.dial(tablet)
	if err != nil {
		return nil, err
	}

	stream, err := c.Backup(ctx, &tabletmanagerdatapb.BackupRequest{
		Concurrency: int64(concurrency),
		AllowMaster: bool(allowMaster),
	})
	if err != nil {
		cc.Close()
		return nil, err
	}
	return &backupStreamAdapter{
		stream: stream,
		cc:     cc,
	}, nil
}
1337491
func (r *BackupFileReader) Close() error {
	if r.ctx != 0 {
		backupRead(syscall.Handle(r.f.Fd()), nil, nil, true, false, &r.ctx)
		runtime.KeepAlive(r.f)
		r.ctx = 0
	}
	return nil
}
405285
func (s *GlobalStore) Export(w io.Writer) (n int, err error) {
	s.mu.RLock()
	defer s.mu.RUnlock()

	tw := tar.NewWriter(w)
	defer tw.Close()

	buf := bytes.NewBuffer(make([]byte, 0, 1024))
	encoder := json.NewEncoder(buf)
	for key, addrs := range s.keyNodes {
		buf.Reset()
		if err = encoder.Encode(mock.ExportedChunk{
			Addrs: addrs,
			Data:  s.data[key],
		}); err != nil {
			return n, err
		}

		data := buf.Bytes()
		hdr := &tar.Header{
			Name: common.Bytes2Hex([]byte(key)),
			Mode: 0644,
			Size: int64(len(data)),
		}
		if err := tw.WriteHeader(hdr); err != nil {
			return n, err
		}
		if _, err := tw.Write(data); err != nil {
			return n, err
		}
		n++
	}
	return n, err
}
277187
func execBackup(ctx context.Context, wr *wrangler.Wrangler, tablet *topodatapb.Tablet, concurrency int, allowMaster bool) error {
	stream, err := wr.TabletManagerClient().Backup(ctx, tablet, concurrency, allowMaster)
	if err != nil {
		return err
	}
	for {
		e, err := stream.Recv()
		switch err {
		case nil:
			logutil.LogEvent(wr.Logger(), e)
		case io.EOF:
			return nil
		default:
			return err
		}
	}
}