// LeaseArchiveTasks leases archive tasks to the requestor from a pull queue.
490159
func (b *server) LeaseArchiveTasks(c context.Context, req *logdog.LeaseRequest) (*logdog.LeaseResponse, error) {
	duration, err := ptypes.Duration(req.LeaseTime)
	if err != nil {
		return nil, err
	}

	logging.Infof(c, "got request to lease %d tasks for %s", req.MaxTasks, req.GetLeaseTime())
	tasks, err := taskqueue.Lease(c, int(req.MaxTasks), ArchiveQueueName, duration)
	if err != nil {
		logging.WithError(err).Errorf(c, "could not lease %d tasks from queue", req.MaxTasks)
		return nil, err
	}
	archiveTasks := make([]*logdog.ArchiveTask, 0, len(tasks))
	for _, task := range tasks {
		at, err := archiveTask(task)
		if err != nil {
			// Ignore malformed name errors, just log them.
			logging.WithError(err).Errorf(c, "while leasing task")
			continue
		}
		// Optimization: Delete the task if it's already archived.
		if isArchived(c, at) {
			if err := taskqueue.Delete(c, ArchiveQueueName, task); err != nil {
				logging.WithError(err).Errorf(c, "failed to delete %s/%s (%s)", at.Project, at.Id, task.Name)
			}
			continue
		}
		archiveTasks = append(archiveTasks, at)
		leaseTask.Add(c, 1, task.RetryCount)
	}
	logging.Infof(c, "Leasing %d tasks", len(archiveTasks))
	return &logdog.LeaseResponse{Tasks: archiveTasks}, nil
}
1049384
func (h *ArchiveHandler) newArchiveFromGuest(u *url.URL) (File, error) {
	r, w := io.Pipe()

	a := &archive{
		name:   u.Path,
		done:   r.Close,
		Reader: r,
		Writer: w,
	}

	var z io.Writer = w
	var c io.Closer = ioutil.NopCloser(nil)

	switch u.Query().Get("format") {
	case "tgz":
		gz := gzip.NewWriter(w)
		z = gz
		c = gz
	}

	tw := tar.NewWriter(z)

	go func() {
		err := h.Write(u, tw)

		_ = tw.Close()
		_ = c.Close()
		if gzipTrailer {
			_, _ = w.Write(gzipHeader)
		}
		_ = w.CloseWithError(err)
	}()

	return a, nil
}
198539
func isArchived(c context.Context, task *logdog.ArchiveTask) bool {
	var err error
	defer func() {
		if err != nil {
			logging.WithError(err).Errorf(c, "while checking if %s/%s is archived", task.Project, task.Id)
		}
	}()
	if c, err = info.Namespace(c, "luci."+task.Project); err != nil {
		return false
	}
	state := (&coordinator.LogStream{ID: coordinator.HashID(task.Id)}).State(c)
	if err = datastore.Get(c, state); err != nil {
		return false
	}
	return state.ArchivalState() == coordinator.ArchivedComplete
}
26073
func Lease(w http.ResponseWriter,
	req *api.CreateLeaseReq,
	clusterLister ClusterLister,
	services k8s.ServiceGetterUpdater,
	azureConfig *config.Azure,
	k8sServiceName string) {

	clusterMap, svc, err := getSvcsAndClusters(clusterLister, services, k8sServiceName)
	if err != nil {
		log.Printf("Error listing Azure clusters or talking to the k8s API -- %s", err)
		htp.Error(w, http.StatusInternalServerError, "Error listing Azure clusters or talking to the k8s API -- %s", err)
		return
	}

	leaseMap, err := leases.ParseMapFromAnnotations(svc.Annotations)
	if err != nil {
		log.Printf("Error parsing leases from Kubernetes annotations -- %s", err)
		htp.Error(w, http.StatusInternalServerError, "error parsing leases from Kubernetes annotations -- %s", err)
		return
	}

	availableCluster, err := searchForFreeCluster(clusterMap, leaseMap, req.ClusterRegex, req.ClusterVersion)
	if err != nil {
		switch e := err.(type) {
		case errNoAvailableOrExpiredClustersFound:
			log.Printf("No available clusters found")
			htp.Error(w, http.StatusConflict, "No available clusters found")
			return
		case errExpiredLeaseAzureMissing:
			log.Printf("Cluster %s has an expired lease but doesn't exist in Azure", e.clusterName)
			htp.Error(w, http.StatusInternalServerError, "Cluster %s has an expired lease but doesn't exist in Azure", e.clusterName)
			return
		default:
			log.Printf("Unknown error %s", e.Error())
			htp.Error(w, http.StatusInternalServerError, "Unknown error %s", e.Error())
			return
		}
	}

	// There is currently no way to fetch the kubeconfig from the Azure API
	// So we must scp the file off the master node
	kubeConfig, err := FetchKubeConfig(*availableCluster.MasterProfile.Fqdn)
	if err != nil {
		log.Printf("Error creating kubeconfig file for cluster %s -- %s", *availableCluster.Name, err)
		htp.Error(w, http.StatusInternalServerError, "Error creating kubeconfig file for cluster %s -- %s", *availableCluster.Name, err)
		return
	}
	newToken := uuid.NewUUID()
	kubeConfigStr, err := k8s.MarshalAndEncodeKubeConfig(kubeConfig)
	if err != nil {
		log.Printf("Error marshaling & encoding kubeconfig -- %s", err)
		htp.Error(w, http.StatusInternalServerError, "Error marshaling & encoding kubeconfig -- %s", err)
		return
	}

	resp := api.CreateLeaseResp{
		KubeConfigStr: kubeConfigStr,
		IP:            *availableCluster.MasterProfile.Fqdn,
		Token:         newToken.String(),
		ClusterName:   *availableCluster.Name,
	}

	leaseMap.CreateLease(newToken, leases.NewLease(*availableCluster.Name, req.ExpirationTime(time.Now())))
	if err := k8s.SaveAnnotations(services, svc, leaseMap); err != nil {
		log.Printf("Error saving new lease to Kubernetes annotations -- %s", err)
		htp.Error(w, http.StatusInternalServerError, "Error saving new lease to Kubernetes annotations -- %s", err)
		return
	}

	if err := json.NewEncoder(w).Encode(resp); err != nil {
		log.Printf("Error encoding json -- %s", err)
		htp.Error(w, http.StatusInternalServerError, "Error encoding json -- %s", err)
		return
	}
}
1103842
func (s *Store) Leases(keys ...lease.Key) map[lease.Key]lease.Info {
	leaseMap := s.fsm.Leases(s.config.Clock.Now, keys...)
	// Add trapdoors into the information from the FSM.
	for k, v := range leaseMap {
		v.Trapdoor = s.config.Trapdoor(k, v.Holder)
		leaseMap[k] = v
	}
	return leaseMap
}
942654
func GetArchive(repo, ref string, format ArchiveFormat) ([]byte, error) {
	return retriever().GetArchive(repo, ref, format)
}
821889
func (opc *operationCallbacks) GetArchiveInfo(charmURL *corecharm.URL) (charm.BundleInfo, error) {
	ch, err := opc.u.st.Charm(charmURL)
	if err != nil {
		return nil, errors.Trace(err)
	}
	return ch, nil
}
836629
func archiveTask(task *taskqueue.Task) (*logdog.ArchiveTask, error) {
	result := logdog.ArchiveTask{}
	err := proto.Unmarshal(task.Payload, &result)
	result.TaskName = task.Name
	return &result, err
}
704080
func (sa *StorageAccessor) StorageAttachment(storageTag names.StorageTag, unitTag names.UnitTag) (params.StorageAttachment, error) {
	if sa.facade.BestAPIVersion() < 2 {
		return params.StorageAttachment{}, errors.NotImplementedf("StorageAttachment() (need V2+)")
	}
	args := params.StorageAttachmentIds{
		Ids: []params.StorageAttachmentId{{
			StorageTag: storageTag.String(),
			UnitTag:    unitTag.String(),
		}},
	}
	var results params.StorageAttachmentResults
	err := sa.facade.FacadeCall("StorageAttachments", args, &results)
	if err != nil {
		return params.StorageAttachment{}, errors.Trace(err)
	}
	if len(results.Results) != 1 {
		return params.StorageAttachment{}, errors.Errorf("expected 1 result, got %d", len(results.Results))
	}
	result := results.Results[0]
	if result.Error != nil {
		return params.StorageAttachment{}, result.Error
	}
	return result.Result, nil
}
802701
func (h *ArchiveHandler) newArchiveToGuest(u *url.URL) (File, error) {
	r, w := io.Pipe()

	buf := bufio.NewReader(r)

	a := &archive{
		name:   u.Path,
		Reader: buf,
		Writer: w,
	}

	var cerr error
	var wg sync.WaitGroup

	a.done = func() error {
		_ = w.Close()
		// We need to wait for unpack to finish to complete its work
		// and to propagate the error if any to Close.
		wg.Wait()
		return cerr
	}

	wg.Add(1)
	go func() {
		defer wg.Done()

		c := func() error {
			// Drain the pipe of tar trailer data (two null blocks)
			if cerr == nil {
				_, _ = io.Copy(ioutil.Discard, a.Reader)
			}
			return nil
		}

		header, _ := buf.Peek(len(gzipHeader))

		if bytes.Equal(header, gzipHeader) {
			gz, err := gzip.NewReader(a.Reader)
			if err != nil {
				_ = r.CloseWithError(err)
				cerr = err
				return
			}

			c = gz.Close
			a.Reader = gz
		}

		tr := tar.NewReader(a.Reader)

		cerr = h.Read(u, tr)

		_ = c()
		_ = r.CloseWithError(cerr)
	}()

	return a, nil
}
1135792
func (_class VMSSClass) GetRetainedSnapshots(sessionID SessionRef, self VMSSRef) (_retval int, _err error) {
	_method := "VMSS.get_retained_snapshots"
	_sessionIDArg, _err := convertSessionRefToXen(fmt.Sprintf("%s(%s)", _method, "session_id"), sessionID)
	if _err != nil {
		return
	}
	_selfArg, _err := convertVMSSRefToXen(fmt.Sprintf("%s(%s)", _method, "self"), self)
	if _err != nil {
		return
	}
	_result, _err := _class.client.APICall(_method, _sessionIDArg, _selfArg)
	if _err != nil {
		return
	}
	_retval, _err = convertIntToGo(_method + " -> ", _result.Value)
	return
}
484461
func (k *kubeletClient) CreateArchive(containerID, sourcePath string) (*bytes.Buffer, error) {
	return k.getCommandOutput(containerID, fmt.Sprintf("command=tar&command=-cf&command=-&command=%s&output=1", sourcePath))
}
1086727
func (s *StorageAPI) StorageAttachments(args params.StorageAttachmentIds) (params.StorageAttachmentResults, error) {
	canAccess, err := s.accessUnit()
	if err != nil {
		return params.StorageAttachmentResults{}, err
	}
	result := params.StorageAttachmentResults{
		Results: make([]params.StorageAttachmentResult, len(args.Ids)),
	}
	for i, id := range args.Ids {
		storageAttachment, err := s.getOneStorageAttachment(canAccess, id)
		if err == nil {
			result.Results[i].Result = storageAttachment
		}
		result.Results[i].Error = common.ServerError(err)
	}
	return result, nil
}
1247247
func (s *storageImpl) registerTasks() {
	// See queue.yaml for "cas-uploads" task queue definition.
	s.tq.RegisterTask(&tasks.VerifyUpload{}, func(c context.Context, m proto.Message) error {
		return s.verifyUploadTask(c, m.(*tasks.VerifyUpload))
	}, "cas-uploads", nil)
	s.tq.RegisterTask(&tasks.CleanupUpload{}, func(c context.Context, m proto.Message) error {
		return s.cleanupUploadTask(c, m.(*tasks.CleanupUpload))
	}, "cas-uploads", nil)
}
672974
func (r *Repository) GetArchiveURL() string {
	if r == nil || r.ArchiveURL == nil {
		return ""
	}
	return *r.ArchiveURL
}
1138487
func (p *Policy) handleArchiving(ctx context.Context, storage logical.Storage) error {
	// We need to move keys that are no longer accessible to archivedKeys, and keys
	// that now need to be accessible back here.
	//
	// For safety, because there isn't really a good reason to, we never delete
	// keys from the archive even when we move them back.

	// Check if we have the latest minimum version in the current set of keys
	_, keysContainsMinimum := p.Keys[strconv.Itoa(p.MinDecryptionVersion)]

	// Sanity checks
	switch {
	case p.MinDecryptionVersion < 1:
		return fmt.Errorf("minimum decryption version of %d is less than 1", p.MinDecryptionVersion)
	case p.LatestVersion < 1:
		return fmt.Errorf("latest version of %d is less than 1", p.LatestVersion)
	case !keysContainsMinimum && p.ArchiveVersion != p.LatestVersion:
		return fmt.Errorf("need to move keys from archive but archive version not up-to-date")
	case p.ArchiveVersion > p.LatestVersion:
		return fmt.Errorf("archive version of %d is greater than the latest version %d",
			p.ArchiveVersion, p.LatestVersion)
	case p.MinEncryptionVersion > 0 && p.MinEncryptionVersion < p.MinDecryptionVersion:
		return fmt.Errorf("minimum decryption version of %d is greater than minimum encryption version %d",
			p.MinDecryptionVersion, p.MinEncryptionVersion)
	case p.MinDecryptionVersion > p.LatestVersion:
		return fmt.Errorf("minimum decryption version of %d is greater than the latest version %d",
			p.MinDecryptionVersion, p.LatestVersion)
	}

	archive, err := p.LoadArchive(ctx, storage)
	if err != nil {
		return err
	}

	if !keysContainsMinimum {
		// Need to move keys *from* archive
		for i := p.MinDecryptionVersion; i <= p.LatestVersion; i++ {
			p.Keys[strconv.Itoa(i)] = archive.Keys[i-p.MinAvailableVersion]
		}

		return nil
	}

	// Need to move keys *to* archive

	// We need a size that is equivalent to the latest version (number of keys)
	// but adding one since slice numbering starts at 0 and we're indexing by
	// key version
	if len(archive.Keys)+p.MinAvailableVersion < p.LatestVersion+1 {
		// Increase the size of the archive slice
		newKeys := make([]KeyEntry, p.LatestVersion-p.MinAvailableVersion+1)
		copy(newKeys, archive.Keys)
		archive.Keys = newKeys
	}

	// We are storing all keys in the archive, so we ensure that it is up to
	// date up to p.LatestVersion
	for i := p.ArchiveVersion + 1; i <= p.LatestVersion; i++ {
		archive.Keys[i-p.MinAvailableVersion] = p.Keys[strconv.Itoa(i)]
		p.ArchiveVersion = i
	}

	// Trim the keys if required
	if p.ArchiveMinVersion < p.MinAvailableVersion {
		archive.Keys = archive.Keys[p.MinAvailableVersion-p.ArchiveMinVersion:]
		p.ArchiveMinVersion = p.MinAvailableVersion
	}

	err = p.storeArchive(ctx, storage, archive)
	if err != nil {
		return err
	}

	// Perform deletion afterwards so that if there is an error saving we
	// haven't messed with the current policy
	for i := p.LatestVersion - len(p.Keys) + 1; i < p.MinDecryptionVersion; i++ {
		delete(p.Keys, strconv.Itoa(i))
	}

	return nil
}
1318859
func NewArchiver(idMappings *idtools.IDMappings) *Archiver {
	if idMappings == nil {
		idMappings = &idtools.IDMappings{}
	}
	return &Archiver{Untar: Untar, TarIDMappings: idMappings, UntarIDMappings: idMappings}
}