// Get logs from the systemd journal. Currently we don't use this for most of
// our logging, since it's not persisted across boot on some systems. But we do
// use it for startup logs.
338845
func tailSystemdJournal(log logger.Logger, userUnits []string, numBytes int) (ret string) {
	log.Debug("+ tailing journalctl for %#v (%d bytes)", userUnits, numBytes)
	defer func() {
		log.Debug("- scanned %d bytes", len(ret))
	}()

	// Journalctl doesn't provide a "last N bytes" flag directly, so instead we
	// use "last N lines". Large log files in practice seem to be about 150
	// bits per line. We'll request lines on that assumption, but if we get
	// more than 2x as many bytes as we expected, we'll stop reading and
	// include a big error.
	guessedLines := numBytes / 150
	maxBytes := numBytes * 2

	// We intentionally avoid the --user flag to journalctl. That would make us
	// skip over the system journal, but in e.g. Ubuntu 16.04, that's where
	// user units write their logs.
	// Unfortunately, this causes permission errors in some operating systems
	// like Debian Stretch, but it is not fatal. as we ignore errors in this
	// function.
	args := []string{
		"--lines=" + strconv.Itoa(guessedLines),
	}
	if len(userUnits) == 0 {
		panic("without --user-unit we would scrape all system logs!!!")
	}
	for _, userUnit := range userUnits {
		args = append(args, "--user-unit="+userUnit)
	}
	journalCmd := exec.Command("journalctl", args...)
	journalCmd.Stderr = os.Stderr
	stdout, err := journalCmd.StdoutPipe()
	if err != nil {
		msg := fmt.Sprintf("Failed to open a pipe for journalctl: %s", err)
		log.Errorf(msg)
		return msg
	}
	err = journalCmd.Start()
	if err != nil {
		msg := fmt.Sprintf("Failed to run journalctl: %s", err)
		log.Errorf(msg)
		return msg
	}

	// Once we start reading output, don't short-circuit on errors. Just log
	// them, and return whatever we got.
	stdoutLimited := io.LimitReader(stdout, int64(maxBytes))
	output, err := ioutil.ReadAll(stdoutLimited)
	if err != nil {
		output = appendError(log, output, "Error reading from journalctl pipe: %s", err)
	}
	// We must close stdout before Wait, or else Wait might deadlock.
	stdout.Close()
	err = journalCmd.Wait()
	if err != nil {
		output = appendError(log, output, "Journalctl exited with an error: %s", err)
	}
	if len(output) >= maxBytes {
		output = appendError(log, output, "Journal lines longer than expected. Logs truncated.")
	}
	return string(output)
}
1154685
func (m *MemLogger) Get(lvl logging.Level, msg string, data map[string]interface{}) *LogEntry {
	return m.GetFunc(func(ent *LogEntry) bool {
		return ent.Level == lvl && ent.Msg == msg && reflect.DeepEqual(data, ent.Data)
	})
}
594305
func (api *API) GetUserAuditLogs(a AuditLogFilter) (AuditLogResponse, error) {
	uri := "/user/audit_logs" + fmt.Sprintf("%s", a)
	res, err := api.makeRequest("GET", uri, nil)
	if err != nil {
		return AuditLogResponse{}, err
	}
	return unmarshalReturn(res)
}
70930
func (ls *LogsService) Get(logId uint) (*Log, *http.Response, error) {
	u, err := urlWithOptions(fmt.Sprintf("/logs/%d", logId), nil)
	if err != nil {
		return nil, nil, err
	}

	req, err := ls.client.NewRequest("GET", u, nil, nil)
	if err != nil {
		return nil, nil, err
	}

	var logResp getLogResponse
	resp, err := ls.client.Do(req, &logResp)
	if err != nil {
		return nil, resp, err
	}

	return &logResp.Log, resp, err
}
333301
func GetAllLogs(log string) ([]string, error) {
	// pattern is used to match all rotated files.
	pattern := fmt.Sprintf("%s.*", log)
	logs, err := filepath.Glob(pattern)
	if err != nil {
		return nil, fmt.Errorf("failed to list all log files with pattern %q: %v", pattern, err)
	}
	inuse, _ := filterUnusedLogs(logs)
	sort.Strings(inuse)
	return append(inuse, log), nil
}
725331
func getLogEntry(req *http.Request) *logrus.Entry {
	return req.Context().Value(logCtxKey).(*logrus.Entry)
}
865718
func (t TaskService) FindLogs(ctx context.Context, filter platform.LogFilter) ([]*platform.Log, int, error) {
	if filter.Task == nil {
		return nil, 0, errors.New("task ID required")
	}

	var urlPath string
	if filter.Run == nil {
		urlPath = path.Join(taskIDPath(*filter.Task), "logs")
	} else {
		urlPath = path.Join(taskIDRunIDPath(*filter.Task, *filter.Run), "logs")
	}

	u, err := newURL(t.Addr, urlPath)
	if err != nil {
		return nil, 0, err
	}
	val := url.Values{}
	if filter.Org != nil {
		val.Set("orgID", filter.Org.String())
	}
	u.RawQuery = val.Encode()

	req, err := http.NewRequest("GET", u.String(), nil)
	if err != nil {
		return nil, 0, err
	}
	SetToken(t.Token, req)

	hc := newClient(u.Scheme, t.InsecureSkipVerify)

	resp, err := hc.Do(req)
	if err != nil {
		return nil, 0, err
	}
	defer resp.Body.Close()

	if err := CheckError(resp); err != nil {
		return nil, 0, err
	}

	var logs []*platform.Log
	if err := json.NewDecoder(resp.Body).Decode(&logs); err != nil {
		return nil, 0, err
	}

	return logs, len(logs), nil
}
563031
func (as *AnalyticalStorage) FindLogs(ctx context.Context, filter influxdb.LogFilter) ([]*influxdb.Log, int, error) {
	var logs []*influxdb.Log
	if filter.Run != nil {
		run, err := as.FindRunByID(ctx, filter.Task, *filter.Run)
		if err != nil {
			return nil, 0, err
		}
		for i := 0; i < len(run.Log); i++ {
			logs = append(logs, &run.Log[i])
		}
		return logs, len(logs), nil
	}

	// add historical logs to the transactional logs.
	runs, n, err := as.FindRuns(ctx, influxdb.RunFilter{Task: filter.Task})
	if err != nil {
		return nil, 0, err
	}

	for _, run := range runs {
		for i := 0; i < len(run.Log); i++ {
			logs = append(logs, &run.Log[i])
		}
	}

	return logs, n, err
}
1083040
func (m *kubeGenericRuntimeManager) GetContainerLogs(ctx context.Context, pod *v1.Pod, containerID kubecontainer.ContainerID, logOptions *v1.PodLogOptions, stdout, stderr io.Writer) (err error) {
	status, err := m.runtimeService.ContainerStatus(containerID.ID)
	if err != nil {
		klog.V(4).Infof("failed to get container status for %v: %v", containerID.String(), err)
		return fmt.Errorf("Unable to retrieve container logs for %v", containerID.String())
	}
	return m.ReadLogs(ctx, status.GetLogPath(), containerID.ID, logOptions, stdout, stderr)
}
659209
func GetLogs(bee string) []LogMessage {
	r := []LogMessage{}

	logMutex.RLock()
	for b, ls := range logs {
		if len(bee) == 0 || bee == b {
			for _, l := range ls {
				r = append(r, l)
			}
		}
	}
	logMutex.RUnlock()

	sort.Sort(LogSorter(r))
	return r
}
1211457
func InitLog(params InitParams, ctx Context) (logger.Logger, error) {
	return InitLogWithPrefix(params, ctx, "kbfs", defaultLogPath(ctx))
}
1078244
func (logger *Logger) InitStdSyslog() error {
	syslogHandler, err := syslog.New("", "", 0, "")
	if err != nil {
		return err
	}
	logger.AddHandler("syslog", syslogHandler)

	return nil
}
728432
func (exporter *logExporter) retrieveLogs() (foundIndexedDay bool, numWarnings int, e error) {
	numWarnings = 0
	foundIndexedDay = false
	// fileIndex is a map of parsedMessage to an index into exporter.outputFiles
	exporter.fileIndex = NewFileIndex(exporter.GroupBy)
	for _, yyyymmdd := range exporter.days {
		// Skip the indexes that are filtered out by the date range
		if (exporter.FromDate != "" && yyyymmdd < exporter.FromDate) || (exporter.ToDate != "" && yyyymmdd > exporter.ToDate) {
			continue
		} else {
			foundIndexedDay = true
		}

		if exporter.Debug {
			log.WithFields(logrus.Fields{
				"date": yyyymmdd,
			}).Info("Querying logstash for given date")
		}
		log := log.WithFields(logrus.Fields{
			"date": yyyymmdd,
		})
		log.Info("Retrieving Logstash Elastic results for one day")

		result, e := exporter.Driver.StartSearch(yyyymmdd, exporter.query)
		if e != nil {
			return foundIndexedDay, numWarnings, fmt.Errorf("failed to search elasticsearch for day %s: %s", yyyymmdd, e)
		}

		if exporter.Debug {
			log.WithFields(logrus.Fields{
				"date":      yyyymmdd,
				"totalmsgs": result.Hits.Total,
			}).Info("Found log messages")
		}

		//TODO: Submit a patch to elastigo to support the "clear scroll" api. Add a "defer" here.
		remaining := result.Hits.Total > 0
		for remaining {
			result, e = exporter.Driver.ScrollSearch(result.ScrollId)
			if e != nil {
				return foundIndexedDay, numWarnings, e
			}
			hits := result.Hits.Hits
			total := len(hits)
			for i := 0; i < total; i++ {
				message, e := parseLogSource(yyyymmdd, hits[i].Source)
				if e != nil {
					return foundIndexedDay, numWarnings, e
				}

				// Ignore log messages sent directly from serviced and serviced-controller; only
				// export logs from the applications themselves.  Note that filebeat is hard-coded
				// to set the _type property to "log" and we only use filebeat for messages from the
				// application services.
				if message.Type != "log" {
					continue
				}

				if len(exporter.minDateFound) == 0 || yyyymmdd < exporter.minDateFound {
					exporter.minDateFound = yyyymmdd
				}
				if len(exporter.maxDateFound) == 0 || yyyymmdd > exporter.maxDateFound {
					exporter.maxDateFound = yyyymmdd
				}

				// add a new tempfile
				if _, found := exporter.fileIndex.FindIndexForMessage(message); !found {
					index := len(exporter.outputFiles)
					filename := filepath.Join(exporter.tempdir, exporter.fileIndex.GetFileName(index, message))
					file, e := os.Create(filename)
					if e != nil {
						return foundIndexedDay, numWarnings, fmt.Errorf("failed to create file %s: %s", filename, e)
					}
					exporter.fileIndex.AddIndexForMessage(index, message)
					outputFile := outputFileInfo{
						File:        file,
						Name:        filename,
						HostID:      message.HostID,
						ContainerID: message.ContainerID,
						LogFileName: message.LogFileName,
						ServiceID:   message.ServiceID,
					}
					exporter.outputFiles = append(exporter.outputFiles, outputFile)
				}
				index, _ := exporter.fileIndex.FindIndexForMessage(message)
				exporter.outputFiles[index].LineCount += len(message.Lines)
				file := exporter.outputFiles[index].File
				filename := exporter.outputFiles[index].Name
				for _, line := range message.Lines {
					formatted := fmt.Sprintf("%016x\t%016x\t%s\n", line.Timestamp, line.Offset, line.Message)
					if _, e := file.WriteString(formatted); e != nil {
						return foundIndexedDay, numWarnings, fmt.Errorf("failed writing to file %s: %s", filename, e)
					}
				}
				if len(message.Warnings) > 0 {
					if _, e := exporter.parseWarningsFile.WriteString(message.Warnings); e != nil {
						return foundIndexedDay, numWarnings, fmt.Errorf("failed writing to file %s: %s", exporter.parseWarningsFilename, e)
					}
					numWarnings++
				}
			}
			remaining = len(hits) > 0
		}
	}
	return foundIndexedDay, numWarnings, nil
}
412829
func (s *UserOperationLogService) GetUserOperationLog(ctx context.Context, id platform.ID, opts platform.FindOptions) ([]*platform.OperationLogEntry, int, error) {
	return s.GetUserOperationLogFn(ctx, id, opts)
}
999320
func (s *server) LogEntries() []*LogEntry {
	s.log.mutex.RLock()
	defer s.log.mutex.RUnlock()
	return s.log.entries
}
349884
func (s *Server) getContainerLogs(request *restful.Request, response *restful.Response) {
	podNamespace := request.PathParameter("podNamespace")
	podID := request.PathParameter("podID")
	containerName := request.PathParameter("containerName")
	ctx := request.Request.Context()

	if len(podID) == 0 {
		// TODO: Why return JSON when the rest return plaintext errors?
		// TODO: Why return plaintext errors?
		response.WriteError(http.StatusBadRequest, fmt.Errorf(`{"message": "Missing podID."}`))
		return
	}
	if len(containerName) == 0 {
		// TODO: Why return JSON when the rest return plaintext errors?
		response.WriteError(http.StatusBadRequest, fmt.Errorf(`{"message": "Missing container name."}`))
		return
	}
	if len(podNamespace) == 0 {
		// TODO: Why return JSON when the rest return plaintext errors?
		response.WriteError(http.StatusBadRequest, fmt.Errorf(`{"message": "Missing podNamespace."}`))
		return
	}

	query := request.Request.URL.Query()
	// backwards compatibility for the "tail" query parameter
	if tail := request.QueryParameter("tail"); len(tail) > 0 {
		query["tailLines"] = []string{tail}
		// "all" is the same as omitting tail
		if tail == "all" {
			delete(query, "tailLines")
		}
	}
	// container logs on the kubelet are locked to the v1 API version of PodLogOptions
	logOptions := &v1.PodLogOptions{}
	if err := legacyscheme.ParameterCodec.DecodeParameters(query, v1.SchemeGroupVersion, logOptions); err != nil {
		response.WriteError(http.StatusBadRequest, fmt.Errorf(`{"message": "Unable to decode query."}`))
		return
	}
	logOptions.TypeMeta = metav1.TypeMeta{}
	if errs := validation.ValidatePodLogOptions(logOptions); len(errs) > 0 {
		response.WriteError(http.StatusUnprocessableEntity, fmt.Errorf(`{"message": "Invalid request."}`))
		return
	}

	pod, ok := s.host.GetPodByName(podNamespace, podID)
	if !ok {
		response.WriteError(http.StatusNotFound, fmt.Errorf("pod %q does not exist", podID))
		return
	}
	// Check if containerName is valid.
	containerExists := false
	for _, container := range pod.Spec.Containers {
		if container.Name == containerName {
			containerExists = true
			break
		}
	}
	if !containerExists {
		for _, container := range pod.Spec.InitContainers {
			if container.Name == containerName {
				containerExists = true
				break
			}
		}
	}
	if !containerExists {
		response.WriteError(http.StatusNotFound, fmt.Errorf("container %q not found in pod %q", containerName, podID))
		return
	}

	if _, ok := response.ResponseWriter.(http.Flusher); !ok {
		response.WriteError(http.StatusInternalServerError, fmt.Errorf("unable to convert %v into http.Flusher, cannot show logs", reflect.TypeOf(response)))
		return
	}
	fw := flushwriter.Wrap(response.ResponseWriter)
	response.Header().Set("Transfer-Encoding", "chunked")
	if err := s.host.GetKubeletContainerLogs(ctx, kubecontainer.GetPodFullName(pod), containerName, logOptions, fw, fw); err != nil {
		response.WriteError(http.StatusBadRequest, err)
		return
	}
}
976428
func (c *ServicedCli) initLog() {
	c.app.Commands = append(c.app.Commands, cli.Command{
		Name:        "log",
		Usage:       "Administers logs",
		Description: "",
		Subcommands: []cli.Command{
			{
				Name:        "export",
				Usage:       "Exports application log data",
				Description: "serviced log export",
				// TODO: BashComplete: c.printLogExportCompletion,
				Action: c.cmdExportLogs,
				Flags: []cli.Flag{
					cli.StringFlag{
						Name:  "from",
						Value: "",
						Usage: "yyyy.mm.dd",
					},
					cli.StringFlag{
						Name:  "to",
						Value: "",
						Usage: "yyyy.mm.dd",
					},
					cli.StringSliceFlag{
						Name:  "service",
						Value: &cli.StringSlice{},
						Usage: "service ID or name (includes all child services)",
					},
					cli.StringSliceFlag{
						Name:  "file",
						Value: &cli.StringSlice{},
						Usage: "the application log filename",
					},
					cli.StringFlag{
						Name:  "out",
						Value: "",
						Usage: "path to output file",
					},
					cli.BoolFlag{
						Name:  "debug, d",
						Usage: "Show additional diagnostic messages",
					},
					cli.StringFlag{
						Name:  "group-by",
						Value: "container",
						Usage: "Group results either by container, service or day",
					},
					cli.BoolFlag{
						Name:  "no-children, n",
						Usage: "Do not export child services",
					},
				},
			},
		},
	})
}