diff --git a/host/host.go b/host/host.go index 4b6a9d44..b6c2a4fc 100644 --- a/host/host.go +++ b/host/host.go @@ -48,6 +48,7 @@ type Frame struct { type Trace struct { Comm string + Executable string Frames []Frame Hash TraceHash KTime times.KTime diff --git a/processmanager/processinfo.go b/processmanager/processinfo.go index 0a374b05..97435ef8 100644 --- a/processmanager/processinfo.go +++ b/processmanager/processinfo.go @@ -82,7 +82,9 @@ func (pm *ProcessManager) updatePidInformation(pid libpf.PID, m *Mapping) (bool, if !ok { // We don't have information for this pid, so we first need to // allocate the embedded map for this process. + executable, _ := os.Readlink(fmt.Sprintf("/proc/%d/exe", pid)) info = &processInfo{ + executable: executable, mappings: make(map[libpf.Address]*Mapping), mappingsByFileID: make(map[host.FileID]map[libpf.Address]*Mapping), tsdInfo: nil, @@ -638,3 +640,17 @@ func (pm *ProcessManager) CleanupPIDs() { log.Debugf("Cleaned up %d dead PIDs", len(deadPids)) } } + +// ExePathForPID returns the full executable path for given PID. +// If the PID is not tracked or belongs to a kernel worker, +// it returns the empty string. +func (pm *ProcessManager) ExePathForPID(pid libpf.PID) string { + var executable string + + pm.mu.RLock() + defer pm.mu.RUnlock() + if procInfo, ok := pm.pidToProcessInfo[pid]; ok { + executable = procInfo.executable + } + return executable +} diff --git a/processmanager/types.go b/processmanager/types.go index d890ab38..f65c032f 100644 --- a/processmanager/types.go +++ b/processmanager/types.go @@ -138,6 +138,8 @@ func (m *Mapping) GetOnDiskFileIdentifier() util.OnDiskFileIdentifier { // processInfo contains information about the executable mappings // and Thread Specific Data of a process. type processInfo struct { + // executable path retrieved from /proc/PID/exe + executable string // executable mappings keyed by start address. mappings map[libpf.Address]*Mapping // executable mappings keyed by host file ID. diff --git a/reporter/internal/pdata/generate.go b/reporter/internal/pdata/generate.go index 16a260b8..3352f26c 100644 --- a/reporter/internal/pdata/generate.go +++ b/reporter/internal/pdata/generate.go @@ -177,6 +177,8 @@ func (p *Pdata) setProfile( semconv.ContainerIDKey, traceKey.ContainerID) attrMgr.AppendOptionalString(sample.AttributeIndices(), semconv.ThreadNameKey, traceKey.Comm) + attrMgr.AppendOptionalString(sample.AttributeIndices(), + semconv.ProcessExecutablePathKey, traceKey.Executable) attrMgr.AppendOptionalString(sample.AttributeIndices(), semconv.ServiceNameKey, traceKey.ApmServiceName) attrMgr.AppendInt(sample.AttributeIndices(), diff --git a/reporter/internal/samples/samples.go b/reporter/internal/samples/samples.go index 53a8515b..80c6f64d 100644 --- a/reporter/internal/samples/samples.go +++ b/reporter/internal/samples/samples.go @@ -8,6 +8,7 @@ import "go.opentelemetry.io/ebpf-profiler/libpf" type TraceEventMeta struct { Timestamp libpf.UnixTime64 Comm string + Executable string APMServiceName string PID, TID libpf.PID CPU int @@ -35,6 +36,8 @@ type TraceAndMetaKey struct { // containerID is annotated based on PID information ContainerID string Pid int64 + // Executable path is retrieved from /proc/PID/exe + executable string // ExtraMeta stores extra meta info that may have been produced by a // `SampleAttrProducer` instance. May be nil. ExtraMeta any diff --git a/reporter/otlp_reporter.go b/reporter/otlp_reporter.go index f4c56e65..4ed59e2c 100644 --- a/reporter/otlp_reporter.go +++ b/reporter/otlp_reporter.go @@ -158,6 +158,7 @@ func (r *OTLPReporter) ReportTraceEvent(trace *libpf.Trace, meta *TraceEventMeta key := samples.TraceAndMetaKey{ Hash: trace.Hash, Comm: meta.Comm, + Executable: meta.Executable, ApmServiceName: meta.APMServiceName, ContainerID: containerID, Pid: int64(meta.PID), diff --git a/tools/coredump/README.md b/tools/coredump/README.md index 29ccdef8..efd9410e 100644 --- a/tools/coredump/README.md +++ b/tools/coredump/README.md @@ -108,11 +108,18 @@ with existing test cases. In this variant we essentially make the kernel think that the target application crashed, causing the kernel to save a coredump for us. -#### Setting the coredump filter +#### Setting the coredump filter (optional) Coredumps normally contain only the anonymous and modified pages to save disk -space. For our test cases, we want a full process memory dump that also contains -the pages mapped into the process from the ELF files. +space. This is sufficient if the mapped in ELF files are available to the +`coredump` utility to be bundled. This is the case if you run +`./coredump new -core core` on the same machine where the core was generated, +or if you supply `-sysroot` as a prefix to find the correct files. + +If the above is not possible, the testing infrastructure has limited support +to allow reading the ELF file data directly from the coredump. In this case +a full process memory dump that also contains the pages mapped into the process +from the ELF files is needed. To get a full process memory dump one has to set the [`coredump_filter`][filter] in advance by running: diff --git a/tools/coredump/new.go b/tools/coredump/new.go index af4a519b..56a34250 100644 --- a/tools/coredump/new.go +++ b/tools/coredump/new.go @@ -29,6 +29,7 @@ type newCmd struct { // User-specified command line arguments. coredumpPath string + sysroot string pid uint64 name string importThreadInfo string @@ -110,6 +111,7 @@ func newNewCmd(store *modulestore.Store) *ffcli.Command { set := flag.NewFlagSet("new", flag.ExitOnError) set.StringVar(&args.coredumpPath, "core", "", "Path of the coredump to import") + set.StringVar(&args.sysroot, "sysroot", "", "Path for the coredump associated ELF files") set.Uint64Var(&args.pid, "pid", 0, "PID to create a fresh coredump for") set.StringVar(&args.name, "name", "", "Name for the test case [required]") set.StringVar(&args.importThreadInfo, "import-thread-info", "", "If this flag is specified, "+ @@ -139,17 +141,19 @@ func (cmd *newCmd) exec(context.Context, []string) (err error) { } var corePath string - prefix := "" + prefix := cmd.sysroot if cmd.coredumpPath != "" { corePath = cmd.coredumpPath } else { // No path provided: create a new dump. - corePath, err = dumpCore(cmd.pid) + corePath, err = dumpCore(cmd.pid, cmd.noModuleBundling) if err != nil { return fmt.Errorf("failed to create coredump: %w", err) } defer os.Remove(corePath) - prefix = fmt.Sprintf("/proc/%d/root/", cmd.pid) + if prefix == "" { + prefix = fmt.Sprintf("/proc/%d/root/", cmd.pid) + } } core, err := newTrackedCoredump(corePath, prefix) @@ -195,32 +199,34 @@ func (cmd *newCmd) exec(context.Context, []string) (err error) { return nil } -func dumpCore(pid uint64) (string, error) { - // Backup current coredump filter mask. - // https://man7.org/linux/man-pages/man5/core.5.html - coredumpFilterPath := fmt.Sprintf("/proc/%d/coredump_filter", pid) - prevMask, err := os.ReadFile(coredumpFilterPath) - if err != nil { - return "", fmt.Errorf("failed to read coredump filter: %w", err) - } - // Adjust coredump filter mask. - //nolint:gosec - err = os.WriteFile(coredumpFilterPath, []byte("0x3f"), 0o644) - if err != nil { - return "", fmt.Errorf("failed to write coredump filter: %w", err) - } - // Restore coredump filter mask upon leaving the function. - defer func() { +func dumpCore(pid uint64, noModuleBundling bool) (string, error) { + if noModuleBundling { + // Backup current coredump filter mask. + // https://man7.org/linux/man-pages/man5/core.5.html + coredumpFilterPath := fmt.Sprintf("/proc/%d/coredump_filter", pid) + prevMask, err := os.ReadFile(coredumpFilterPath) + if err != nil { + return "", fmt.Errorf("failed to read coredump filter: %w", err) + } + // Adjust coredump filter mask. //nolint:gosec - err2 := os.WriteFile(coredumpFilterPath, prevMask, 0o644) - if err2 != nil { - log.Warnf("Failed to restore previous coredump filter: %v", err2) + err = os.WriteFile(coredumpFilterPath, []byte("0x3f"), 0o644) + if err != nil { + return "", fmt.Errorf("failed to write coredump filter: %w", err) } - }() + // Restore coredump filter mask upon leaving the function. + defer func() { + //nolint:gosec + err2 := os.WriteFile(coredumpFilterPath, append([]byte("0x"), prevMask...), 0o644) + if err2 != nil { + log.Warnf("Failed to restore previous coredump filter: %v", err2) + } + }() + } // `gcore` only accepts a path-prefix, not an exact path. //nolint:gosec - err = exec.Command("gcore", "-o", gcorePathPrefix, strconv.FormatUint(pid, 10)).Run() + err := exec.Command("gcore", "-o", gcorePathPrefix, strconv.FormatUint(pid, 10)).Run() if err != nil { return "", fmt.Errorf("gcore failed: %w", err) } diff --git a/tracehandler/tracehandler.go b/tracehandler/tracehandler.go index ebc39ac0..bb0d657c 100644 --- a/tracehandler/tracehandler.go +++ b/tracehandler/tracehandler.go @@ -127,6 +127,7 @@ func (m *traceHandler) HandleTrace(bpfTrace *host.Trace) { TID: bpfTrace.TID, APMServiceName: "", // filled in below CPU: bpfTrace.CPU, + Executable: bpfTrace.Executable, } if !m.reporter.SupportsReportTraceEvent() { diff --git a/tracer/tracer.go b/tracer/tracer.go index 78067738..64caeafd 100644 --- a/tracer/tracer.go +++ b/tracer/tracer.go @@ -856,11 +856,13 @@ func (t *Tracer) loadBpfTrace(raw []byte, cpu int) *host.Trace { panic("unexpected record size") } + pid := libpf.PID(ptr.pid) trace := &host.Trace{ Comm: C.GoString((*C.char)(unsafe.Pointer(&ptr.comm))), + Executable: t.processManager.ExePathForPID(pid), APMTraceID: *(*libpf.APMTraceID)(unsafe.Pointer(&ptr.apm_trace_id)), APMTransactionID: *(*libpf.APMTransactionID)(unsafe.Pointer(&ptr.apm_transaction_id)), - PID: libpf.PID(ptr.pid), + PID: pid, TID: libpf.PID(ptr.tid), KTime: times.KTime(ptr.ktime), CPU: cpu,