diff --git a/README.md b/README.md index 5f3c1bd1..2ab16aa7 100644 --- a/README.md +++ b/README.md @@ -476,7 +476,7 @@ The host agent code is tested with three test suites: tests. This works great for the user-land portion of the agent, but is unable to test any of the unwinding logic and BPF interaction. - **coredump test suite**\ - The coredump test suite (`utils/coredump`) we compile the whole BPF unwinder + The coredump test suite (`tools/coredump`) we compile the whole BPF unwinder code into a user-mode executable, then use the information from a coredump to simulate a realistic environment to test the unwinder code in. The coredump suite essentially implements all required BPF helper functions in user-space, diff --git a/tools/coredump/coredump.go b/tools/coredump/coredump.go index eff7b37c..c3506d79 100644 --- a/tools/coredump/coredump.go +++ b/tools/coredump/coredump.go @@ -10,6 +10,8 @@ import ( "fmt" "os" "runtime" + "strconv" + "strings" "time" "unsafe" @@ -114,16 +116,44 @@ func (c *symbolizationCache) symbolize(ty libpf.FrameType, fileID libpf.FileID, } if data, ok := c.symbols[libpf.NewFrameID(fileID, lineNumber)]; ok { - return fmt.Sprintf("%s+%d in %s:%d", - data.FunctionName, data.FunctionOffset, - data.SourceFile, data.SourceLine), nil + return formatSymbolizedFrame(data, true), nil } sourceFile, ok := c.files[fileID] if !ok { sourceFile = fmt.Sprintf("%08x", fileID) } - return fmt.Sprintf("%s+0x%x", sourceFile, lineNumber), nil + return formatUnsymbolizedFrame(sourceFile, lineNumber), nil +} + +func formatSymbolizedFrame(frame *reporter.FrameMetadataArgs, functionOffsets bool) string { + var funcOffset string + if functionOffsets { + funcOffset = "+" + strconv.Itoa(int(frame.FunctionOffset)) + } + return fmt.Sprintf("%s%s in %s:%d", + frame.FunctionName, funcOffset, + frame.SourceFile, frame.SourceLine) +} + +func formatUnsymbolizedFrame(file string, addr libpf.AddressOrLineno) string { + return fmt.Sprintf("%s+0x%x", file, addr) +} + +func parseUnsymbolizedFrame(frame string) (file string, addr libpf.AddressOrLineno, err error) { + fileS, addrS, found := strings.Cut(frame, "+0x") + if !found { + err = fmt.Errorf("bad frame string: %q", frame) + return + } + file = fileS + var addrU uint64 + addrU, err = strconv.ParseUint(addrS, 16, 64) + if err != nil { + return + } + addr = libpf.AddressOrLineno(addrU) + return } func ExtractTraces(ctx context.Context, pr process.Process, debug bool, diff --git a/tools/coredump/gosym.go b/tools/coredump/gosym.go new file mode 100644 index 00000000..110e1a8b --- /dev/null +++ b/tools/coredump/gosym.go @@ -0,0 +1,162 @@ +package main + +import ( + "context" + "debug/elf" + "debug/gosym" + "errors" + "flag" + "fmt" + "os" + "path/filepath" + + "github.com/peterbourgon/ff/v3/ffcli" + "go.opentelemetry.io/ebpf-profiler/libpf" + "go.opentelemetry.io/ebpf-profiler/reporter" + "go.opentelemetry.io/ebpf-profiler/tools/coredump/modulestore" +) + +type gosymCmd struct { + store *modulestore.Store + casePath string +} + +func newGosymCmd(store *modulestore.Store) *ffcli.Command { + args := &gosymCmd{store: store} + + set := flag.NewFlagSet("gosym", flag.ExitOnError) + set.StringVar(&args.casePath, "case", "", "Path of the test case to debug") + + return &ffcli.Command{ + Name: "gosym", + Exec: args.exec, + ShortUsage: "gosym", + ShortHelp: "Symbolize go test case", + FlagSet: set, + } +} + +func (cmd *gosymCmd) exec(context.Context, []string) (err error) { + // Validate arguments. + if cmd.casePath == "" { + return errors.New("please specify `-case`") + } + + var test *CoredumpTestCase + test, err = readTestCase(cmd.casePath) + if err != nil { + return fmt.Errorf("failed to read test case: %w", err) + } + + symTable, addrs, err := goModuleAddrs(cmd.store, test) + if err != nil { + return fmt.Errorf("failed to find go module addresses: %w", err) + } + + for addr, originFrames := range addrs { + file, line, fn := symTable.PCToLine(uint64(addr)) + for _, originFrame := range originFrames { + frame := reporter.FrameMetadataArgs{ + FunctionName: fn.Name, + SourceFile: file, + SourceLine: libpf.SourceLineno(line), + } + *originFrame = formatSymbolizedFrame(&frame, false) + " (" + *originFrame + ")" + } + } + + return writeTestCaseJSON(os.Stdout, test) +} + +// goModuleAddrs returns the symtable for the go module of test case and the +// addresses to symbolize for it mapped to pointers to the frames in the test +// case that reference them. +func goModuleAddrs(store *modulestore.Store, c *CoredumpTestCase) ( + *gosym.Table, map[libpf.AddressOrLineno][]*string, error, +) { + var symTable *gosym.Table + var module *ModuleInfo + var errs []error + for i := range c.Modules { + table, err := gosymTable(store, &c.Modules[i]) + if err != nil { + errs = append(errs, err) + continue + } else if symTable != nil { + return nil, nil, errors.New("multiple go modules found") + } + symTable = table + module = &c.Modules[i] + } + + if module == nil { + return nil, nil, fmt.Errorf("no go module found: %w", errors.Join(errs...)) + } + + addrs := map[libpf.AddressOrLineno][]*string{} + moduleName := filepath.Base(module.LocalPath) + for _, thread := range c.Threads { + for i, frame := range thread.Frames { + frameModuleName, addr, err := parseUnsymbolizedFrame(frame) + if err != nil { + continue + } + + if frameModuleName != moduleName { + continue + } + + addrs[addr] = append(addrs[addr], &thread.Frames[i]) + } + } + return symTable, addrs, nil +} + +func gosymTable(store *modulestore.Store, module *ModuleInfo) (*gosym.Table, error) { + reader, err := store.OpenReadAt(module.Ref) + if err != nil { + return nil, fmt.Errorf("failed to open module: %w", err) + } + defer reader.Close() + + exe, err := elf.NewFile(reader) + if err != nil { + return nil, err + } + + // Look up the address of the .text section. + var textAddr uint64 + if sect := exe.Section(".text"); sect != nil { + textAddr = sect.Addr + } + + // But prefer the runtime.text symbol if it exists. This is modeled after go + // tool addr2line, see src/cmd/internal/objfile/objfile.go in the Go tree. + symbols, err := exe.Symbols() + if err == nil { + for _, sym := range symbols { + if sym.Name == "runtime.text" { + textAddr = sym.Value + } + } + } + + if textAddr == 0 { + return nil, errors.New("missing .text section and runtime.text symbol") + } + + // TODO(fg): The section headers might be stripped, in which case we could + // try to locate gopclntab using alternative heuristics. See + // nativeunwind/elfunwindinfo/elfgopclntab.go for code that does this. + pclntab := exe.Section(".gopclntab") + if pclntab == nil { + return nil, errors.New("missing .gopclntab section") + } + + lineTableData, err := pclntab.Data() + if err != nil { + return nil, err + } + lineTable := gosym.NewLineTable(lineTableData, textAddr) + return gosym.NewTable(nil, lineTable) +} diff --git a/tools/coredump/json.go b/tools/coredump/json.go index 388691de..4f904387 100644 --- a/tools/coredump/json.go +++ b/tools/coredump/json.go @@ -8,6 +8,7 @@ package main import ( "encoding/json" "fmt" + "io" "os" "path/filepath" "runtime" @@ -73,13 +74,17 @@ func writeTestCase(path string, c *CoredumpTestCase, allowOverwrite bool) error return fmt.Errorf("failed to create JSON file: %w", err) } - enc := json.NewEncoder(jsonFile) + return writeTestCaseJSON(jsonFile, c) +} + +// writeTestCaseJSON writes a test case to the given writer as JSON. +func writeTestCaseJSON(w io.Writer, c *CoredumpTestCase) error { + enc := json.NewEncoder(w) enc.SetIndent("", " ") enc.SetEscapeHTML(false) if err := enc.Encode(c); err != nil { return fmt.Errorf("JSON Marshall failed: %w", err) } - return nil } diff --git a/tools/coredump/main.go b/tools/coredump/main.go index d24aeb37..9be2501a 100644 --- a/tools/coredump/main.go +++ b/tools/coredump/main.go @@ -58,6 +58,7 @@ func main() { newRebaseCmd(store), newUploadCmd(store), newGdbCmd(store), + newGosymCmd(store), }, Exec: func(context.Context, []string) error { return flag.ErrHelp