WIP: x64_*,arm64: Add decoders

format/all/all.go

@@ -27,6 +27,7 @@ import (
 	_ "github.com/wader/fq/format/icc"
 	_ "github.com/wader/fq/format/id3"
 	_ "github.com/wader/fq/format/inet"
+	_ "github.com/wader/fq/format/isa"
 	_ "github.com/wader/fq/format/jpeg"
 	_ "github.com/wader/fq/format/json"
 	_ "github.com/wader/fq/format/markdown"

format/elf/elf.go

@@ -16,14 +16,23 @@ import (
 	"github.com/wader/fq/pkg/scalar"
 )
 
+var x86_64Group decode.Group
+var arm64Group decode.Group
+
 func init() {
 	interp.RegisterFormat(
 		format.Elf,
 		&decode.Format{
 			Description: "Executable and Linkable Format",
 			Groups:      []*decode.Group{format.Probe},
 			DecodeFn:    elfDecode,
-		})
+			Dependencies: []decode.Dependency{
+				// TODO: x86 32/16?
+				{Groups: []*decode.Group{format.X86_64}, Out: &x86_64Group},
+				{Groups: []*decode.Group{format.Arm64}, Out: &arm64Group},
+			},
+		},
+	)
 }
 
 const (
@@ -167,6 +176,15 @@ var phTypeNames = scalar.UintRangeToScalar{
 	{Range: [2]uint64{0x70000000, 0x7fffffff}, S: scalar.Uint{Sym: "proc", Description: "Processor-specific"}},
 }
 
+var machineToFormatFn = map[int]func(d *decode.D, base uint64, symLookup func(uint64) (string, uint64)){
+	EM_X86_64: func(d *decode.D, base uint64, symLookup func(uint64) (string, uint64)) {
+		d.Format(&x86_64Group, format.X86_64In{Base: int64(base), SymLookup: symLookup})
+	},
+	EM_ARM64: func(d *decode.D, base uint64, symLookup func(uint64) (string, uint64)) {
+		d.Format(&arm64Group, format.ARM64In{Base: int64(base), SymLookup: symLookup})
+	},
+}
+
 const (
 	NT_PRSTATUS             = 1
 	NT_PRFPREG              = 2
@@ -980,6 +998,8 @@ func elfDecodeDynamicTags(d *decode.D, ec elfContext, dc dynamicContext) {
 }
 
 func elfDecodeSectionHeader(d *decode.D, ec elfContext, sh sectionHeader) {
+	var execInstr bool
+
 	shFlags := func(d *decode.D, archBits int) {
 		d.FieldStruct("flags", func(d *decode.D) {
 			if d.Endian == decode.LittleEndian {
@@ -988,7 +1008,7 @@ func elfDecodeSectionHeader(d *decode.D, ec elfContext, sh sectionHeader) {
 				d.FieldBool("strings")
 				d.FieldBool("merge")
 				d.FieldU1("unused0")
-				d.FieldBool("execinstr")
+				execInstr = d.FieldBool("execinstr")
 				d.FieldBool("alloc")
 				d.FieldBool("write")
 				d.FieldBool("tls")
@@ -1018,13 +1038,14 @@ func elfDecodeSectionHeader(d *decode.D, ec elfContext, sh sectionHeader) {
 				d.FieldBool("strings")
 				d.FieldBool("merge")
 				d.FieldU1("unused2")
-				d.FieldBool("execinstr")
+				execInstr = d.FieldBool("execinstr")
 				d.FieldBool("alloc")
 				d.FieldBool("write")
 			}
 		})
 	}
 
+	var addr uint64
 	var offset int64
 	var size int64
 	var entSize int64
@@ -1035,7 +1056,7 @@ func elfDecodeSectionHeader(d *decode.D, ec elfContext, sh sectionHeader) {
 		d.FieldU32("name", strTable(ec.strTabMap[STRTAB_SHSTRTAB]))
 		typ = d.FieldU32("type", sectionHeaderTypeMap, scalar.UintHex)
 		shFlags(d, ec.archBits)
-		d.FieldU("addr", ec.archBits, scalar.UintHex)
+		addr = d.FieldU("addr", ec.archBits, scalar.UintHex)
 		offset = int64(d.FieldU("offset", ec.archBits)) * 8
 		size = int64(d.FieldU32("size", scalar.UintHex) * 8)
 		d.FieldU32("link")
@@ -1046,7 +1067,7 @@ func elfDecodeSectionHeader(d *decode.D, ec elfContext, sh sectionHeader) {
 		d.FieldU32("name", strTable(ec.strTabMap[STRTAB_SHSTRTAB]))
 		typ = d.FieldU32("type", sectionHeaderTypeMap, scalar.UintHex)
 		shFlags(d, ec.archBits)
-		d.FieldU("addr", ec.archBits, scalar.UintHex)
+		addr = d.FieldU("addr", ec.archBits, scalar.UintHex)
 		offset = int64(d.FieldU("offset", ec.archBits, scalar.UintHex) * 8)
 		size = int64(d.FieldU64("size") * 8)
 		d.FieldU32("link")
@@ -1079,9 +1100,34 @@ func elfDecodeSectionHeader(d *decode.D, ec elfContext, sh sectionHeader) {
 			elfDecodeSymbolTable(d, ec, int(size/entSize), ec.strTabMap[STRTAB_DYNSTR])
 		})
 	case SHT_PROGBITS:
-		// TODO: name progbits?
-		// TODO: decode opcodes
-		d.FieldRawLen("data", size)
+		// TODO: verify this, seems to result in strange relative addresses
+		symLookup := func(symAddr uint64) (string, uint64) {
+			var best *symbol
+
+			for _, sh := range ec.sections {
+				for i, s := range sh.symbols {
+					if symAddr >= s.value && (best == nil || symAddr-s.value < best.value-s.value) {
+						best = &sh.symbols[i]
+					}
+				}
+			}
+			if best == nil {
+				return "", 0
+			}
+			return strIndexNull(int(best.name), ec.strTabMap[STRTAB_STRTAB]), best.value
+		}
+
+		// TODO: name progbits? instructions?
+		if fn, ok := machineToFormatFn[ec.machine]; execInstr && ok {
+			d.FieldArray("code", func(d *decode.D) {
+				d.FramedFn(size, func(d *decode.D) {
+					fn(d, addr, symLookup)
+				})
+			})
+		} else {
+			d.FieldRawLen("data", size)
+		}
+
 	case SHT_GNU_HASH:
 		d.FieldStruct("gnu_hash", func(d *decode.D) {
 			elfDecodeGNUHash(d, ec, size, ec.strTabMap[STRTAB_DYNSTR])

format/format.go

@@ -35,6 +35,7 @@ var (
 	Apev2              = &decode.Group{Name: "apev2"}
 	AppleBookmark      = &decode.Group{Name: "apple_bookmark"}
 	Ar                 = &decode.Group{Name: "ar"}
+	Arm64              = &decode.Group{Name: "arm64"}
 	Asn1Ber            = &decode.Group{Name: "asn1_ber"}
 	Av1Ccr             = &decode.Group{Name: "av1_ccr"}
 	Av1Frame           = &decode.Group{Name: "av1_frame"}
@@ -120,8 +121,8 @@ var (
 	ProtobufWidevine   = &decode.Group{Name: "protobuf_widevine"}
 	PsshPlayready      = &decode.Group{Name: "pssh_playready"}
 	Rtmp               = &decode.Group{Name: "rtmp"}
-	SllPacket          = &decode.Group{Name: "sll_packet"}
 	Sll2Packet         = &decode.Group{Name: "sll2_packet"}
+	SllPacket          = &decode.Group{Name: "sll_packet"}
 	Tar                = &decode.Group{Name: "tar"}
 	TcpSegment         = &decode.Group{Name: "tcp_segment"}
 	Tiff               = &decode.Group{Name: "tiff"}
@@ -138,6 +139,9 @@ var (
 	Wasm               = &decode.Group{Name: "wasm"}
 	Wav                = &decode.Group{Name: "wav"}
 	Webp               = &decode.Group{Name: "webp"}
+	X86_16             = &decode.Group{Name: "x86_16"}
+	X86_32             = &decode.Group{Name: "x86_32"}
+	X86_64             = &decode.Group{Name: "x86_64"}
 	Xml                = &decode.Group{Name: "xml"}
 	Yaml               = &decode.Group{Name: "yaml"}
 	Zip                = &decode.Group{Name: "zip"}
@@ -341,3 +345,13 @@ type BitCoinBlockIn struct {
 type TLSIn struct {
 	Keylog string `doc:"NSS Key Log content"`
 }
+
+type X86_64In struct {
+	SymLookup func(symAddr uint64) (string, uint64)
+	Base      int64
+}
+
+type ARM64In struct {
+	SymLookup func(symAddr uint64) (string, uint64)
+	Base      int64
+}

format/isa/arm64.go

@@ -0,0 +1,58 @@
+package isa
+
+import (
+	"strings"
+
+	"github.com/wader/fq/format"
+	"github.com/wader/fq/pkg/decode"
+	"github.com/wader/fq/pkg/interp"
+	"github.com/wader/fq/pkg/scalar"
+	"golang.org/x/arch/arm64/arm64asm"
+)
+
+func init() {
+	interp.RegisterFormat(
+		format.Arm64,
+		&decode.Format{
+			Description: "ARM64 instructions",
+			DecodeFn:    decodeARM64,
+			RootArray:   true,
+			RootName:    "instructions",
+		})
+}
+
+func decodeARM64(d *decode.D) any {
+	var symLookup func(uint64) (string, uint64)
+	var base int64
+	var ai format.ARM64In
+
+	if d.ArgAs(&ai) {
+		symLookup = ai.SymLookup
+		base = ai.Base
+	}
+
+	bb := d.BytesRange(0, int(d.BitsLeft()/8))
+	// TODO: uint64?
+	pc := base
+
+	for !d.End() {
+		d.FieldStruct("instruction", func(d *decode.D) {
+			i, err := arm64asm.Decode(bb)
+			if err != nil {
+				d.Fatalf("failed to decode arm64 instruction: %s", err)
+			}
+
+			// TODO: other syntax
+			d.FieldRawLen("opcode", int64(4)*8, scalar.BitBufSym(arm64asm.GoSyntax(i, uint64(pc), symLookup, nil)), scalar.RawHex)
+
+			// TODO: Enc?
+			d.FieldValueUint("op", uint64(i.Enc), scalar.UintSym(strings.ToLower(i.Op.String())), scalar.UintHex)
+
+			bb = bb[4:]
+			pc += int64(4)
+		})
+
+	}
+
+	return nil
+}

format/isa/x86.go

@@ -0,0 +1,78 @@
+package isa
+
+import (
+	"strings"
+
+	"github.com/wader/fq/format"
+	"github.com/wader/fq/pkg/decode"
+	"github.com/wader/fq/pkg/interp"
+	"github.com/wader/fq/pkg/scalar"
+	"golang.org/x/arch/x86/x86asm"
+)
+
+func init() {
+	// amd64?
+	interp.RegisterFormat(
+		format.X86_64,
+		&decode.Format{
+			Description: "x86-64 instructions",
+			DecodeFn:    func(d *decode.D) any { return decodeX86(d, 64) },
+			RootArray:   true,
+			RootName:    "instructions",
+		})
+	interp.RegisterFormat(
+		format.X86_32,
+		&decode.Format{
+			Description: "x86-32 instructions",
+			DecodeFn:    func(d *decode.D) any { return decodeX86(d, 32) },
+			RootArray:   true,
+			RootName:    "instructions",
+		})
+	interp.RegisterFormat(
+		format.X86_16,
+		&decode.Format{
+			Description: "x86-16 instructions",
+			DecodeFn:    func(d *decode.D) any { return decodeX86(d, 16) },
+			RootArray:   true,
+			RootName:    "instructions",
+		})
+}
+
+func decodeX86(d *decode.D, mode int) any {
+	var symLookup func(uint64) (string, uint64)
+	var base int64
+	var xi format.X86_64In
+
+	if d.ArgAs(&xi) {
+		symLookup = xi.SymLookup
+		base = xi.Base
+	}
+
+	bb := d.BytesRange(0, int(d.BitsLeft()/8))
+	// TODO: uint64?
+	pc := base
+
+	for !d.End() {
+		d.FieldStruct("instruction", func(d *decode.D) {
+			i, err := x86asm.Decode(bb, mode)
+			if err != nil {
+				d.Fatalf("failed to decode x86 instruction: %s", err)
+			}
+
+			d.FieldRawLen("opcode", int64(i.Len)*8, scalar.BitBufSym(x86asm.IntelSyntax(i, uint64(pc), symLookup)), scalar.RawHex)
+
+			// log.Printf("i.Len: %#+v\n", i.Len)
+			// log.Printf("i.Opcode: %x\n", i.Opcode)
+			// log.Printf("i: %#+v\n", i)
+
+			// TODO: rebuild op lower?
+			d.FieldValueUint("op", uint64(i.Opcode), scalar.UintSym(strings.ToLower(i.Op.String())), scalar.UintHex)
+
+			bb = bb[i.Len:]
+			pc += int64(i.Len)
+		})
+
+	}
+
+	return nil
+}

go.mod

@@ -74,6 +74,8 @@ require (
 	gopkg.in/yaml.v3 v3.0.1
 )
 
+require golang.org/x/arch v0.0.0-20220401014709-5424468ecbac
+
 require (
 	github.com/itchyny/timefmt-go v0.1.5 // indirect
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect

go.sum

@@ -27,6 +27,8 @@ github.com/wader/gojq v0.12.1-0.20230308145020-2de2194791c0 h1:OjBLxUJRtmoYbNtgB
 github.com/wader/gojq v0.12.1-0.20230308145020-2de2194791c0/go.mod h1:jQY39j9tgky+JYcJrKNz5OYTe/sPDAw7FvVj13JGqVk=
 github.com/wader/readline v0.0.0-20230307172220-bcb7158e7448 h1:AzpBtmgdXa3uznrb3esNeEoaLqtNEwckRmaUH0qWD6w=
 github.com/wader/readline v0.0.0-20230307172220-bcb7158e7448/go.mod h1:Zgz8IJWvJoe7NK23CCPpC109XMCqJCpUhpHcnnA4XaM=
+golang.org/x/arch v0.0.0-20220401014709-5424468ecbac h1:05z6X/pDgf2qll8x7kbRRVdr33GjdV/GOCGiQfnaJS8=
+golang.org/x/arch v0.0.0-20220401014709-5424468ecbac/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
 golang.org/x/crypto v0.8.0 h1:pd9TJtTueMTVQXzk8E2XESSMQDj/U7OUu0PqJqPXQjQ=
 golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE=
 golang.org/x/exp v0.0.0-20230131160201-f062dba9d201 h1:BEABXpNXLEz0WxtA+6CQIz2xkg80e+1zrhWyMcq8VzE=