Skip to content

Commit

Permalink
fit: Fix field casing to snake_case. Misc cleanup.
Browse files Browse the repository at this point in the history
  • Loading branch information
@mlofjard
mlofjard committed Feb 10, 2024
1 parent 8862280 commit 33e5851
Show file tree
Hide file tree
Showing 8 changed files with 70,026 additions and 69,996 deletions.
2 changes: 1 addition & 1 deletion format/all/all.fqtest
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ elf Executable and Linkable Format
ether8023_frame Ethernet 802.3 frame
exif Exchangeable Image File Format
fairplay_spc FairPlay Server Playback Context
fit ANT FIT
fit Garmin Flexible and Interoperable Data Transfer
flac Free Lossless Audio Codec file
flac_frame FLAC frame
flac_metadatablock FLAC metadatablock
Expand Down
125 changes: 77 additions & 48 deletions format/fit/fit.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ func init() {
interp.RegisterFormat(
format.FIT,
&decode.Format{
Description: "ANT FIT",
Description: "Garmin Flexible and Interoperable Data Transfer",
Groups: []*decode.Group{format.Probe},
DecodeFn: decodeFIT,
})
Expand Down Expand Up @@ -75,15 +75,20 @@ type devFieldDefMap map[uint64]map[uint64]mappers.FieldDef
type localFieldDefMap map[uint64]map[uint64]mappers.FieldDef
type localMsgIsDevDef map[uint64]bool

// "Magic" numbers
const (
developerFieldDescMesgNo = 206 // Special data message used as dynamic field definition message
)

func fitDecodeFileHeader(d *decode.D, fc *fitContext) {
frameStart := d.Pos()

headerSize := d.FieldU8("headerSize")
d.FieldU8("protocolVersion")
d.FieldU16("profileVersion")
dataSize := d.FieldU32("dataSize")
headerSize := d.FieldU8("header_size")
d.FieldU8("protocol_version")
d.FieldU16("profile_version")
dataSize := d.FieldU32("data_size")

d.FieldRawLen("dataType", 4*8, d.AssertBitBuf([]byte(".FIT")))
d.FieldRawLen("data_type", 4*8, d.AssertBitBuf([]byte(".FIT")))
if headerSize == 14 {
headerCRC := calcCRC(d.BytesRange(frameStart, int(headerSize)-2))
d.FieldU16("crc", d.UintValidate(uint64(headerCRC)))
Expand All @@ -93,18 +98,18 @@ func fitDecodeFileHeader(d *decode.D, fc *fitContext) {
}

func fitDecodeDataRecordHeader(d *decode.D, drc *dataRecordContext) {
headerType := d.FieldU1("headerType", scalar.UintMapDescription{0: "Normal header", 1: "Compressed header"})
headerType := d.FieldU1("header_type", scalar.UintMapSymStr{0: "normal", 1: "compressed"})
drc.compressed = headerType == 1
if drc.compressed {
localMessageType := d.FieldU2("localMessageType")
d.FieldU32("timeOffset")
localMessageType := d.FieldU2("local_message_type")
d.FieldU32("time_offset")
drc.localMessageType = localMessageType
drc.data = true
} else {
mTypeIsDef := d.FieldU1("messageType", scalar.UintMapDescription{0: "Data message", 1: "Definition message"})
hasDeveloperFields := d.FieldBool("hasDeveloperFields")
mTypeIsDef := d.FieldU1("message_type", scalar.UintMapSymStr{0: "data", 1: "definition"})
hasDeveloperFields := d.FieldBool("has_developer_fields")
d.FieldBool("reserved")
localMessageType := d.FieldU4("localMessageType")
localMessageType := d.FieldU4("local_message_type")

drc.hasDeveloperFields = hasDeveloperFields
drc.localMessageType = localMessageType
Expand All @@ -114,7 +119,7 @@ func fitDecodeDataRecordHeader(d *decode.D, drc *dataRecordContext) {

func fitDecodeDefinitionMessage(d *decode.D, drc *dataRecordContext, lmfd localFieldDefMap, dmfd devFieldDefMap, isDevMap localMsgIsDevDef) {
d.FieldU8("reserved")
endian := d.FieldU8("architecture")
endian := d.FieldU8("architecture", scalar.UintMapSymStr{0: "little_endian", 1: "big_endian"})
switch endian {
case 0:
d.Endian = decode.LittleEndian
Expand All @@ -123,51 +128,75 @@ func fitDecodeDefinitionMessage(d *decode.D, drc *dataRecordContext, lmfd localF
default:
d.Fatalf("Unknown endian %d", endian)
}
messageNo := d.FieldU16("globalMessageNumber", mappers.TypeDefMap["mesg_num"])
if messageNo == 206 { // developer field_description
isDevMap[drc.localMessageType] = true
} else {
isDevMap[drc.localMessageType] = false
}

messageNo := d.FieldU16("global_message_number", mappers.TypeDefMap["mesg_num"])
isDevMap[drc.localMessageType] = messageNo == developerFieldDescMesgNo

numFields := d.FieldU8("fields")
lmfd[drc.localMessageType] = make(map[uint64]mappers.FieldDef, numFields)
d.FieldArray("fieldDefinitions", func(d *decode.D) {

d.FieldArray("field_definitions", func(d *decode.D) {
for i := uint64(0); i < numFields; i++ {
d.FieldStruct("fieldDefinition", func(d *decode.D) {
fieldDefNo := d.FieldU8("fieldDefNo", mappers.FieldDefMap[messageNo])
d.FieldStruct("field_definition", func(d *decode.D) {
fieldDefNo := d.FieldU8("field_definition_number", mappers.FieldDefMap[messageNo])
size := d.FieldU8("size")
baseType := d.FieldU8("baseType", mappers.TypeDefMap["fit_base_type"])
baseType := d.FieldU8("base_type", mappers.TypeDefMap["fit_base_type"])

var typ = mappers.TypeDefMap["fit_base_type"][baseType].Name
fDefLookup, isSet := mappers.FieldDefMap[messageNo][fieldDefNo]
if isSet {
var foundName = fDefLookup.Name
lmfd[drc.localMessageType][i] = mappers.FieldDef{Name: foundName, Type: typ, Size: size, Format: fDefLookup.Type, Unit: fDefLookup.Unit, Scale: fDefLookup.Scale, Offset: fDefLookup.Offset}
lmfd[drc.localMessageType][i] = mappers.FieldDef{
Name: foundName,
Type: typ,
Size: size,
Format: fDefLookup.Type,
Unit: fDefLookup.Unit,
Scale: fDefLookup.Scale,
Offset: fDefLookup.Offset,
}
} else {
var foundName = fmt.Sprintf("UNKNOWN_%d", fieldDefNo)
lmfd[drc.localMessageType][i] = mappers.FieldDef{Name: foundName, Type: typ, Size: size, Format: "unknown"}
lmfd[drc.localMessageType][i] = mappers.FieldDef{
Name: foundName,
Type: typ,
Size: size,
Format: "unknown",
}
}
})
}
})
if drc.hasDeveloperFields {
numDevFields := d.FieldU8("devFields")
numDevFields := d.FieldU8("developer_fields")

d.FieldArray("devFieldDefinitions", func(d *decode.D) {
d.FieldArray("developer_field_definitions", func(d *decode.D) {
for i := numFields; i < (numDevFields + numFields); i++ {
d.FieldStruct("devFieldDefinition", func(d *decode.D) {
fieldDefNo := d.FieldU8("fieldDefNo")
d.FieldStruct("developer_field_definition", func(d *decode.D) {
fieldDefNo := d.FieldU8("field_definition_number")
size := d.FieldU8("size")
devDataIdx := d.FieldU8("devDataIdx")
devDataIdx := d.FieldU8("developer_data_index")

fDefLookup, isSet := dmfd[devDataIdx][fieldDefNo]

if isSet {
var foundName = fDefLookup.Name
lmfd[drc.localMessageType][i] = mappers.FieldDef{Name: foundName, Type: fDefLookup.Type, Size: size, Unit: fDefLookup.Unit, Scale: fDefLookup.Scale, Offset: fDefLookup.Offset}
lmfd[drc.localMessageType][i] = mappers.FieldDef{
Name: foundName,
Type: fDefLookup.Type,
Size: size,
Unit: fDefLookup.Unit,
Scale: fDefLookup.Scale,
Offset: fDefLookup.Offset,
}
} else {
var foundName = fmt.Sprintf("UNKNOWN_%d", fieldDefNo)
lmfd[drc.localMessageType][i] = mappers.FieldDef{Name: foundName, Type: "UNKNOWN", Size: size, Format: "unknown"}
lmfd[drc.localMessageType][i] = mappers.FieldDef{
Name: foundName,
Type: "UNKNOWN",
Size: size,
Format: "UNKNOWN",
}
}
})
}
Expand All @@ -176,14 +205,6 @@ func fitDecodeDefinitionMessage(d *decode.D, drc *dataRecordContext, lmfd localF

}

func ensureDevFieldMap(dmfd devFieldDefMap, devIdx uint64) {
_, devIsSet := dmfd[devIdx]

if !devIsSet {
dmfd[devIdx] = make(map[uint64]mappers.FieldDef)
}
}

func fieldUint(fieldFn func(string, ...scalar.UintMapper) uint64, expectedSize uint64, fDef mappers.FieldDef, uintFormatter scalar.UintFn, fdc *fileDescriptionContext) {
var val uint64

Expand Down Expand Up @@ -242,7 +263,7 @@ func fieldFloat(fieldFn func(string, ...scalar.FltMapper) float64, expectedSize
}

func fieldString(d *decode.D, fDef mappers.FieldDef, fdc *fileDescriptionContext) {
val := d.FieldUTF8NullFixedLen(fDef.Name, int(fDef.Size), scalar.StrMapSymStr{"": "[invalid]"})
val := d.FieldUTF8NullFixedLen(fDef.Name, int(fDef.Size), scalar.StrMapDescription{"": "Invalid"})

// Save developer field definitions
switch fDef.Name {
Expand Down Expand Up @@ -301,8 +322,16 @@ func fitDecodeDataMessage(d *decode.D, drc *dataRecordContext, lmfd localFieldDe
}

if isDevDep {
ensureDevFieldMap(dmfd, fdc.devIdx)
dmfd[fdc.devIdx][fdc.fDefNo] = mappers.FieldDef{Name: fdc.name, Type: fdc.typ, Unit: fdc.unit, Scale: 0, Offset: 0}
if _, ok := dmfd[fdc.devIdx]; !ok {
dmfd[fdc.devIdx] = make(map[uint64]mappers.FieldDef)
}
dmfd[fdc.devIdx][fdc.fDefNo] = mappers.FieldDef{
Name: fdc.name,
Type: fdc.typ,
Unit: fdc.unit,
Scale: 0,
Offset: 0,
}
}
}

Expand All @@ -316,16 +345,16 @@ func decodeFIT(d *decode.D) any {

d.FieldStruct("header", func(d *decode.D) { fitDecodeFileHeader(d, &fc) })

d.FieldArray("dataRecords", func(d *decode.D) {
d.FieldArray("data_records", func(d *decode.D) {
for d.Pos() < int64((fc.headerSize+fc.dataSize)*8) {
d.FieldStruct("dataRecord", func(d *decode.D) {
d.FieldStruct("data_record", func(d *decode.D) {
var drc dataRecordContext
d.FieldStruct("dataRecordHeader", func(d *decode.D) { fitDecodeDataRecordHeader(d, &drc) })
d.FieldStruct("record_header", func(d *decode.D) { fitDecodeDataRecordHeader(d, &drc) })
switch drc.data {
case true:
d.FieldStruct("dataMessage", func(d *decode.D) { fitDecodeDataMessage(d, &drc, lmfd, dmfd, isDevMap) })
d.FieldStruct("data_message", func(d *decode.D) { fitDecodeDataMessage(d, &drc, lmfd, dmfd, isDevMap) })
case false:
d.FieldStruct("definitionMessage", func(d *decode.D) { fitDecodeDefinitionMessage(d, &drc, lmfd, dmfd, isDevMap) })
d.FieldStruct("definition_message", func(d *decode.D) { fitDecodeDefinitionMessage(d, &drc, lmfd, dmfd, isDevMap) })
}
})
}
Expand Down
3 changes: 2 additions & 1 deletion format/fit/fit.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,12 @@
The field is read as 3 separate bytes where the first 12 bits are speed and the last 12 bits are distance.
- There are still lots of UNKOWN fields due to gaps in Garmins SDK Profile documentation. (Currently FIT SDK 21.126)
- Dynamically referenced fields are named incorrectly and lacks scaling, offset and units (just raw values)
- Compressed timestamp messages are not accumulated against last known full timestamp.

### Convert stream of data messages to JSON array

```
$ fq '[.dataRecords[] | select(.dataRecordHeader.messageType == 0).dataMessage]' file.fit
$ fq '[.data_records[] | select(.record_header.message_type == "data").data_message]' file.fit
```

### Authors
Expand Down
6 changes: 3 additions & 3 deletions format/fit/mappers/values.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ var invalidFloat = map[string]float64{
func GetUintFormatter(fDef FieldDef) scalar.UintFn {
return scalar.UintFn(func(s scalar.Uint) (scalar.Uint, error) {
if s.Actual == invalidUint[fDef.Type] {
s.Sym = "[invalid]"
s.Description = "Invalid"
return s, nil
}
if fDef.Scale != 0.0 && fDef.Offset != 0 {
Expand All @@ -64,7 +64,7 @@ func GetUintFormatter(fDef FieldDef) scalar.UintFn {
func GetSintFormatter(fDef FieldDef) scalar.SintFn {
return scalar.SintFn(func(s scalar.Sint) (scalar.Sint, error) {
if s.Actual == invalidSint[fDef.Type] {
s.Sym = "[invalid]"
s.Description = "Invalid"
return s, nil
}
if fDef.Unit == "semicircles" {
Expand All @@ -90,7 +90,7 @@ func GetSintFormatter(fDef FieldDef) scalar.SintFn {
func GetFloatFormatter(fDef FieldDef) scalar.FltFn {
return scalar.FltFn(func(s scalar.Flt) (scalar.Flt, error) {
if s.Actual == invalidFloat[fDef.Type] {
s.Sym = "[invalid]"
s.Description = "Invalid"
return s, nil
}
if fDef.Scale != 0.0 && fDef.Offset != 0 {
Expand Down
Loading

0 comments on commit 33e5851

Please sign in to comment.