Merge pull request #19 from jeremyje/metrics

Fix ndjson formatting

Makefile

@@ -86,4 +86,7 @@ clean:
 	-chmod -R +w build/
 	rm -rf build/
 
+convert:
+	$(GO) run cmd/converter/converter.go
+
 .PHONY: all run lint test clean

cmd/converter/converter.go

@@ -0,0 +1,122 @@
+package main
+
+import (
+	"bufio"
+	"encoding/json"
+	"fmt"
+	"log"
+	"os"
+	"time"
+
+	"github.com/jeremyje/coretemp-exporter/drivers/common"
+	pb "github.com/jeremyje/coretemp-exporter/proto"
+	"google.golang.org/protobuf/encoding/protojson"
+	"google.golang.org/protobuf/types/known/timestamppb"
+)
+
+// HardwareInfo includes CPU and motherboard information about the host machine.
+type HardwareInfo struct {
+	// Load as a percentage [0-100].
+	Load []int `json:"load" yaml:"load"`
+	// TJMax is the thermal junction maximum of the CPU. Once the temperature hits this limit the CPU will thermal throttle.
+	TJMax []float64 `json:"tjMax" yaml:"tjMax"`
+	// CoreCount is the total number of cores across all CPUs on the machine.
+	CoreCount int `json:"coreCount" yaml:"coreCount"`
+	// CPUCount is the number of CPUs on the machine.
+	CPUCount int `json:"cpuCount" yaml:"cpuCount"`
+	// TemperatureCelcius is the temperature of each core expressed in Celcius.
+	TemperatureCelcius []float64 `json:"temperatureCelcius" yaml:"temperatureCelcius"`
+	// VID is the voltage requested by the CPU.
+	VID float64 `json:"vid" yaml:"vid"`
+	// CPUSpeed is the clock frequency of the CPU.
+	CPUSpeed float64 `json:"cpuSpeed" yaml:"cpuSpeed"`
+	// FSBSpeed is the clock frequency of the front side bus.
+	FSBSpeed float64 `json:"fsbSpeed" yaml:"fsbSpeed"`
+	// Multiplier is the front side bus multipler of the CPU.
+	Multiplier float64 `json:"multiplier" yaml:"multiplier"`
+	// CPUName is the name and model of the CPU.
+	CPUName string `json:"cpuName" yaml:"cpuName"`
+	// Timestamp is the time that the sample was taken.
+	Timestamp time.Time `json:"timestamp" yaml:"timestamp"`
+}
+
+func main() {
+	os.Remove("new.ndjson")
+	fp, err := os.Create("new.ndjson")
+	check(err)
+	defer fp.Close()
+
+	ln := 0
+	filenames := []string{"cputemps.log", "cputemps.ndjson"}
+	for _, filename := range filenames {
+		log.Printf("OPEN %s", filename)
+		in, err := os.Open(filename)
+		check(err)
+		defer in.Close()
+		scanner := bufio.NewScanner(in)
+		for scanner.Scan() {
+			ln++
+			if ln%10000 == 0 {
+				log.Printf("Line: %s:%d", filename, ln)
+			}
+			pba := &pb.MachineMetrics{}
+			line := scanner.Bytes()
+			if len(line) < 10 {
+				continue
+			}
+			err := json.Unmarshal(line, &pba)
+			if err != nil || pba == nil || len(pba.GetName()) == 0 {
+				info := &HardwareInfo{}
+				err := json.Unmarshal(line, &info)
+				check(err)
+				pba = convertToProto(info)
+				if len(pba.Name) == 0 {
+					check(fmt.Errorf("empty name: %+v", pba))
+				}
+			}
+
+			newLine, err := protojson.Marshal(pba)
+			check(err)
+			_, err = fp.Write(newLine)
+			check(err)
+			_, err = fp.WriteString("\n")
+			check(err)
+		}
+	}
+}
+
+func copyInt(i []int) []int32 {
+	a := []int32{}
+	for _, v := range i {
+		a = append(a, int32(v))
+	}
+	return a
+}
+
+func convertToProto(info *HardwareInfo) *pb.MachineMetrics {
+	return &pb.MachineMetrics{
+		Name: "quartz",
+		Device: []*pb.DeviceMetrics{
+			{
+				Name:        info.CPUName,
+				Kind:        "cpu",
+				Temperature: common.Average(info.TemperatureCelcius),
+				Cpu: &pb.CpuDeviceMetrics{
+					Load:            copyInt(info.Load),
+					Temperature:     info.TemperatureCelcius,
+					NumCores:        int32(info.CoreCount),
+					FrequencyMhz:    info.CPUSpeed,
+					FsbFrequencyMhz: info.FSBSpeed,
+				},
+			},
+		},
+		Timestamp: timestamppb.New(info.Timestamp),
+	}
+}
+
+func check(err error) {
+	if err != nil {
+		log.Printf("ERROR: %s", err)
+		panic(err)
+	}
+}

drivers/common/common_test.go

@@ -16,14 +16,14 @@ package common
 
 import (
 	_ "embed"
-	"encoding/json"
 	"fmt"
 	"strings"
 	"testing"
 	"time"
 
 	"github.com/google/go-cmp/cmp"
 	pb "github.com/jeremyje/coretemp-exporter/proto"
+	"google.golang.org/protobuf/encoding/protojson"
 	"google.golang.org/protobuf/testing/protocmp"
 	"google.golang.org/protobuf/types/known/timestamppb"
 	"gopkg.in/yaml.v3"
@@ -63,19 +63,19 @@ func TestBasic(t *testing.T) {
 	}
 
 	gotProto := &pb.MachineMetrics{}
-	if err := json.Unmarshal(wantJSON, gotProto); err != nil {
+	if err := protojson.Unmarshal(wantJSON, gotProto); err != nil {
 		t.Error(err)
 	}
 	if diff := cmp.Diff(want, gotProto, protocmp.Transform()); diff != "" {
-		t.Errorf("json.Unmarshal() mismatch (-want +got):\n%s", diff)
+		t.Errorf("protojson.Unmarshal() mismatch (-want +got):\n%s", diff)
 	}
-	if gotJSON, err := json.Marshal(want); err != nil {
+	if gotJSON, err := protojson.Marshal(want); err != nil {
 		t.Error(err)
 	} else {
 		gotJSON = nolinefeed(gotJSON)
 		if diff := cmp.Diff(wantJSON, gotJSON); diff != "" {
-			t.Logf("json.Marshal\n============\n%s", string(gotJSON))
-			t.Errorf("json.Marshal() mismatch (-want +got):\n%s", diff)
+			t.Logf("protojson.Marshal\n============\n%s", string(gotJSON))
+			t.Errorf("protojson.Marshal() mismatch (-want +got):\n%s", diff)
 		}
 	}
 	if gotYAML, err := yaml.Marshal(want); err != nil {
@@ -92,21 +92,21 @@ func TestBasic(t *testing.T) {
 func TestMarshalJSON(t *testing.T) {
 	data := &pb.MachineMetrics{}
 	want := nolinefeed(exampleJSON)
-	if err := json.Unmarshal(want, data); err != nil {
+	if err := protojson.Unmarshal(want, data); err != nil {
 		t.Error(err)
 	}
 	if diff := cmp.Diff([]int32{0, 1, 2, 3}, data.GetDevice()[0].GetCpu().GetLoad()); diff != "" {
-		t.Errorf("json.Unmarshal() mismatch (-want +got):\n%s", diff)
+		t.Errorf("protojson.Unmarshal() mismatch (-want +got):\n%s", diff)
 	}
-	m, err := json.Marshal(data)
+	m, err := protojson.Marshal(data)
 	if err != nil {
 		t.Error(err)
 	}
 
 	m = nolinefeed(m)
 	if diff := cmp.Diff(want, m); diff != "" {
-		t.Logf("json.Marshal\n============\n%s", string(m))
-		t.Errorf("json.Marshal() mismatch (-want +got):\n%s", diff)
+		t.Logf("protojson.Marshal\n============\n%s", string(m))
+		t.Errorf("protojson.Marshal() mismatch (-want +got):\n%s", diff)
 	}
 }
 

drivers/common/testdata/example.json

@@ -1 +1 @@
-{"name":"test","device":[{"name":"vermeer","kind":"cpu","temperature":37.325,"cpu":{"load":[0,1,2,3],"temperature":[1,4,65.4,78.9],"num_cores":4,"frequency_mhz":5000.2,"fsb_frequency_mhz":100.4}}],"timestamp":{"seconds":1136214245}}
+{"name":"test","device":[{"name":"vermeer","kind":"cpu","temperature":37.325,"cpu":{"load":[0,1,2,3],"temperature":[1,4,65.4,78.9],"numCores":4,"frequencyMhz":5000.2,"fsbFrequencyMhz":100.4}}],"timestamp":"2006-01-02T15:04:05Z"}

internal/file.go

@@ -16,10 +16,11 @@ package internal
 
 import (
 	"context"
-	"encoding/json"
 	"log"
 	"os"
 
+	"google.golang.org/protobuf/encoding/protojson"
+
 	pb "github.com/jeremyje/coretemp-exporter/proto"
 )
 
@@ -42,7 +43,7 @@ func newFileSink(name string) (*fileSink, error) {
 }
 
 func (s *fileSink) Observe(ctx context.Context, info *pb.MachineMetrics) {
-	line, err := json.Marshal(info)
+	line, err := protojson.Marshal(info)
 	if err == nil {
 		if _, err := s.fp.Write(line); err != nil {
 			log.Printf("ERROR: %s", err)