ndjson to csv file conversion tool

.github/workflows/ci.yaml

@@ -89,6 +89,8 @@ jobs:
           LICENSE
           build/bin/linux_amd64/coretemp-exporter
           build/bin/windows_amd64/coretemp-exporter.exe
+          build/bin/linux_amd64/converter
+          build/bin/windows_amd64/converter.exe
 
     - name: Publish Application
       uses: actions/upload-artifact@v3
@@ -99,3 +101,5 @@ jobs:
           LICENSE
           build/bin/linux_amd64/coretemp-exporter
           build/bin/windows_amd64/coretemp-exporter.exe
+          build/bin/linux_amd64/converter
+          build/bin/windows_amd64/converter.exe

.gitignore

@@ -35,3 +35,7 @@
 /coretemp-exporter
 build/
 *.ndjson
+*.csv
+
+!internal/testdata/cputemps.csv
+!internal/testdata/cputemps.ndjson

Makefile

@@ -39,7 +39,7 @@ LINUX_NICHE_PLATFORMS =
 WINDOWS_PLATFORMS = windows_386 windows_amd64
 MAIN_PLATFORMS = windows_amd64 linux_amd64 linux_arm64
 ALL_PLATFORMS = $(LINUX_PLATFORMS) $(LINUX_NICHE_PLATFORMS) $(WINDOWS_PLATFORMS) $(foreach niche,$(NICHE_PLATFORMS),$(niche)_amd64 $(niche)_arm64)
-ALL_APPS = coretemp-exporter
+ALL_APPS = coretemp-exporter converter
 
 MAIN_BINARIES = $(foreach app,$(ALL_APPS),$(foreach platform,$(MAIN_PLATFORMS),build/bin/$(platform)/$(app)$(if $(findstring windows_,$(platform)),.exe,)))
 ALL_BINARIES = $(foreach app,$(ALL_APPS),$(foreach platform,$(ALL_PLATFORMS),build/bin/$(platform)/$(app)$(if $(findstring windows_,$(platform)),.exe,)))
@@ -126,6 +126,6 @@ clean:
 	rm -rf build/
 
 convert:
-	$(GO) run cmd/converter/converter.go
+	$(GO) run cmd/converter/converter.go -input=cputemps.ndjson -output=cputemps.csv -mode=csv
 
 .PHONY: all run lint test clean

cmd/converter/converter.go

@@ -16,22 +16,37 @@ package main
 
 import (
 	"flag"
+	"fmt"
 	"log"
 	"strings"
 
 	"github.com/jeremyje/coretemp-exporter/internal"
 )
 
 var (
-	oldFilesFlag = flag.String("old", "cputemps.log,cputemps.ndjson", "Comma separated list of old files.")
-	newFileFlag  = flag.String("new", "new.ndjson", "The new ndjson file")
+	inputFilesFlag = flag.String("input", "cputemps.log,cputemps.ndjson", "Comma separated list of old files.")
+	outputFileFlag = flag.String("output", "new.ndjson", "The new ndjson file")
+	modeFlag       = flag.String("mode", "csv", "Conversion mode (update, csv)")
 )
 
 func main() {
-	if err := internal.ConvertV1ToV2(&internal.ConvertArgs{
-		OldFiles: strings.Split(*oldFilesFlag, ","),
-		NewFile:  *newFileFlag,
-	}); err != nil {
+	flag.Parse()
+	var err error
+	switch *modeFlag {
+	case "csv":
+		err = internal.ConvertCSV(&internal.ConvertCSVArgs{
+			InputFile:  strings.Split(*inputFilesFlag, ","),
+			OutputFile: *outputFileFlag,
+		})
+	case "update":
+		err = internal.ConvertV1ToV2(&internal.ConvertArgs{
+			InputFiles: strings.Split(*inputFilesFlag, ","),
+			OutputFile: *outputFileFlag,
+		})
+	default:
+		err = fmt.Errorf("mode '%s' is not supported", *modeFlag)
+	}
+	if err != nil {
 		log.Fatal(err)
 	}
 }

install/compose/prometheus/prometheus.yaml

@@ -64,3 +64,5 @@ scrape_configs:
   - targets:
     - host.docker.internal:8181
     - coretemp-exporter:8181
+    # List any machines you'd like to monitor here.
+    - quartz:8181

internal/convert_csv.go

@@ -0,0 +1,103 @@
+// Copyright 2022 Jeremy Edwards
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package internal
+
+import (
+	"bufio"
+	"encoding/csv"
+	"fmt"
+	"log"
+	"os"
+	"strconv"
+
+	pb "github.com/jeremyje/coretemp-exporter/proto"
+	"google.golang.org/protobuf/encoding/protojson"
+)
+
+type ConvertCSVArgs struct {
+	InputFile  []string
+	OutputFile string
+}
+
+func ConvertCSV(args *ConvertCSVArgs) error {
+	os.Remove(args.OutputFile)
+	fp, err := os.Create(args.OutputFile)
+	if err != nil {
+		return fmt.Errorf("cannot create output file '%s', %w", args.OutputFile, err)
+	}
+
+	cw := csv.NewWriter(fp)
+	cw.Write([]string{
+		"ts_year", "ts_month", "ts_day", "ts_hour", "ts_min", "ts_second", // Timestamp
+		"active_cores", "total_cores", "frequency", "avg_load", // Basic CPU metrics
+		"temperature"}) // CPU Temperature
+	defer cw.Flush()
+	return scanNdJson(args.InputFile, func(item *pb.MachineMetrics) error {
+		if len(item.GetDevice()) == 0 {
+			return nil
+		}
+		t := item.GetTimestamp().AsTime()
+
+		cpuMetrics := item.GetDevice()[0]
+		activeCores := 0
+		totalLoad := 0
+		for _, coreLoad := range cpuMetrics.Cpu.GetLoad() {
+			if coreLoad > 0 {
+				activeCores++
+			}
+			totalLoad += int(coreLoad)
+		}
+		numCores := len(cpuMetrics.Cpu.GetLoad())
+		avgLoad := float64(totalLoad) / float64(numCores)
+
+		return cw.Write([]string{
+			strconv.Itoa(t.Year()), strconv.Itoa(int(t.Month())), strconv.Itoa(t.Day()), strconv.Itoa(t.Hour()), strconv.Itoa(t.Minute()), strconv.Itoa(t.Second()),
+			strconv.Itoa(activeCores), strconv.Itoa(numCores), fmt.Sprintf("%f", cpuMetrics.GetCpu().GetFrequencyMhz()), fmt.Sprintf("%f", avgLoad),
+			fmt.Sprintf("%f", cpuMetrics.GetTemperature()),
+		})
+	})
+}
+
+func scanNdJson(inputFiles []string, consumerFunc func(line *pb.MachineMetrics) error) error {
+	for _, inputFile := range inputFiles {
+		fp, err := os.Open(inputFile)
+		if err != nil {
+			return fmt.Errorf("cannot open '%s', %w", inputFile, err)
+		}
+		defer fp.Close()
+
+		ln := 0
+		scanner := bufio.NewScanner(fp)
+		for scanner.Scan() {
+			ln++
+			if ln%10000 == 0 {
+				log.Printf("Line: %s:%d", inputFile, ln)
+			}
+			line := scanner.Bytes()
+			if len(line) < 10 {
+				continue
+			}
+			mm := &pb.MachineMetrics{}
+			err := protojson.Unmarshal(line, mm)
+			if err != nil || mm == nil || len(mm.GetName()) == 0 {
+				return fmt.Errorf("cannot read line '%s:%d', %w", inputFile, ln, err)
+			}
+			if err := consumerFunc(mm); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}

internal/convert_csv_test.go

@@ -0,0 +1,55 @@
+// Copyright 2022 Jeremy Edwards
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package internal
+
+import (
+	_ "embed"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+)
+
+var (
+	//go:embed testdata/cputemps.ndjson
+	cputempsNdjson []byte
+	//go:embed testdata/cputemps.csv
+	cputempsCsv []byte
+)
+
+func TestConvertCSV(t *testing.T) {
+	dir := t.TempDir()
+	input := filepath.Join(dir, "input.ndjson")
+	output := filepath.Join(dir, "output.csv")
+	if err := os.WriteFile(input, cputempsNdjson, 0664); err != nil {
+		t.Fatalf("cannot write '%s', %s", input, err)
+	}
+
+	if err := ConvertCSV(&ConvertCSVArgs{
+		InputFile:  []string{input},
+		OutputFile: output,
+	}); err != nil {
+		t.Fatalf("cannot write csv '%s', %s", output, err)
+	}
+
+	actual, err := os.ReadFile(output)
+	if err != nil {
+		t.Errorf("cannot read back '%s', %s", output, err)
+	}
+	if diff := cmp.Diff(string(actual), string(cputempsCsv)); diff != "" {
+		t.Errorf(diff)
+	}
+}

internal/converter.go → internal/convert_update.go

@@ -29,8 +29,8 @@ import (
 )
 
 type ConvertArgs struct {
-	OldFiles []string
-	NewFile  string
+	InputFiles []string
+	OutputFile string
 }
 
 // HardwareInfo includes CPU and motherboard information about the host machine.
@@ -60,15 +60,15 @@ type HardwareInfo struct {
 }
 
 func ConvertV1ToV2(args *ConvertArgs) error {
-	os.Remove(args.NewFile)
-	fp, err := os.Create(args.NewFile)
+	os.Remove(args.OutputFile)
+	fp, err := os.Create(args.OutputFile)
 	if err != nil {
 		return err
 	}
 	defer fp.Close()
 
 	ln := 0
-	for _, filename := range args.OldFiles {
+	for _, filename := range args.InputFiles {
 		log.Printf("OPEN %s", filename)
 		in, err := os.Open(filename)
 		if err != nil {

internal/testdata/cputemps.csv

@@ -0,0 +1,101 @@
+ts_year,ts_month,ts_day,ts_hour,ts_min,ts_second,active_cores,total_cores,frequency,avg_load,temperature
+2023,1,22,23,23,59,12,12,3700.041504,2.250000,68.250000
+2023,1,22,23,24,0,2,12,3600.040283,0.833333,68.875000
+2023,1,22,23,24,1,8,12,3720.041748,1.083333,70.125000
+2023,1,22,23,24,2,3,12,3700.041504,0.333333,69.625000
+2023,1,22,23,24,3,2,12,3700.041504,0.166667,68.750000
+2023,1,22,23,24,4,4,12,3600.040283,1.416667,68.625000
+2023,1,22,23,24,5,4,12,3660.041016,0.916667,68.500000
+2023,1,22,23,24,6,8,12,4500.050293,1.416667,68.500000
+2023,1,22,23,24,7,4,12,3700.041504,0.583333,68.875000
+2023,1,22,23,24,8,2,12,3640.041016,0.250000,68.250000
+2023,1,22,23,24,9,7,12,3700.041504,1.000000,69.125000
+2023,1,22,23,24,10,3,12,3640.041016,1.583333,70.125000
+2023,1,22,23,24,11,3,12,3700.041504,0.500000,69.500000
+2023,1,22,23,24,12,7,12,3740.041992,0.833333,68.625000
+2023,1,22,23,24,13,1,12,3700.041504,0.166667,68.125000
+2023,1,22,23,24,14,2,12,3600.040283,0.833333,68.000000
+2023,1,22,23,24,15,9,12,3680.041260,1.250000,67.750000
+2023,1,22,23,24,16,2,12,3600.040283,0.166667,66.875000
+2023,1,22,23,24,17,2,12,3700.041504,0.166667,66.250000
+2023,1,22,23,24,18,7,12,2880.032227,0.666667,65.375000
+2023,1,22,23,24,19,3,12,3700.041504,1.166667,66.125000
+2023,1,22,23,24,20,2,12,4550.051270,0.833333,66.000000
+2023,1,22,23,24,21,3,12,3720.041748,0.333333,66.000000
+2023,1,22,23,24,22,5,12,3700.041504,0.500000,65.375000
+2023,1,22,23,24,23,1,12,3740.041992,0.416667,65.125000
+2023,1,22,23,24,24,4,12,3600.040283,1.750000,65.625000
+2023,1,22,23,24,25,11,12,4525.050781,2.250000,69.125000
+2023,1,22,23,24,26,2,12,3620.040771,0.833333,69.500000
+2023,1,22,23,24,27,8,12,3600.040283,0.750000,69.125000
+2023,1,22,23,24,28,3,12,3640.041016,1.000000,69.000000
+2023,1,22,23,24,29,5,12,3600.040283,1.250000,69.375000
+2023,1,22,23,24,30,7,12,3680.041260,1.083333,68.375000
+2023,1,22,23,24,31,6,12,3600.040283,0.916667,68.125000
+2023,1,22,23,24,32,2,12,3680.041260,0.416667,68.375000
+2023,1,22,23,24,33,3,12,4550.051270,0.500000,68.000000
+2023,1,22,23,24,34,6,12,3660.041016,1.583333,69.000000
+2023,1,22,23,24,35,7,12,3700.041504,1.000000,68.625000
+2023,1,22,23,24,36,1,12,3680.041260,0.083333,67.750000
+2023,1,22,23,24,37,5,12,3680.041260,0.583333,68.375000
+2023,1,22,23,24,38,8,12,3600.040283,1.083333,69.375000
+2023,1,22,23,24,39,7,12,3660.041016,1.750000,69.625000
+2023,1,22,23,24,40,5,12,3680.041260,1.416667,69.000000
+2023,1,22,23,24,41,1,12,3600.040283,0.166667,68.750000
+2023,1,22,23,24,42,5,12,3660.041016,1.250000,69.125000
+2023,1,22,23,24,43,7,12,3700.041504,0.916667,69.625000
+2023,1,22,23,24,44,3,12,3680.041260,1.500000,69.500000
+2023,1,22,23,24,45,3,12,3660.041016,1.083333,69.875000
+2023,1,22,23,24,46,7,12,3640.041016,1.333333,69.625000
+2023,1,22,23,24,47,2,12,3600.040283,0.500000,69.625000
+2023,1,22,23,24,48,10,12,3740.041992,0.916667,68.875000
+2023,1,22,23,24,49,5,12,3600.040283,1.583333,68.625000
+2023,1,22,23,24,50,4,12,3660.041016,1.000000,68.000000
+2023,1,22,23,24,51,4,12,3680.041260,0.666667,68.375000
+2023,1,22,23,24,52,10,12,3700.041504,1.083333,68.375000
+2023,1,22,23,24,53,4,12,2880.032227,0.750000,67.500000
+2023,1,22,23,24,54,4,12,3600.040283,1.666667,67.375000
+2023,1,22,23,24,55,12,12,3680.041260,1.666667,66.875000
+2023,1,22,23,24,56,3,12,3600.040283,0.583333,66.750000
+2023,1,22,23,24,57,4,12,3700.041504,0.416667,66.500000
+2023,1,22,23,24,58,2,12,3700.041504,0.166667,65.875000
+2023,1,22,23,24,59,6,12,3700.041504,1.583333,66.250000
+2023,1,22,23,25,0,4,12,3660.041016,1.333333,66.500000
+2023,1,22,23,25,1,3,12,3700.041504,1.166667,67.343750
+2023,1,22,23,25,2,8,12,3680.041260,1.083333,67.875000
+2023,1,22,23,25,3,3,12,3700.041504,0.333333,67.375000
+2023,1,22,23,25,4,2,12,3600.040283,1.083333,67.500000
+2023,1,22,23,25,5,4,12,3700.041504,0.916667,66.625000
+2023,1,22,23,25,6,8,12,3600.040283,1.000000,66.875000
+2023,1,22,23,25,7,0,12,3740.041992,0.000000,66.000000
+2023,1,22,23,25,8,3,12,3700.041504,0.500000,65.625000
+2023,1,22,23,25,9,8,12,3680.041260,1.833333,67.125000
+2023,1,22,23,25,10,4,12,3680.041260,1.250000,67.750000
+2023,1,22,23,25,11,3,12,3700.041504,0.500000,67.125000
+2023,1,22,23,25,12,7,12,3700.041504,1.333333,67.250000
+2023,1,22,23,25,13,6,12,3600.040283,0.750000,66.822917
+2023,1,22,23,25,14,3,12,3720.041748,1.333333,67.375000
+2023,1,22,23,25,15,4,12,4525.050781,1.083333,67.250000
+2023,1,22,23,25,16,5,12,2880.032227,0.666667,67.375000
+2023,1,22,23,25,17,3,12,3720.041748,0.833333,67.000000
+2023,1,22,23,25,18,5,12,3720.041748,1.000000,67.500000
+2023,1,22,23,25,19,6,12,3700.041504,1.916667,68.250000
+2023,1,22,23,25,20,2,12,3640.041016,1.000000,68.875000
+2023,1,22,23,25,21,10,12,3660.041016,2.083333,71.250000
+2023,1,22,23,25,22,8,12,3700.041504,1.166667,70.875000
+2023,1,22,23,25,23,1,12,3700.041504,0.083333,69.625000
+2023,1,22,23,25,24,7,12,3700.041504,2.166667,69.875000
+2023,1,22,23,25,25,4,12,3680.041260,0.916667,68.625000
+2023,1,22,23,25,26,7,12,3700.041504,0.666667,67.875000
+2023,1,22,23,25,27,1,12,3700.041504,0.083333,66.750000
+2023,1,22,23,25,28,2,12,3700.041504,0.166667,66.750000
+2023,1,22,23,25,29,5,12,3640.041016,2.083333,68.250000
+2023,1,22,23,25,30,9,12,3680.041260,1.166667,68.375000
+2023,1,22,23,25,31,5,12,3700.041504,0.833333,67.500000
+2023,1,22,23,25,32,1,12,3700.041504,0.500000,67.125000
+2023,1,22,23,25,33,1,12,3700.041504,0.083333,66.000000
+2023,1,22,23,25,34,3,12,3680.041260,1.333333,68.125000
+2023,1,22,23,25,35,2,12,3700.041504,0.583333,67.770833
+2023,1,22,23,25,36,3,12,3680.041260,0.583333,67.250000
+2023,1,22,23,25,37,2,12,3700.041504,0.166667,66.375000
+2023,1,22,23,25,38,2,12,3600.040283,0.250000,65.500000