Statistical analysis working for generator runs

client/build.gradle

@@ -42,6 +42,8 @@ dependencies {
     // TODO do we need this dependency?
     implementation group: 'biz.paluch.logging', name: 'logstash-gelf', version: '1.15.1'
 
+    // https://mvnrepository.com/artifact/com.google.guava/guava
+    implementation group: 'com.google.guava', name: 'guava', version: '33.3.1-jre'
 }
 
 shadowJar {

client/src/main/java/me/retrodaredevil/solarthing/program/subprogram/analyze/AnalyzeMain.java

@@ -12,6 +12,8 @@
 import me.retrodaredevil.solarthing.packets.collection.DefaultInstanceOptions;
 import me.retrodaredevil.solarthing.program.subprogram.analyze.analyzers.generator.GeneratorRunAnalyzer;
 import me.retrodaredevil.solarthing.program.subprogram.analyze.analyzers.generator.entry.GeneratorRunEntry;
+import me.retrodaredevil.solarthing.program.subprogram.analyze.analyzers.generator.entry.GeneratorStatistics;
+import me.retrodaredevil.solarthing.program.subprogram.analyze.statistics.fx.FXStatisticCollection;
 import org.apache.commons.cli.CommandLine;
 import org.apache.commons.cli.CommandLineParser;
 import org.apache.commons.cli.DefaultParser;
@@ -30,6 +32,7 @@
 import java.time.format.DateTimeFormatter;
 import java.time.format.DateTimeParseException;
 import java.util.List;
+import java.util.stream.Collectors;
 
 public final class AnalyzeMain {
 	private AnalyzeMain() { throw new UnsupportedOperationException(); }
@@ -112,7 +115,18 @@ public static int analyze(String[] args) {
 		try {
 			List<GeneratorRunEntry> rows = executor.analyze(startTime, endTime);
 			for (GeneratorRunEntry entry : rows) {
-				System.out.println("Entry:  Start: " + entry.startTime().atZone(zoneId) + " End: " + entry.endTime().atZone(zoneId) + " duration: " + Duration.between(entry.startTime(), entry.endTime()));
+				GeneratorStatistics stats = entry.wholeStatistics();
+				String details = stats.getFXAddresses().stream()
+						.sorted()
+						.map(address -> {
+							FXStatisticCollection fxStats = stats.getStatistics(address);
+
+							return "\n\tFX" + address
+									+ "  Voltage: mean: " + fxStats.stats().inputVoltageRaw().mean() + " upper 99%: " + fxStats.percentiles().inputVoltageRaw().p99() + " lower 1%: " + fxStats.percentiles().inputVoltageRaw().p1()
+									;
+						})
+						.collect(Collectors.joining(""));
+				System.out.println("Entry:  Start: " + entry.startTime().atZone(zoneId) + " End: " + entry.endTime().atZone(zoneId) + " duration: " + Duration.between(entry.startTime(), entry.endTime()) + details);
 			}
 		} catch (AnalysisException e) {
 			e.getCause().printStackTrace(System.err);

client/src/main/java/me/retrodaredevil/solarthing/program/subprogram/analyze/analyzers/generator/GeneratorRunAnalyzer.java

@@ -7,6 +7,8 @@
 import me.retrodaredevil.solarthing.program.subprogram.analyze.Analyzer;
 import me.retrodaredevil.solarthing.program.subprogram.analyze.DataChunk;
 import me.retrodaredevil.solarthing.program.subprogram.analyze.analyzers.generator.entry.GeneratorRunEntry;
+import me.retrodaredevil.solarthing.program.subprogram.analyze.analyzers.generator.entry.GeneratorStatistics;
+import me.retrodaredevil.solarthing.program.subprogram.analyze.statistics.fx.FXStatisticAnalysis;
 import me.retrodaredevil.solarthing.solar.outback.fx.ACMode;
 import me.retrodaredevil.solarthing.solar.outback.fx.FXStatusPacket;
 
@@ -39,9 +41,15 @@ public GeneratorRunAnalyzer(DefaultInstanceOptions defaultInstanceOptions, int f
 	private GeneratorRunEntry analyzeRow(List<InstancePacketGroup> packets) {
 		var first = packets.get(0);
 		var last = packets.get(packets.size() - 1);
+		Instant startTime = Instant.ofEpochMilli(first.getDateMillis());
+		Instant endTime = Instant.ofEpochMilli(last.getDateMillis());
 		return new GeneratorRunEntry(
-				Instant.ofEpochMilli(first.getDateMillis()),
-				Instant.ofEpochMilli(last.getDateMillis())
+				startTime,
+				endTime,
+				new GeneratorStatistics(
+						startTime, endTime,
+						FXStatisticAnalysis.analyze(packets)
+				)
 		);
 	}
 

client/src/main/java/me/retrodaredevil/solarthing/program/subprogram/analyze/analyzers/generator/entry/GeneratorRunEntry.java

@@ -17,6 +17,11 @@
  */
 public record GeneratorRunEntry(
 		Instant startTime,
-		Instant endTime
+		Instant endTime,
+		GeneratorStatistics wholeStatistics
+//		GeneratorStatistics acUseStatistics,
+//		List<GeneratorStatistics> acUseStatisticList,
+//		List<GeneratorStatistics> acDropStatistics,
+//		Map<OperationalMode, List<GeneratorStatistics>> operationalModeToStatisticsMap
 ) {
 }

client/src/main/java/me/retrodaredevil/solarthing/program/subprogram/analyze/analyzers/generator/entry/GeneratorStatistics.java

@@ -1,16 +1,33 @@
 package me.retrodaredevil.solarthing.program.subprogram.analyze.analyzers.generator.entry;
 
+import me.retrodaredevil.solarthing.program.subprogram.analyze.statistics.fx.FXStatisticCollection;
+import me.retrodaredevil.solarthing.solar.outback.OutbackIdentifier;
+
 import java.time.Instant;
+import java.util.Collection;
+import java.util.Map;
 
 /**
  * Represents statistics of some time period of a generator run, not necessarily the entire generator run
- *
- * @deprecated Not yet implemented
  */
-@Deprecated
 public record GeneratorStatistics(
 		Instant startTime,
 		Instant endTime,
-		float averageBatteryVoltage,
-		float average
-) {}
+//		float averageBatteryVoltage,
+//		float batteryVoltageStandardDeviation,
+		Map<Integer, FXStatisticCollection> fxStatistics
+) {
+	public Collection<Integer> getFXAddresses() {
+		return fxStatistics.keySet();
+	}
+	public FXStatisticCollection getStatistics(int fxAddress) {
+		FXStatisticCollection r = fxStatistics.get(fxAddress);
+		if (r == null) {
+			throw new IllegalArgumentException("Unknown FX address: " + fxAddress);
+		}
+		return r;
+	}
+	public FXStatisticCollection getStatistics(OutbackIdentifier outbackIdentifier) {
+		return getStatistics(outbackIdentifier.getAddress());
+	}
+}

client/src/main/java/me/retrodaredevil/solarthing/program/subprogram/analyze/analyzers/package-info.java

@@ -1,4 +1,5 @@
 /**
  * Contains implementations of {@link me.retrodaredevil.solarthing.program.subprogram.analyze.Analyzer}
+ * and contains data classes (usually records) to represent each Analyzer's entry/result.
  */
 package me.retrodaredevil.solarthing.program.subprogram.analyze.analyzers;

client/src/main/java/me/retrodaredevil/solarthing/program/subprogram/analyze/statistics/BasicStatistics.java

@@ -0,0 +1,7 @@
+package me.retrodaredevil.solarthing.program.subprogram.analyze.statistics;
+
+public record BasicStatistics(
+		float average,
+		float maximum
+) {
+}

client/src/main/java/me/retrodaredevil/solarthing/program/subprogram/analyze/statistics/CommonPercentiles.java

@@ -0,0 +1,22 @@
+package me.retrodaredevil.solarthing.program.subprogram.analyze.statistics;
+
+import com.google.common.math.Quantiles;
+
+import java.util.List;
+import java.util.Map;
+
+public record CommonPercentiles(
+		double p0_1, double p1, double p10,
+		double p90, double p99, double p99_9
+) {
+	private static final Quantiles.ScaleAndIndexes PERCENTILE_1000 = Quantiles.scale(1000).indexes(1, 10, 100, 900, 990, 999);
+
+	public static CommonPercentiles fromDataset(List<? extends Number> dataset) {
+		Map<Integer, Double> result = PERCENTILE_1000.compute(dataset);
+		return new CommonPercentiles(
+				result.get(1), result.get(10), result.get(100),
+				result.get(900), result.get(990), result.get(999)
+		);
+	}
+
+}

client/src/main/java/me/retrodaredevil/solarthing/program/subprogram/analyze/statistics/fx/FXStatistic.java

@@ -0,0 +1,94 @@
+package me.retrodaredevil.solarthing.program.subprogram.analyze.statistics.fx;
+
+import java.util.function.BiConsumer;
+import java.util.function.Function;
+import java.util.function.Supplier;
+
+/**
+ * Represents a collection of <em>some statistic</em> over some period of time for a single FX.
+ * {@link me.retrodaredevil.solarthing.solar.outback.fx.FXStatusPacket}
+ *
+ * Depending on what was happening during the time period this represents (AC Use, AC Drop, Silent, Bulk, the entire generator run time)
+ * or the type of statistic this is, some properties may not be useful.
+ * Usually many of these properties become more useful for the shorter periods of time that are focused on.
+ *
+ */
+public record FXStatistic<T>(
+		T batteryVoltage,
+
+		T inverterCurrentRaw,
+		T inverterWattage,
+
+		T chargerCurrentRaw,
+		T chargerWattage,
+
+		T buyCurrentRaw,
+		T buyWattage,
+
+		T sellCurrentRaw,
+		T sellWattage,
+
+		T inputVoltageRaw,
+		T outputVoltageRaw
+) {
+
+	public static <T> FXStatistic<T> initUsing(Supplier<T> initializer) {
+		return new FXStatistic<>(
+				initializer.get(),
+
+				initializer.get(),
+				initializer.get(),
+
+				initializer.get(),
+				initializer.get(),
+
+				initializer.get(),
+				initializer.get(),
+
+				initializer.get(),
+				initializer.get(),
+
+				initializer.get(),
+				initializer.get()
+		);
+	}
+
+	public <U> FXStatistic<U> convert(Function<T, U> converter) {
+		return new FXStatistic<>(
+				converter.apply(batteryVoltage),
+
+				converter.apply(inverterCurrentRaw),
+				converter.apply(inverterWattage),
+
+				converter.apply(chargerCurrentRaw),
+				converter.apply(chargerWattage),
+
+				converter.apply(buyCurrentRaw),
+				converter.apply(buyWattage),
+
+				converter.apply(sellCurrentRaw),
+				converter.apply(sellWattage),
+
+				converter.apply(inputVoltageRaw),
+				converter.apply(outputVoltageRaw)
+		);
+	}
+	public <U> void apply(FXStatistic<U> other, BiConsumer<T, U> function) {
+		function.accept(batteryVoltage, other.batteryVoltage);
+
+		function.accept(inverterCurrentRaw, other.inverterCurrentRaw);
+		function.accept(inverterWattage, other.inverterWattage);
+
+		function.accept(chargerCurrentRaw, other.chargerCurrentRaw);
+		function.accept(chargerWattage, other.chargerWattage);
+
+		function.accept(buyCurrentRaw, other.buyCurrentRaw);
+		function.accept(buyWattage, other.buyWattage);
+
+		function.accept(sellCurrentRaw, other.sellCurrentRaw);
+		function.accept(sellWattage, other.sellWattage);
+
+		function.accept(inputVoltageRaw, other.inputVoltageRaw);
+		function.accept(outputVoltageRaw, other.outputVoltageRaw);
+	}
+}

client/src/main/java/me/retrodaredevil/solarthing/program/subprogram/analyze/statistics/fx/FXStatisticAnalysis.java

@@ -0,0 +1,63 @@
+package me.retrodaredevil.solarthing.program.subprogram.analyze.statistics.fx;
+
+import com.google.common.math.Stats;
+import me.retrodaredevil.solarthing.packets.Packet;
+import me.retrodaredevil.solarthing.packets.collection.InstancePacketGroup;
+import me.retrodaredevil.solarthing.program.subprogram.analyze.statistics.CommonPercentiles;
+import me.retrodaredevil.solarthing.solar.outback.fx.FXStatusPacket;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public final class FXStatisticAnalysis {
+	private FXStatisticAnalysis() { throw new UnsupportedOperationException(); }
+
+	/**
+	 *
+	 * @param packets A list of packet groups, where each packet group must have at least one FX packet.
+	 * @return A map of Outback address to {@link FXStatisticCollection} for that given FX
+	 */
+	public static Map<Integer, FXStatisticCollection> analyze(List<InstancePacketGroup> packets) {
+		Map<Integer, FXStatistic<List<Number>>> accumulators = new HashMap<>();
+		for (InstancePacketGroup packetGroup : packets) {
+			for (Packet packet : packetGroup.getPackets()) {
+				if (packet instanceof FXStatusPacket fx) {
+					int address = fx.getAddress();
+					FXStatistic<List<Number>> accumulator = accumulators.computeIfAbsent(address, _address -> FXStatistic.initUsing(ArrayList::new));
+					accumulator.apply(packetToStatistic(fx), List::add);
+				}
+			}
+		}
+		Map<Integer, FXStatisticCollection> statsMap = new HashMap<>();
+		for (Map.Entry<Integer, FXStatistic<List<Number>>> entry : accumulators.entrySet()) {
+			FXStatistic<Stats> stats = entry.getValue().convert(Stats::of);
+			FXStatistic<CommonPercentiles> percentiles = entry.getValue().convert(CommonPercentiles::fromDataset);
+
+			statsMap.put(entry.getKey(), new FXStatisticCollection(stats, percentiles));
+		}
+		return statsMap;
+	}
+
+	private static FXStatistic<Number> packetToStatistic(FXStatusPacket packet) {
+		return new FXStatistic<>(
+				packet.getBatteryVoltage(),
+
+				packet.getInverterCurrentRaw(),
+				packet.getInverterWattage(),
+
+				packet.getChargerCurrentRaw(),
+				packet.getChargerWattage(),
+
+				packet.getBuyCurrentRaw(),
+				packet.getBuyWattage(),
+
+				packet.getSellCurrentRaw(),
+				packet.getSellWattage(),
+
+				packet.getInputVoltageRaw(),
+				packet.getOutputVoltageRaw()
+		);
+	}
+}

client/src/main/java/me/retrodaredevil/solarthing/program/subprogram/analyze/statistics/fx/FXStatisticCollection.java

@@ -0,0 +1,10 @@
+package me.retrodaredevil.solarthing.program.subprogram.analyze.statistics.fx;
+
+import com.google.common.math.Stats;
+import me.retrodaredevil.solarthing.program.subprogram.analyze.statistics.CommonPercentiles;
+
+public record FXStatisticCollection(
+		FXStatistic<Stats> stats,
+		FXStatistic<CommonPercentiles> percentiles
+) {
+}

client/src/main/java/me/retrodaredevil/solarthing/program/subprogram/analyze/statistics/package-info.java

@@ -0,0 +1,4 @@
+/**
+ * Contains utilities to analyze a set of data and immutable data carriers representing the results of statistical analysis.
+ */
+package me.retrodaredevil.solarthing.program.subprogram.analyze.statistics;

client/src/main/java/me/retrodaredevil/solarthing/program/subprogram/run/OutbackMateMain.java

@@ -40,7 +40,6 @@
 import java.io.InputStream;
 import java.time.Duration;
 import java.util.*;
-import java.util.stream.Collectors;
 
 import static java.util.Objects.requireNonNull;
 
@@ -68,7 +67,7 @@ public static int connectMate(MateProgramOptions options, boolean isValidate) th
 			PacketListReceiverHandlerBundle bundle = handlersResult.getBundle();
 			List<DataRequesterResult> dataRequesterResults = options.getDataRequesterList().stream()
 					.map(dataRequester -> dataRequester.create(new RequestObject(bundle.getEventHandler().getPacketListReceiverAccepter())))
-					.collect(Collectors.toList());
+					.toList();
 
 
 			List<PacketListReceiver> packetListReceiverList = new ArrayList<>(Arrays.asList(