Handle changes and corruption in Journal files

src/ElitePlayerJournal.net452.csproj

@@ -9,7 +9,7 @@
     <AppDesignerFolder>Properties</AppDesignerFolder>
     <RootNamespace>NZgeek.ElitePlayerJournal</RootNamespace>
     <AssemblyName>NZgeek.ElitePlayerJournal</AssemblyName>
-    <TargetFrameworkVersion>v4.5.2</TargetFrameworkVersion>
+    <TargetFrameworkVersion>v4.6.1</TargetFrameworkVersion>
     <FileAlignment>512</FileAlignment>
     <TargetFrameworkProfile />
   </PropertyGroup>

src/EventFactory.cs

@@ -19,7 +19,6 @@ internal static class EventFactory
             { EventType.Passengers, typeof(Events.Startup.Passengers) },
             { EventType.Progress, typeof(Events.Startup.Progress) },
             { EventType.Rank, typeof(Events.Startup.Rank) },
-
             // Travel
             { EventType.Docked, typeof(Events.Travel.Docked) },
             { EventType.DockingCancelled, typeof(Events.Travel.DockingEvent) },
@@ -28,49 +27,38 @@ internal static class EventFactory
             { EventType.DockingRequested, typeof(Events.Travel.DockingEvent) },
             { EventType.DockingTimeout, typeof(Events.Travel.DockingEvent) },
             { EventType.FSDJump, typeof(Events.Travel.FrameShiftJump) },
-            { EventType.Liftoff, typeof(Events.Travel.PlanetaryEvent) },
             { EventType.Location, typeof(Events.Travel.Location) },
-            { EventType.SupercruiseEntry, typeof(Events.Travel.SupercruiseEvent) },
             { EventType.SupercruiseExit, typeof(Events.Travel.SupercruiseExit) },
-            { EventType.Touchdown, typeof(Events.Travel.PlanetaryEvent) },
-            { EventType.Undocked, typeof(Events.Travel.DockingEvent) },
-            { EventType.StartJump, typeof(Events.Travel.StartJump) },
-
             // Combat
-            { EventType.Bounty, typeof(Events.Combat.BountyAward) },
-            { EventType.CapShipBond, typeof(Events.Combat.BondAward) },
-            { EventType.FactionKillBond, typeof(Events.Combat.BondAward) },
-
             // Exploration
             { EventType.Screenshot, typeof(Events.Exploration.Screenshot) },
-
             // Trade
-
             // Station Services
-
             // Powerplay
-
             // Other events
             { EventType.ReceiveText, typeof(Events.Other.ReceiveText) },
             { EventType.SendText, typeof(Events.Other.SendText) },
         };
 
-        private static readonly JsonSerializerSettings JsonSettings = new JsonSerializerSettings
-        {
-            DateParseHandling = DateParseHandling.DateTime,
-            DateFormatString = "yyyy'-'MM'-'dd'T'HH':'mm':'ss'Z'",
-            DateTimeZoneHandling = DateTimeZoneHandling.Utc,
-            FloatParseHandling = FloatParseHandling.Decimal,
-            MissingMemberHandling = MissingMemberHandling.Ignore,
-            NullValueHandling = NullValueHandling.Ignore,
-        };
-
         public static Event CreateEvent(JournalFile journalFile, int lineNumber, string eventData)
         {
-            var basicEvent = JsonConvert.DeserializeObject<EventBase>(eventData, JsonSettings);
+            EventBase basicEvent;
+            try
+            {
+                basicEvent = JsonConvert.DeserializeObject<EventBase>(eventData);
+            }
+            catch (JsonReaderException)
+            {
+                // Can occur if the line of JSON is corrupt or incomplete, not sure why this happens but it does :-(
+                // Ignore this line and carry on.
+                Console.WriteLine("    WARNING: Corrupt JSON line detected, skipping. " + eventData);
+                return null;
+            }
+
+            if (basicEvent == null) return null; // This happens when the log file contains bad data or has nulls at EOF.
 
             var resultEvent = CreateEvent(journalFile, lineNumber, basicEvent.Type);
-            resultEvent.LoadJson(eventData, JsonSettings);
+            resultEvent.LoadJson(eventData);
             return resultEvent;
         }
 

src/Events/Event.cs

@@ -1,6 +1,5 @@
 using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
-using NZgeek.ElitePlayerJournal.Converters;
 using System;
 using System.Collections.Generic;
 
@@ -10,8 +9,14 @@ namespace NZgeek.ElitePlayerJournal.Events
     public class Event : EventBase, IComparable<Event>
     {
         public Event()
+            : this(EventType.Unknown)
+        {
+        }
+
+        protected Event(EventType eventType)
         {
             Timestamp = DateTime.UtcNow;
+            RawType = eventType.ToString();
             UnmappedValues = new Dictionary<string, JToken>();
         }
 
@@ -22,14 +27,14 @@ public Event()
         public Journal Journal => JournalFile?.Journal;
 
         [JsonProperty(PropertyName = "timestamp")]
-        public DateTime Timestamp { get; private set; }
+        public DateTime Timestamp { get; set; }
 
         [JsonExtensionData]
         protected IDictionary<string, JToken> UnmappedValues { get; }
 
         public override string ToString() => Type == EventType.Unknown
-            ? $"[{Timestamp:yyyy'-'MM'-'dd' 'HH':'mm':'ss}] <<{RawType}>>"
-            : $"[{Timestamp:yyyy'-'MM'-'dd' 'HH':'mm':'ss}] {Type}";
+            ? $"[{Timestamp:yyyyMMdd-HHmmss}] <<{RawType}>>"
+            : $"[{Timestamp:yyyyMMdd-HHmmss}] {Type}";
 
         public int CompareTo(Event other)
         {
@@ -44,10 +49,22 @@ public int CompareTo(Event other)
             return Comparer<EventType>.Default.Compare(Type, other.Type);
         }
 
-        internal void LoadJson(string json, JsonSerializerSettings settings)
+        public void LoadJson(string json)
         {
             UnmappedValues.Clear();
-            JsonConvert.PopulateObject(json, this, settings);
+            try
+            {
+                JsonConvert.PopulateObject(json, this);
+            }
+            catch (JsonSerializationException ex)
+            {
+                if (ex.InnerException is ArgumentException && ex.InnerException.Message == "An item with the same key has already been added.")
+                {
+                    Console.WriteLine("    WARNING: JSON with two or more blanked keyed properties detected. " + json);
+                    return;
+                }
+                throw;
+            }
         }
 
         protected string GetLocalisableText(string key)

src/Events/Travel/Docked.cs

@@ -10,12 +10,6 @@ public class Docked : DockingEvent, ISystemEvent
         [JsonProperty("StationType")]
         public string StationType { get; private set; }
 
-        [JsonProperty("StationFaction")]
-        public string StationFaction { get; private set; }
-
-        [JsonProperty("FactionState")]
-        public string StationFactionState { get; private set; }
-
         public string StationAllegiance => GetLocalisableText("StationAllegiance");
 
         public string StationEconomy => GetLocalisableText("StationEconomy");

src/Events/Travel/FrameShiftJump.cs

@@ -11,7 +11,7 @@ public class FrameShiftJump : LocationBase
         public decimal FuelUsed { get; private set; }
 
         [JsonProperty("FuelLevel")]
-        public decimal FuelLevel { get; private set; }
+        public decimal FuelLevel { get; set; }
 
         public override string ToString() => $"{base.ToString()} => {Distance}Ly ({FuelUsed}T fuel used, {FuelLevel}T remaining)";
     }

src/Events/Travel/LocationBase.cs

@@ -1,5 +1,4 @@
 using Newtonsoft.Json;
-using NZgeek.ElitePlayerJournal.Events.Types;
 
 namespace NZgeek.ElitePlayerJournal.Events.Travel
 {
@@ -21,15 +20,6 @@ public abstract class LocationBase : Event, ISystemEvent
 
         public string SystemSecurity => GetLocalisableText("SystemSecurity");
 
-        [JsonProperty("SystemFaction")]
-        public string SystemFaction { get; private set; }
-
-        [JsonProperty("FactionState")]
-        public string SystemFactionState { get; private set; }
-
-        [JsonProperty("Factions")]
-        public SystemFaction[] Factions { get; private set; }
-
         public override string ToString() => $"{base.ToString()} @ {SystemName}";
     }
 }

src/Journal.cs

@@ -16,6 +16,9 @@ public Journal()
 
             var pictures = Environment.GetFolderPath(Environment.SpecialFolder.MyPictures);
             ScreenShotFolder = Path.Combine(pictures, "Frontier Developments", "Elite Dangerous");
+
+            Console.WriteLine("Journal Folder:    " + JournalFolder);
+            Console.WriteLine("Screenshot Folder: " + ScreenShotFolder);
         }
 
         public string JournalFolder { get; set; }
@@ -52,8 +55,10 @@ public IEnumerable<Event> FindEvents(DateTime fromDate, DateTime toDate, params
         {
             if (eventTypes == null) throw new ArgumentNullException(nameof(eventTypes));
 
+            Console.WriteLine("Action: Find events - " + string.Join(", ", eventTypes));
             foreach (var journalFile in _journalFiles)
             {
+                Console.WriteLine("    searching " + journalFile.File.Name);
                 var firstPossibleEvent = journalFile.GameStarted;
                 var lastPossibleEvent = firstPossibleEvent + TimeSpan.FromDays(1);
 

src/JournalFile.cs

@@ -70,7 +70,7 @@ private void LoadEvents()
                             continue;
 
                         var gameEvent = EventFactory.CreateEvent(this, lineNumber, eventData);
-                        events.Add(gameEvent, gameEvent);
+                        if (gameEvent != null) events.Add(gameEvent, gameEvent);
                     }
                 }
             }