#1787 Enhances DMR CSBK message processing by auto-detecting alternat…

…e CRC mask values employed by the DMR network.

src/main/java/io/github/dsheirer/module/decode/dmr/DmrCrcMaskManager.java

@@ -24,20 +24,38 @@
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Date;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
+import java.util.SortedSet;
 import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.concurrent.TimeUnit;
+
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 /**
- * Manages alternate CRC masks that may be employed in a system.
+ * Manages alternate CRC masks that may be employed in a system.  Some vendors employ non-standard CRC checksum masks
+ * or initial fill values.  This utility tracks these non-standard values by observation count and time and accepts
+ * these alternate mask values to automatically correct or validate messages when those messages employ these
+ * alternate mask values.  Alternate mask values must be observed at least 3x times in a 2-minute window against the
+ * same message content, in order for the alternate mask value to be used.  Tracked values that do not get another
+ * observation count within a 2-minute window will be ejected from the cache and excluded as a valid CRC mask candidate.
+ *
+ * Notes: different alternate CRC mask values can be used independently in each timeslot.  Further, I've seen multiple
+ * CRC mask values being employed within the same timeslot.  This indicates that systems can use multiple CRC mask
+ * values against CSBK messages and some other function is involved in determining which CRC mask value to use when
+ * checking the CRC of the transmitted CSBK ... fun and games!
  */
 public class DmrCrcMaskManager
 {
     private Logger LOGGER = LoggerFactory.getLogger(DmrCrcMaskManager.class);
-    private Map<Integer, ResidualCrcMaskTracker> mCsbkResidualTrackerMap = new TreeMap<>();
-    private int mDominantMask = 0;
+    private Map<Integer, MaskTracker> mCsbkTrackerMapTS1 = new TreeMap<>();
+    private Map<Integer, MaskTracker> mCsbkTrackerMapTS2 = new TreeMap<>();
+    private Set<MaskTracker> mSortedTrackersTS1 = new TreeSet<>();
+    private Set<MaskTracker> mSortedTrackersTS2 = new TreeSet<>();
 
     /**
      * Constructs an instance
@@ -46,35 +64,27 @@ public DmrCrcMaskManager()
     {
     }
 
-    /**
-     * Creates a deep copy (ie clone) of all tracked values
-     * @return list of tracker clones.
-     */
-    public List<ResidualCrcMaskTracker> cloneTrackers()
-    {
-        List<ResidualCrcMaskTracker>trackers = new ArrayList<>();
-        for(ResidualCrcMaskTracker tracker: mCsbkResidualTrackerMap.values())
-        {
-            trackers.add(tracker.clone());
-        }
-
-        return trackers;
-    }
-
     /**
      * Logs the current tracked CRC values and counts.
      */
     public void log()
     {
-        List<ResidualCrcMaskTracker> trackers = new ArrayList<>(mCsbkResidualTrackerMap.values());
-        Collections.sort(trackers);
 
         StringBuilder sb = new StringBuilder();
-        sb.append("DMR CRC Mask Manager - Current Tracked Values\n");
-        for(ResidualCrcMaskTracker tracker: trackers)
+
+        sb.append("DMR CRC Mask Manager - TS1 Current Tracked Values\n");
+        for(MaskTracker tracker: mSortedTrackersTS1)
         {
             sb.append("\tTracked Value: ").append(String.format("0x%04X", tracker.getTrackedValue()));
-            sb.append(" Count:").append(tracker.getCount());
+            sb.append(" Count:").append(tracker.getObservationCount());
+            sb.append(" Last Observed: " + new Date(tracker.getLastUpdated())).append("\n");
+        }
+
+        sb.append("DMR CRC Mask Manager - TS2 Current Tracked Values\n");
+        for(MaskTracker tracker: mSortedTrackersTS2)
+        {
+            sb.append("\tTracked Value: ").append(String.format("0x%04X", tracker.getTrackedValue()));
+            sb.append(" Count:").append(tracker.getObservationCount());
             sb.append(" Last Observed: " + new Date(tracker.getLastUpdated())).append("\n");
         }
 
@@ -83,104 +93,139 @@ public void log()
 
     /**
      * Checks the CSBK message that has a failed CRC checksum to detect if an alternate CRC mask value is employed and
-     * if so, attempts to correct the message by using the alternate mask value once the alternate mask value is
-     * detected to be used consistently enough.
-     * @param csbk to re-check.
+     * if so, attempts to correct the message.
+     * @param csbk to re-check using the currently tracked mask value.
      */
     public void check(CSBKMessage csbk)
     {
         if(!csbk.isValid())
         {
-            int residual = CRCDMR.calculateResidual(csbk.getMessage(), 0, 80);
+            int alternateMask = CRCDMR.calculateResidual(csbk.getMessage(), 0, 80);
 
-            if(mCsbkResidualTrackerMap.containsKey(residual))
+            Map<Integer,MaskTracker> map;
+            Set<MaskTracker> sortedSet;
+
+            if(csbk.getTimeslot() == 1)
             {
-                mCsbkResidualTrackerMap.get(residual).increment();
+                map = mCsbkTrackerMapTS1;
+                sortedSet = mSortedTrackersTS1;
             }
             else
             {
-                mCsbkResidualTrackerMap.put(residual, new ResidualCrcMaskTracker(residual));
-            }
-
-            List<ResidualCrcMaskTracker> trackers = new ArrayList<>(mCsbkResidualTrackerMap.values());
-            Collections.sort(trackers);
-
-            if(trackers.size() > 5)
-            {
-                mCsbkResidualTrackerMap.remove(trackers.get(0).getTrackedValue());
+                map = mCsbkTrackerMapTS2;
+                sortedSet = mSortedTrackersTS2;
             }
 
-            if(trackers.size() > 0)
+            if(map.containsKey(alternateMask))
             {
-                ResidualCrcMaskTracker dominant = trackers.get(trackers.size() - 1);
+                MaskTracker matchingTracker = map.get(alternateMask);
+                matchingTracker.increment(csbk.getTimestamp());
 
-                if(dominant.isValid())
+                if(matchingTracker.isValid())
                 {
-                    mDominantMask = dominant.getTrackedValue();
+                    csbk.checkCRC(matchingTracker.getTrackedValue());
                 }
             }
-
-            if(mDominantMask != 0)
+            else
             {
-                csbk.checkCRC(mDominantMask);
-//
-//                if(csbk.isValid())
-//                {
-//                    LOGGER.info("CSBK fixed using alternate mask: " + Integer.toHexString(mDominantMask).toUpperCase());
-//                }
+                //Attempt to use other tracked mask values in order of observed count and also remove stale trackers
+                boolean found = false;
+                MaskTracker next;
+                Iterator<MaskTracker> it = sortedSet.iterator();
+                while(it.hasNext())
+                {
+                    next = it.next();
+
+                    //If the tracker is stale, remove it
+                    if(next.isStale(csbk.getTimestamp()))
+                    {
+                        it.remove();
+                        map.remove(next.getTrackedValue());
+                    }
+                    else
+                    {
+                        //If we haven't found a match yet, attempt to correct the message using this tracked mask value
+                        if(!found)
+                        {
+                            csbk.checkCRC(next.getTrackedValue());
+
+                            if(csbk.isValid())
+                            {
+                                found = true;
+                                next.increment(csbk.getTimestamp());
+                            }
+                        }
+                    }
+                }
+
+                //If we didn't find a perfect match or even a close match (with 1 bit error), create a new tracked value.
+                if(!found)
+                {
+                    MaskTracker tracker = new MaskTracker(alternateMask, csbk.getTimestamp());
+                    map.put(alternateMask, tracker);
+                    sortedSet.add(tracker);
+                }
             }
         }
     }
 
     /**
-     * Residual CRC mask value tracker.  Tracks the number of observances of a residual CRC calculated value to detect
-     * when an alternate CRC mask pattern is employed and automatically correct CRC values for messages.
+     * Alternate CRC mask value tracker.  Tracks the number of observances of the mask value to detect when an alternate
+     * CRC mask pattern is employed and can then be used to automatically correct CRC values for messages.
      */
-    public class ResidualCrcMaskTracker implements Comparable<ResidualCrcMaskTracker>
+    public class MaskTracker implements Comparable<MaskTracker>
     {
+        private static final long STALENESS_TIME_THRESHOLD = TimeUnit.MINUTES.toMillis(2);
+        private static final int STALENESS_COUNT_THRESHOLD = 100;
         private int mTrackedValue;
-        private int mCount;
+        private int mObservationCount;
+        private int mStalenessCount;
         private long mLastUpdated;
 
         /**
-         * Constructs an instance for the specified residual, sets the count to one and sets the last update to now.
+         * Constructs an instance for the specified mask, sets the observation count to one and sets the last update to
+         * the supplied timestamp.
          * @param residual value to track.
+         * @param timestamp for the observation
          */
-        public ResidualCrcMaskTracker(int residual)
+        public MaskTracker(int residual, long timestamp)
         {
             mTrackedValue = residual;
-            mCount = 1;
-            mLastUpdated = System.currentTimeMillis();
+            mObservationCount = 1;
+            mLastUpdated = timestamp;
         }
 
         /**
-         * Create a deep copy of this instance.
-         * @return cloned instance.
+         * Count of observations for this mask value.
+         * @return count
          */
-        public ResidualCrcMaskTracker clone()
+        public int getObservationCount()
         {
-            ResidualCrcMaskTracker clone = new ResidualCrcMaskTracker(getTrackedValue());
-            clone.mCount = mCount;
-            clone.mLastUpdated = mLastUpdated;
-            return clone;
+            return mObservationCount;
         }
 
         /**
-         * Count of number of times this residual has been observed.
-         * @return count
+         * Indicates that the residual value has been observed at least 3x times within the preceding 2 minute window,
+         * indicating that it is (likely) valid.
+         * @return true if the tracked mask value is valid.
          */
-        public int getCount()
+        public boolean isValid()
         {
-            return mCount;
+            return mObservationCount >= 3;
         }
 
         /**
-         * Indicates that the residual value has been observed at least 3x times indicating that it is possibly valid.
-         * @return
+         * Indicates if this tracker is stale, relative to the supplied timestamp.  Stale indicates that this tracked
+         * value hasn't been observed in the preceding 2 minutes, or has been checked for staleness more than 100 times
+         * without an updated observation.
+         * @param timestamp to compare for staleness check.
+         * @return true if this tracker is stale.
          */
-        public boolean isValid()
+        public boolean isStale(long timestamp)
         {
-            return mCount >= 3;
+            mStalenessCount++;
+            return mStalenessCount > STALENESS_COUNT_THRESHOLD ||
+                    Math.abs(timestamp - mLastUpdated) > STALENESS_TIME_THRESHOLD;
         }
 
         /**
@@ -192,38 +237,44 @@ public long getLastUpdated()
         }
 
         /**
-         * Residual tracked value.
-         * @return residual
+         * Tracked alternate mask value.
+         * @return tracked mask value
          */
         public int getTrackedValue()
         {
             return mTrackedValue;
         }
 
-        public void increment()
+        /**
+         * Increment the observation count and use the supplied timestamp as the last updated timestamp.
+         * @param timestamp for the observed message.
+         */
+        public void increment(long timestamp)
         {
-            mCount++;
-            //Prevent rollover by keeping value at max integer value.
-            if(mCount < 0)
+            mStalenessCount = 0;
+            mObservationCount++;
+            //Detect integer rollover and apply max integer value.
+            if(mObservationCount < 0)
             {
-                mCount = Integer.MAX_VALUE;
+                mObservationCount = Integer.MAX_VALUE;
             }
-            mLastUpdated = System.currentTimeMillis();
+            mLastUpdated = timestamp;
         }
 
         /**
-         * Custom sort order by count and then by last updated timestamp.
+         * Custom (reversed) sort order by largest count and then by latest updated timestamp.
          * @param o the object to be compared.
-         * @return
+         * @return comparison value.
          */
         @Override
-        public int compareTo(ResidualCrcMaskTracker o)
+        public int compareTo(MaskTracker o)
         {
-            int comparison = Integer.compare(getCount(), o.getCount());
+            //Multiply by -1 to apply a reverse ordering
+            int comparison = Integer.compare(getObservationCount(), o.getObservationCount()) * -1;
 
             if(comparison == 0)
             {
-                comparison = Long.compare(getLastUpdated(), o.getLastUpdated());
+                comparison = Long.compare(getLastUpdated(), o.getLastUpdated()) * -1;
             }
 
             return comparison;

src/main/java/io/github/dsheirer/module/decode/dmr/message/data/csbk/standard/announcement/AdjacentSiteInformation.java

@@ -151,6 +151,10 @@ public String toString()
         }
 
         sb.append("CC:").append(getSlotType().getColorCode());
+        if(hasRAS())
+        {
+            sb.append(" RAS:").append(getBPTCReservedBits());
+        }
         sb.append(" ").append(getSystemIdentityCode().getModel());
         sb.append(" NEIGHBOR NETWORK:").append(getNeighborSystemIdentityCode().getNetwork());
         sb.append(" SITE:").append(getNeighborSystemIdentityCode().getSite());