From 7a03331955f5161fd5966be6d64552e898fc8eeb Mon Sep 17 00:00:00 2001 From: Dennis Sheirer Date: Sun, 26 Nov 2023 08:15:25 -0500 Subject: [PATCH] #1787 Enhances DMR CSBK message processing by auto-detecting alternate CRC mask values employed by the DMR network. --- .../module/decode/dmr/DmrCrcMaskManager.java | 223 +++++++++++------- .../announcement/AdjacentSiteInformation.java | 4 + 2 files changed, 141 insertions(+), 86 deletions(-) diff --git a/src/main/java/io/github/dsheirer/module/decode/dmr/DmrCrcMaskManager.java b/src/main/java/io/github/dsheirer/module/decode/dmr/DmrCrcMaskManager.java index 481d92c1f..01e54dde3 100644 --- a/src/main/java/io/github/dsheirer/module/decode/dmr/DmrCrcMaskManager.java +++ b/src/main/java/io/github/dsheirer/module/decode/dmr/DmrCrcMaskManager.java @@ -24,20 +24,38 @@ import java.util.ArrayList; import java.util.Collections; import java.util.Date; +import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.SortedSet; import java.util.TreeMap; +import java.util.TreeSet; +import java.util.concurrent.TimeUnit; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** - * Manages alternate CRC masks that may be employed in a system. + * Manages alternate CRC masks that may be employed in a system. Some vendors employ non-standard CRC checksum masks + * or initial fill values. This utility tracks these non-standard values by observation count and time and accepts + * these alternate mask values to automatically correct or validate messages when those messages employ these + * alternate mask values. Alternate mask values must be observed at least 3x times in a 2-minute window against the + * same message content, in order for the alternate mask value to be used. Tracked values that do not get another + * observation count within a 2-minute window will be ejected from the cache and excluded as a valid CRC mask candidate. + * + * Notes: different alternate CRC mask values can be used independently in each timeslot. Further, I've seen multiple + * CRC mask values being employed within the same timeslot. This indicates that systems can use multiple CRC mask + * values against CSBK messages and some other function is involved in determining which CRC mask value to use when + * checking the CRC of the transmitted CSBK ... fun and games! */ public class DmrCrcMaskManager { private Logger LOGGER = LoggerFactory.getLogger(DmrCrcMaskManager.class); - private Map mCsbkResidualTrackerMap = new TreeMap<>(); - private int mDominantMask = 0; + private Map mCsbkTrackerMapTS1 = new TreeMap<>(); + private Map mCsbkTrackerMapTS2 = new TreeMap<>(); + private Set mSortedTrackersTS1 = new TreeSet<>(); + private Set mSortedTrackersTS2 = new TreeSet<>(); /** * Constructs an instance @@ -46,35 +64,27 @@ public DmrCrcMaskManager() { } - /** - * Creates a deep copy (ie clone) of all tracked values - * @return list of tracker clones. - */ - public List cloneTrackers() - { - Listtrackers = new ArrayList<>(); - for(ResidualCrcMaskTracker tracker: mCsbkResidualTrackerMap.values()) - { - trackers.add(tracker.clone()); - } - - return trackers; - } - /** * Logs the current tracked CRC values and counts. */ public void log() { - List trackers = new ArrayList<>(mCsbkResidualTrackerMap.values()); - Collections.sort(trackers); StringBuilder sb = new StringBuilder(); - sb.append("DMR CRC Mask Manager - Current Tracked Values\n"); - for(ResidualCrcMaskTracker tracker: trackers) + + sb.append("DMR CRC Mask Manager - TS1 Current Tracked Values\n"); + for(MaskTracker tracker: mSortedTrackersTS1) { sb.append("\tTracked Value: ").append(String.format("0x%04X", tracker.getTrackedValue())); - sb.append(" Count:").append(tracker.getCount()); + sb.append(" Count:").append(tracker.getObservationCount()); + sb.append(" Last Observed: " + new Date(tracker.getLastUpdated())).append("\n"); + } + + sb.append("DMR CRC Mask Manager - TS2 Current Tracked Values\n"); + for(MaskTracker tracker: mSortedTrackersTS2) + { + sb.append("\tTracked Value: ").append(String.format("0x%04X", tracker.getTrackedValue())); + sb.append(" Count:").append(tracker.getObservationCount()); sb.append(" Last Observed: " + new Date(tracker.getLastUpdated())).append("\n"); } @@ -83,104 +93,139 @@ public void log() /** * Checks the CSBK message that has a failed CRC checksum to detect if an alternate CRC mask value is employed and - * if so, attempts to correct the message by using the alternate mask value once the alternate mask value is - * detected to be used consistently enough. - * @param csbk to re-check. + * if so, attempts to correct the message. + * @param csbk to re-check using the currently tracked mask value. */ public void check(CSBKMessage csbk) { if(!csbk.isValid()) { - int residual = CRCDMR.calculateResidual(csbk.getMessage(), 0, 80); + int alternateMask = CRCDMR.calculateResidual(csbk.getMessage(), 0, 80); - if(mCsbkResidualTrackerMap.containsKey(residual)) + Map map; + Set sortedSet; + + if(csbk.getTimeslot() == 1) { - mCsbkResidualTrackerMap.get(residual).increment(); + map = mCsbkTrackerMapTS1; + sortedSet = mSortedTrackersTS1; } else { - mCsbkResidualTrackerMap.put(residual, new ResidualCrcMaskTracker(residual)); - } - - List trackers = new ArrayList<>(mCsbkResidualTrackerMap.values()); - Collections.sort(trackers); - - if(trackers.size() > 5) - { - mCsbkResidualTrackerMap.remove(trackers.get(0).getTrackedValue()); + map = mCsbkTrackerMapTS2; + sortedSet = mSortedTrackersTS2; } - if(trackers.size() > 0) + if(map.containsKey(alternateMask)) { - ResidualCrcMaskTracker dominant = trackers.get(trackers.size() - 1); + MaskTracker matchingTracker = map.get(alternateMask); + matchingTracker.increment(csbk.getTimestamp()); - if(dominant.isValid()) + if(matchingTracker.isValid()) { - mDominantMask = dominant.getTrackedValue(); + csbk.checkCRC(matchingTracker.getTrackedValue()); } } - - if(mDominantMask != 0) + else { - csbk.checkCRC(mDominantMask); -// -// if(csbk.isValid()) -// { -// LOGGER.info("CSBK fixed using alternate mask: " + Integer.toHexString(mDominantMask).toUpperCase()); -// } + //Attempt to use other tracked mask values in order of observed count and also remove stale trackers + boolean found = false; + MaskTracker next; + Iterator it = sortedSet.iterator(); + while(it.hasNext()) + { + next = it.next(); + + //If the tracker is stale, remove it + if(next.isStale(csbk.getTimestamp())) + { + it.remove(); + map.remove(next.getTrackedValue()); + } + else + { + //If we haven't found a match yet, attempt to correct the message using this tracked mask value + if(!found) + { + csbk.checkCRC(next.getTrackedValue()); + + if(csbk.isValid()) + { + found = true; + next.increment(csbk.getTimestamp()); + } + } + } + } + + //If we didn't find a perfect match or even a close match (with 1 bit error), create a new tracked value. + if(!found) + { + MaskTracker tracker = new MaskTracker(alternateMask, csbk.getTimestamp()); + map.put(alternateMask, tracker); + sortedSet.add(tracker); + } } } } /** - * Residual CRC mask value tracker. Tracks the number of observances of a residual CRC calculated value to detect - * when an alternate CRC mask pattern is employed and automatically correct CRC values for messages. + * Alternate CRC mask value tracker. Tracks the number of observances of the mask value to detect when an alternate + * CRC mask pattern is employed and can then be used to automatically correct CRC values for messages. */ - public class ResidualCrcMaskTracker implements Comparable + public class MaskTracker implements Comparable { + private static final long STALENESS_TIME_THRESHOLD = TimeUnit.MINUTES.toMillis(2); + private static final int STALENESS_COUNT_THRESHOLD = 100; private int mTrackedValue; - private int mCount; + private int mObservationCount; + private int mStalenessCount; private long mLastUpdated; /** - * Constructs an instance for the specified residual, sets the count to one and sets the last update to now. + * Constructs an instance for the specified mask, sets the observation count to one and sets the last update to + * the supplied timestamp. * @param residual value to track. + * @param timestamp for the observation */ - public ResidualCrcMaskTracker(int residual) + public MaskTracker(int residual, long timestamp) { mTrackedValue = residual; - mCount = 1; - mLastUpdated = System.currentTimeMillis(); + mObservationCount = 1; + mLastUpdated = timestamp; } /** - * Create a deep copy of this instance. - * @return cloned instance. + * Count of observations for this mask value. + * @return count */ - public ResidualCrcMaskTracker clone() + public int getObservationCount() { - ResidualCrcMaskTracker clone = new ResidualCrcMaskTracker(getTrackedValue()); - clone.mCount = mCount; - clone.mLastUpdated = mLastUpdated; - return clone; + return mObservationCount; } /** - * Count of number of times this residual has been observed. - * @return count + * Indicates that the residual value has been observed at least 3x times within the preceding 2 minute window, + * indicating that it is (likely) valid. + * @return true if the tracked mask value is valid. */ - public int getCount() + public boolean isValid() { - return mCount; + return mObservationCount >= 3; } /** - * Indicates that the residual value has been observed at least 3x times indicating that it is possibly valid. - * @return + * Indicates if this tracker is stale, relative to the supplied timestamp. Stale indicates that this tracked + * value hasn't been observed in the preceding 2 minutes, or has been checked for staleness more than 100 times + * without an updated observation. + * @param timestamp to compare for staleness check. + * @return true if this tracker is stale. */ - public boolean isValid() + public boolean isStale(long timestamp) { - return mCount >= 3; + mStalenessCount++; + return mStalenessCount > STALENESS_COUNT_THRESHOLD || + Math.abs(timestamp - mLastUpdated) > STALENESS_TIME_THRESHOLD; } /** @@ -192,38 +237,44 @@ public long getLastUpdated() } /** - * Residual tracked value. - * @return residual + * Tracked alternate mask value. + * @return tracked mask value */ public int getTrackedValue() { return mTrackedValue; } - public void increment() + /** + * Increment the observation count and use the supplied timestamp as the last updated timestamp. + * @param timestamp for the observed message. + */ + public void increment(long timestamp) { - mCount++; - //Prevent rollover by keeping value at max integer value. - if(mCount < 0) + mStalenessCount = 0; + mObservationCount++; + //Detect integer rollover and apply max integer value. + if(mObservationCount < 0) { - mCount = Integer.MAX_VALUE; + mObservationCount = Integer.MAX_VALUE; } - mLastUpdated = System.currentTimeMillis(); + mLastUpdated = timestamp; } /** - * Custom sort order by count and then by last updated timestamp. + * Custom (reversed) sort order by largest count and then by latest updated timestamp. * @param o the object to be compared. - * @return + * @return comparison value. */ @Override - public int compareTo(ResidualCrcMaskTracker o) + public int compareTo(MaskTracker o) { - int comparison = Integer.compare(getCount(), o.getCount()); + //Multiply by -1 to apply a reverse ordering + int comparison = Integer.compare(getObservationCount(), o.getObservationCount()) * -1; if(comparison == 0) { - comparison = Long.compare(getLastUpdated(), o.getLastUpdated()); + comparison = Long.compare(getLastUpdated(), o.getLastUpdated()) * -1; } return comparison; diff --git a/src/main/java/io/github/dsheirer/module/decode/dmr/message/data/csbk/standard/announcement/AdjacentSiteInformation.java b/src/main/java/io/github/dsheirer/module/decode/dmr/message/data/csbk/standard/announcement/AdjacentSiteInformation.java index 3baae7139..bb3a0c35a 100644 --- a/src/main/java/io/github/dsheirer/module/decode/dmr/message/data/csbk/standard/announcement/AdjacentSiteInformation.java +++ b/src/main/java/io/github/dsheirer/module/decode/dmr/message/data/csbk/standard/announcement/AdjacentSiteInformation.java @@ -151,6 +151,10 @@ public String toString() } sb.append("CC:").append(getSlotType().getColorCode()); + if(hasRAS()) + { + sb.append(" RAS:").append(getBPTCReservedBits()); + } sb.append(" ").append(getSystemIdentityCode().getModel()); sb.append(" NEIGHBOR NETWORK:").append(getNeighborSystemIdentityCode().getNetwork()); sb.append(" SITE:").append(getNeighborSystemIdentityCode().getSite());