Merge pull request #42 from lambdaisland/laurence/add-remove-unchanged

add remove-unchanged API
lambdaisland · Mar 30, 2023 · 34d5e17 · 34d5e17
2 parents 61440d4 + 41d3153
commit 34d5e17
Show file tree

Hide file tree

Showing 10 changed files with 250 additions and 14 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -3,6 +3,8 @@
 ## Added
 
 - Enable print tests in babashka
+- Add a `lambdaisland.deep-diff2/minimize` function, which removes any items
+  that haven't changed from the diff.
 
 ## Fixed
 

diff --git a/README.md b/README.md
@@ -88,6 +88,21 @@ For fine grained control you can create a custom Puget printer, and supply it to
 
 For more advanced uses like incorporating diffs into your own Fipp documents, see `lambdaisland.deep-diff2.printer/format-doc`, `lambdaisland.deep-diff2.printer/print-doc`.
 
+### Minimizing
+
+If you are only interested in the changes, and not in any values that haven't
+changed, then you can use `ddiff/minimize` to return a more compact diff.
+
+This is especially useful for potentially large nested data structures, for
+example a JSON response coming from a web service.
+
+```clj
+(-> (ddiff/diff {:a "apple" :b "pear"} {:a "apple" :b "banana"})
+    ddiff/minimize
+    ddiff/pretty-print)
+;; {:b -"pear" +"banana"}
+```
+
 ### Print handlers for custom or built-in types
 
 In recent versions deep-diff2 initializes its internal copy of Puget with

diff --git a/package.json b/package.json
@@ -4,6 +4,6 @@
   "dependencies": {
     "react": "^16.13.1",
     "react-dom": "^16.13.1",
-    "ws": "^8.11.0"
+    "ws": "^8.13.0"
   }
 }
diff --git a/repl_sessions/poke.clj b/repl_sessions/poke.clj
@@ -0,0 +1,62 @@
+(ns repl-sessions.poke
+  (:require [lambdaisland.deep-diff2 :as ddiff]))
+
+(seq #{{:foo 1M} {:bar 2}}) ;; => ({:foo 1M} {:bar 2})
+(seq #{{:foo 1} {:bar 2}})  ;; => ({:bar 2} {:foo 1})
+
+(def d1 {{:foo 1M} {:bar 2}})
+(def d2 {{:foo 1} {:bar 2}})
+(ddiff/pretty-print (ddiff/diff d1 d2))
+;; #{+{:foo 1} -{:foo 1M} {:bar 2}}
+
+(def d1 #{{:foo 1M}})
+(def d2 #{{:foo 1}})
+(ddiff/pretty-print (ddiff/diff d1 d2))
+
+(-> (ddiff/diff {:a "apple" :b "pear"} {:a "apple" :b "banana"})
+    ddiff/minimize
+    ddiff/pretty-print)
+;; {:b -"pear" +"banana"}
+
+;; {:b -2 +3}
+
+[#{1.1197369622161879e-14 1.3019822841584656e-21 0.6875
+   #uuid "a907a7fe-d2eb-482d-b1cc-3acfc12daf55"
+   -30
+   :X/*!1:3
+   :u7*A/p?2IG5d*!Nl
+   :**d7ws
+   "ý"
+   "ÔB*àñS�¬ÚûV¡ç�¯±·á£H�
+                            �û?'V$ëY;CL�k-oOV"
+   !U-h_C*A7/x0_n1
+   A-*wn./o_?4w18-!
+   "ìêÜ¼à4�^¤mÐðkt�ê1_ò�· À�4\n@J\"2�9)cd-\t®"
+   y3W-2
+   #uuid "6d507164-f8b9-401d-8c44-d6b0e310c248"
+   "M"
+   :cy7-3
+   :w4/R.-s?9V5
+   #uuid "1bcb00c9-88b9-4eae-9fea-60600dfaefa0"
+   -20
+   #uuid "269ab6f9-f19d-4c9d-a0cb-51150e52e9f7"
+   -235024979
+   :O:m_9.9+A/N+usPa6.HA*G
+   228944.657438457
+   :x/w?
+   :__+o+sut9!t/?0l
+   "�â��«"
+   false
+   #uuid "b6295f83-8176-47b5-946e-466f74226629"
+   e3zQ!E*5
+   :T5rb
+   :++y:2
+   -7364
+   zG/ex23
+   "¡"
+   -4318364480
+   :D+?2?!/Hrc!jA7z_2
+   :z-I/!8Uq+d?
+   -0.5588235294117647
+   -0.5925925925925926
+   -0.8108108108108109}]
diff --git a/src/lambdaisland/deep_diff2.cljc b/src/lambdaisland/deep_diff2.cljc
@@ -1,6 +1,8 @@
 (ns lambdaisland.deep-diff2
-  (:require [lambdaisland.deep-diff2.diff-impl :as diff-impl]
-            [lambdaisland.deep-diff2.printer-impl :as printer-impl]))
+  (:require
+   [lambdaisland.deep-diff2.diff-impl :as diff-impl]
+   [lambdaisland.deep-diff2.printer-impl :as printer-impl]
+   [lambdaisland.deep-diff2.minimize-impl :as minimize]))
 
 (defn diff
   "Compare two values recursively.
@@ -41,3 +43,8 @@
    (-> diff
        (printer-impl/format-doc printer)
        (printer-impl/print-doc printer))))
+
+(defn minimize
+  "Return a minimal diff, removing any values that haven't changed."
+  [diff]
+  (minimize/minimize diff))
diff --git a/src/lambdaisland/deep_diff2/diff_impl.cljc b/src/lambdaisland/deep_diff2/diff_impl.cljc
@@ -97,11 +97,31 @@
          (diff-seq-insertions ins)
          (into []))))
 
+(defn diff-set [exp act]
+  (into
+   (into #{}
+         (map (fn [e]
+                (if (contains? act e)
+                  e
+                  (->Deletion e))))
+         exp)
+   (map ->Insertion)
+   (remove #(contains? exp %) act)))
+
+(let [exp {false 0, 0 0}
+      act {false 0, 0 0}
+      exp-ks (keys exp)
+      act-ks (concat (filter #(contains? (set (keys act)) %) exp-ks)
+                     (remove #(contains? (set exp-ks) %) (keys act)))
+      [del ins] (del+ins exp-ks act-ks)]
+  [del ins])
+(del+ins [0 false] [0 false])
+
 (defn diff-map [exp act]
   (first
    (let [exp-ks (keys exp)
-         act-ks (concat (filter (set (keys act)) exp-ks)
-                        (remove (set exp-ks) (keys act)))
+         act-ks (concat (filter #(contains? (set (keys act)) %) exp-ks)
+                        (remove #(contains? (set exp-ks) %) (keys act)))
          [del ins] (del+ins exp-ks act-ks)]
      (reduce
       (fn [[m idx] k]
@@ -162,10 +182,7 @@
 (extend-protocol Diff
   #?(:clj java.util.Set :cljs cljs.core/PersistentHashSet)
   (-diff-similar [exp act]
-    (let [exp-seq (seq exp)
-          act-seq (seq act)]
-      (set (diff-seq exp-seq (concat (filter act exp-seq)
-                                     (remove exp act-seq))))))
+    (diff-set exp act))
   #?@(:clj
       [java.util.List
        (-diff-similar [exp act] (diff-seq exp act))

diff --git a/src/lambdaisland/deep_diff2/minimize_impl.cljc b/src/lambdaisland/deep_diff2/minimize_impl.cljc
@@ -0,0 +1,47 @@
+(ns lambdaisland.deep-diff2.minimize-impl
+  "Provide API for manipulate the diff structure data "
+  (:require [clojure.walk :refer [postwalk]]
+            #?(:clj [lambdaisland.deep-diff2.diff-impl]
+               :cljs [lambdaisland.deep-diff2.diff-impl :refer [Mismatch Deletion Insertion]]))
+  #?(:clj (:import [lambdaisland.deep_diff2.diff_impl Mismatch Deletion Insertion])))
+
+(defn diff-item?
+  "Checks if x is a Mismatch, Deletion, or Insertion"
+  [x]
+  (or (instance? Mismatch x)
+      (instance? Deletion x)
+      (instance? Insertion x)))
+
+(defn has-diff-item?
+  "Checks if there are any diff items in x or sub-tree of x"
+  [x]
+  (or (diff-item? x)
+      (and (map? x) (some #(or (has-diff-item? (key %))
+                               (has-diff-item? (val %))) x))
+      (and (coll? x) (some has-diff-item? x))))
+
+(defn minimize
+  "Postwalk diff, removing values that are unchanged"
+  [diff]
+  (let [y (postwalk
+           (fn [x]
+             (cond
+               (map-entry? x)
+               ;; Either k or v of a map-entry contains/is? diff-item,
+               ;; keep the map-entry. Otherwise, remove it.
+               (when (or (has-diff-item? (key x))
+                         (has-diff-item? (val x)))
+                 x)
+
+               (map? x)
+               x
+
+               (coll? x)
+               (into (empty x) (filter has-diff-item?) x)
+
+               :else
+               x))
+           diff)]
+    (cond
+      (coll? y) y
+      :else     nil)))
diff --git a/test/lambdaisland/deep_diff2/diff_test.cljc b/test/lambdaisland/deep_diff2/diff_test.cljc
@@ -212,10 +212,16 @@
                          (gen/such-that (complement NaN?) gen/simple-type)))
 
 (defspec round-trip-diff 100
-  (prop/for-all [x gen-any-except-NaN
-                 y gen-any-except-NaN]
-    (let [diff (diff/diff x y)]
-      (= [x y] [(diff/left-undiff diff) (diff/right-undiff diff)]))))
+  (prop/for-all
+   [x gen-any-except-NaN
+    y gen-any-except-NaN]
+   (let [diff (diff/diff x y)]
+     (= [x y] [(diff/left-undiff diff) (diff/right-undiff diff)]))))
+
+(defspec diff-same-is-same 100
+  (prop/for-all
+   [x gen-any-except-NaN]
+   (= x (diff/diff x x))))
 
 (deftest diff-seq-test
   (is (= [(diff/->Insertion 1) 2 (diff/->Insertion 3)]

diff --git a/test/lambdaisland/deep_diff2/minimize_test.cljc b/test/lambdaisland/deep_diff2/minimize_test.cljc
@@ -0,0 +1,78 @@
+(ns lambdaisland.deep-diff2.minimize-test
+  (:require [clojure.test :refer [deftest testing is are]]
+            [lambdaisland.deep-diff2.diff-test :as diff-test]
+            [clojure.test.check.clojure-test :refer [defspec]]
+            [clojure.test.check.generators :as gen]
+            [clojure.test.check.properties :as prop]
+            [lambdaisland.deep-diff2.diff-impl :as diff]
+            [lambdaisland.deep-diff2 :as ddiff]))
+
+(deftest basic-strip-test
+  (testing "diff without minimize"
+    (let [x {:a 1 :b 2 :d {:e 1} :g [:e [:k 14 :g 15]]}
+          y {:a 1 :c 3 :d {:e 15} :g [:e [:k 14 :g 15]]}]
+      (is (= (ddiff/diff x y)
+             {:a 1
+              (diff/->Deletion :b) 2
+              :d {:e (diff/->Mismatch 1 15)}
+              :g [:e [:k 14 :g 15]]
+              (diff/->Insertion :c) 3}))))
+  (testing "diff with minimize"
+    (let [x {:a 1 :b 2 :d {:e 1} :g [:e [:k 14 :g 15]]}
+          y {:a 1 :c 3 :d {:e 15} :g [:e [:k 14 :g 15]]}]
+      (is (= (ddiff/minimize (ddiff/diff x y))
+             {(diff/->Deletion :b) 2
+              :d {:e (diff/->Mismatch 1 15)}
+              (diff/->Insertion :c) 3})))))
+
+(deftest minimize-on-diff-test
+  (testing "diffing atoms"
+    (testing "when different"
+      (is (= (ddiff/minimize
+              (ddiff/diff :a :b))
+             (diff/->Mismatch :a :b))))
+
+    (testing "when equal"
+      (is (= (ddiff/minimize
+              (ddiff/diff :a :a))
+             nil))))
+
+  (testing "diffing collections"
+    (testing "when different collection types"
+      (is (= (ddiff/minimize
+              (ddiff/diff [:a :b] #{:a :b}))
+             (diff/->Mismatch [:a :b] #{:a :b}))))
+
+    (testing "when equal with clojure set"
+      (is (= (ddiff/minimize
+              (ddiff/diff #{:a :b} #{:a :b}))
+             #{})))
+
+    (testing "when different with clojure set"
+      (is (= (ddiff/minimize
+              (ddiff/diff #{:a :b :c} #{:a :b :d}))
+             #{(diff/->Insertion :d) (diff/->Deletion :c)})))
+
+    (testing "when equal with clojure vector"
+      (is (= (ddiff/minimize
+              (ddiff/diff [:a :b] [:a :b]))
+             [])))
+
+    (testing "when equal with clojure hashmap"
+      (is (= (ddiff/minimize
+              (ddiff/diff {:a 1} {:a 1}))
+             {})))
+
+    (testing "when equal with clojure nesting vector"
+      (is (= (ddiff/minimize
+              (ddiff/diff [:a [:b :c :d]] [:a [:b :c :d]]))
+             [])))))
+
+;; "diff itself and minimize yields empty"
+(defspec diff-itself 100
+  (prop/for-all
+   [x diff-test/gen-any-except-NaN]
+   (if (coll? x)
+     (= (ddiff/minimize (ddiff/diff x x))
+        (empty x))
+     (nil? (ddiff/minimize (ddiff/diff x x))))))
diff --git a/tests.edn b/tests.edn
@@ -1,4 +1,6 @@
 #kaocha/v1
 {:tests [{:id :clj}
          {:id :cljs
-          :type :kaocha.type/cljs}]}
+          :type :kaocha.type/cljs}]
+ :kaocha/bindings {kaocha.stacktrace/*stacktrace-filters* []}
+ }