switching to read-objs for add-source-to-interns-with-reader

project.clj

@@ -5,4 +5,5 @@
   :dependencies [[org.clojure/clojure "1.6.0"]
                  [org.clojure/tools.reader "0.8.13"]
                  [org.clojure/tools.analyzer.jvm "0.6.5"]
-                 [org.rksm/system-files "0.1.3-SNAPSHOT"]])
+                 [org.rksm/system-files "0.1.3-SNAPSHOT"]
+                 [com.keminglabs/cljx "0.6.0"]])

src/rksm/cloxp_source_reader/ast_reader.clj

@@ -37,7 +37,7 @@
   (let [read (read-objs src)
         forms (map :form read)
         ast (ana-jvm/analyze
-             (list forms)
+             forms
              {:context :eval, :locals {}, :ns namespace})
         defs (find-defs ast)]
     (merge-read-objs-with-ast defs read)))

src/rksm/cloxp_source_reader/core.clj

@@ -2,7 +2,8 @@
   (:require [clojure.tools.reader.reader-types :as trt]
             [clojure.tools.reader :as tr]
             [clojure.string :as s]
-            [rksm.system-files :refer (source-reader-for-ns)])
+            [rksm.system-files :refer (source-reader-for-ns)]
+            (cljx core rules))
   (:import (java.io LineNumberReader InputStreamReader PushbackReader)
            (clojure.lang RT)))
 
@@ -33,7 +34,9 @@
   :column} map. Note: this is more that the typical reader gives us."
   [rdr-or-src]
   ; FIXME this is hacked...
-  (let [rdr (trt/indexing-push-back-reader (trt/source-logging-push-back-reader rdr-or-src))]
+  (let [rdr (trt/indexing-push-back-reader
+             (trt/source-logging-push-back-reader
+              rdr-or-src))]
     (loop [result []]
       (let [start-line (trt/get-line-number rdr)
             start-column (trt/get-column-number rdr)]
@@ -46,72 +49,42 @@
                 src-lines (assoc (vec src-lines) 0 (nth first-line-ws-match 2))
                 src (s/join "\n" src-lines)
                 line (+ (count no-ws-lines) start-line)
-                column (+ start-column (count (second first-line-ws-match)))]
+                column (+ start-column (count (second first-line-ws-match)))
+                meta (meta o)
+                def? (def? o)
+                name (if def? (name-of-def o))]
             (when (= \newline (trt/peek-char rdr))
               (trt/read-char rdr)
               (purge-string! rdr))
-            (recur (conj result {:form (if (def? o)
-                                         (with-meta o (assoc (meta o) :source src))
-                                         o)
-                                 :source src
-                                 :line line
-                                 :column column})))
+            (recur (conj result
+                         (merge {:form o, :source src,
+                                 :line line, :column column,
+                                 :end-line (trt/get-line-number rdr),
+                                 :end-column (trt/get-column-number rdr)}
+                                (if def?
+                                  {:form (with-meta o (assoc meta :source src)),
+                                   :name name})))))
           result)))))
 
-(defn- read-next-obj
-  "follows the reader while it core/reads an object and returns the string in
-  range of what was read"
-  [rdr]
-  (let [text (StringBuilder.) 
-        pbr (proxy [PushbackReader] [rdr]
-                   (read []
-                         (let [i (proxy-super read)]
-                           (if (> i -1) (.append text (char i)))
-                           i)))]
-    (if (= :unknown *read-eval*)
-      (throw (IllegalStateException. "Unable to read source while *read-eval* is :unknown."))
-      (let [form (tr/read (PushbackReader. pbr) false nil)
-            source (str text)]
-        {:form form, :source source}))))
-
-(defn- read-entity-source
-  "goes forward in line numbering reader until line of entity is reached and
-  reads that as an object"
-  [{:keys [lrdr sources], :as record} {expected-name :name, :as meta-entity}]
-  lrdr
-  (or (when-let [line (:line meta-entity)]
-        (dotimes [_ (dec (- line (.getLineNumber lrdr)))] (.readLine lrdr))
-        (let [{:keys [form source]} (read-next-obj lrdr)
-              name (name-of-def form)
-              new-meta (merge meta-entity {:source source})]
-          (if (or (nil? expected-name) (= name (:name meta-entity)))
-            (update-in record [:sources] conj new-meta)
-            record)))
-      record))
-
 (defn add-source-to-interns-with-reader
   "interns are supposed to be meta-data-like maps, at least including :line for
   the entity to be read"
-  [rdr interns]
-  {:pre [every? #(number? (:line %)) interns]}
-  (let [interns (sort-by :line interns)
-        source-data {:lrdr (LineNumberReader. rdr), :sources []}]
-    (if-let [result (reduce read-entity-source source-data interns)]
-      (:sources result)
-      interns)))
-
-(defn add-source-to-interns
-  "alternative for `source-for-symbol`. Instead of using clojure.repl this
-  functions uses the classloader info provided by system-files to find more
-  recent versions of the source.
-  NOTE: If there are multiple versions a lib on the classpath than it is
-  possible that this function will retrieve code that i not actually in the
-  system! (and the system meta data will clash with the actual file contents)"
-  [ns interns & [ns-file-path]]
-  (if-let [rdr (source-reader-for-ns ns ns-file-path)]
-    (with-open [rdr rdr]
-      (add-source-to-interns-with-reader rdr interns))
-    interns))
+  [rdr interns & [opts]]
+  {:pre [every? #((or (contains? (:line %)) 
+                      (contains? (:name %)))) interns]}
+  (let [objs (read-objs (slurp rdr))
+        obj-map (apply sorted-map (mapcat (juxt :name identity) objs))]
+    (sort-by
+     :line
+     (keep (fn [{n :name l :line c :column, :as meta-entity}]
+             (if-let [{s :source}
+                      (if n
+                        (get obj-map n)
+                        (->> objs
+                          (filter (fn [{c2 :column, l2 :line}] (and (= c c2) (= l l2))))
+                          first))]
+               (assoc meta-entity :source s)))
+           interns))))
 
 (defn add-source-to-interns
   "alternative for `source-for-symbol`. Instead of using clojure.repl this
@@ -120,10 +93,20 @@
   NOTE: If there are multiple versions a lib on the classpath than it is
   possible that this function will retrieve code that i not actually in the
   system! (and the system meta data will clash with the actual file contents)"
-  [ns interns & [ns-file-path]]
-  (if-let [rdr (source-reader-for-ns ns ns-file-path)]
-    (with-open [rdr rdr]
-      (add-source-to-interns-with-reader rdr interns))
+  [ns interns & [{:keys [file cljx?]} :as opts]]
+  [file cljx?]
+  (if-let [rdr (source-reader-for-ns ns file)]
+    (let [cljx? (or cljx? (and
+                           (nil? cljx?)
+                           (string? file)
+                           (re-find #"\.cljx$" file)))
+          rdr (if cljx? 
+                (->  (slurp rdr)
+                  (cljx.core/transform cljx.rules/clj-rules)
+                  java.io.StringReader.)
+                rdr)]
+      (with-open [rdr rdr]
+        (add-source-to-interns-with-reader rdr interns opts)))
     interns))
 
 ; -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-

test/rksm/cloxp_source_reader/test.clj

@@ -25,9 +25,12 @@
   (testing "simple read"
     (is (= [{:form '(ns rksm.cloxp-source-reader.test.dummy-3),
              :source "(ns rksm.cloxp-source-reader.test.dummy-3)",
-             :line 1,
-             :column 1}
-            {:form '(def x 23), :source "(def x 23)", :line 2, :column 3}]
+             :line 1, :column 1
+             :end-line 2, :end-column 1}
+            {:form '(def x 23), :source "(def x 23)",
+             :name 'x
+             :line 2, :column 3,
+             :end-line 3, :end-column 1}]
          (src-rdr/read-objs "(ns rksm.cloxp-source-reader.test.dummy-3)\n  (def x 23)\n")))))
 
 (deftest ast-reader-parsing
@@ -37,11 +40,15 @@
           expected [{:ns 'rksm.cloxp-source-reader.test.dummy-3,
                      :name 'foo,
                      :source "(defn foo [] `~23)",
-                     :line 4}
+                     :line 4
+                    ;  :column 1 :end-line 3, :end-column 1
+                     }
                     {:ns 'rksm.cloxp-source-reader.test.dummy-3,
                      :name 'b,
                      :source "(defmacro b [] `~23)"
-                     :line 2}]]
+                     :line 2,
+                    ;  :column 1, :end-line 3, :end-column 1
+                     }]]
       (is (= expected
              (map #(select-keys % [:name :ns :source :line])
                   (ast-rdr/read-and-parse src 'rksm.cloxp-source-reader.test.dummy-3)))))))
@@ -70,14 +77,14 @@
                (src-rdr/add-source-to-interns-with-reader source entities)))))
 
     (testing "more meta entities than source"
-      (is (= [{:source "(def x 23)" :column 1,:line 1} {:source "" :column 1,:line 6}]
+      (is (= [{:source "(def x 23)" :column 1,:line 1}]
              (let [entities [{:column 1,:line 1} {:column 1,:line 6}]
                    source (java.io.StringReader. "(def x 23)")]
                (src-rdr/add-source-to-interns-with-reader source entities)))))
 
     (testing "not entities in source"
       "this might be kind of unexpected but the reader des not care bout lines"
-      (is (= [{:source "(def y 24)" :column 1,:line 3} {:source "" :column 1,:line 6}]
+      (is (= [{:source "(def y 24)" :column 1,:line 3}]
              (let [entities [{:column 1,:line 3} {:column 1,:line 6}]
                    source (java.io.StringReader. "(def x 23)\n\n(def y 24)")]
                (src-rdr/add-source-to-interns-with-reader source entities)))))))