initial, extraction from system files, system navigator, cloxp trace

.gitignore

@@ -0,0 +1,3 @@
+/pom.xml
+/target/
+/.nrepl-port

project.clj

@@ -0,0 +1,5 @@
+(defproject org.rksm/cloxp-source-reader "0.1.0-SNAPSHOT"
+  :description "Source reading, parsing, and querying for cloxp."
+  :license "MIT"
+  :url "http://github.com/cloxp/cloxp-source-reader"
+  :dependencies [[org.clojure/clojure "1.6.0"]])

src/rksm/cloxp_source_reader/ast_reader.clj

@@ -0,0 +1,43 @@
+(ns rksm.cloxp-source-reader.ast-reader
+  (:require [clojure.tools.analyzer.ast :as ana-ast]
+            [clojure.tools.analyzer.jvm :as ana-jvm]
+            [clojure.set :as set]
+            [rksm.cloxp-source-reader.core :refer (read-objs)]))
+
+; -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
+; parsing via tools.analyzer
+
+(defn- find-defs
+  [expr]
+   (let [child-result (mapcat find-defs (ana-ast/children expr))
+         my-result (if (= :def (:op expr))
+                     (->> expr ((juxt :env #(hash-map :name (:name %)))) (apply merge)))]
+     (if my-result 
+       (conj child-result my-result)
+       child-result)))
+
+(defn- file-loc-id
+  [{line :line, column :column}]
+  {:line line, :column column})
+
+(defn- merge-read-objs-with-ast
+  [ast-defs read-objs]
+  (let [indexed-a (apply merge (map #(hash-map (file-loc-id %) %) ast-defs))
+        indexed-b (apply merge (map #(hash-map (file-loc-id %) %) read-objs))
+        ids-both (set/intersection (-> indexed-a keys set) (-> indexed-b keys set))
+        a (map (partial get indexed-a) ids-both)
+        b (map (partial get indexed-b) ids-both)]
+    (->> (concat a b)
+      (group-by file-loc-id)
+      vals
+      (map (partial apply merge)))))
+
+(defn read-and-parse
+  [src namespace]
+  (let [read (read-objs src)
+        forms (map :form read)
+        ast (ana-jvm/analyze
+             (list forms)
+             {:context :eval, :locals {}, :ns namespace})
+        defs (find-defs ast)]
+    (merge-read-objs-with-ast defs read)))

src/rksm/cloxp_source_reader/core.clj

@@ -0,0 +1,115 @@
+(ns rksm.cloxp-source-reader.core
+  (:require [clojure.tools.reader.reader-types :as trt]
+            [clojure.tools.reader :as tr]
+            [clojure.string :as s]
+            [rksm.system-files :refer (source-reader-for-ns)])
+  (:import (java.io LineNumberReader InputStreamReader PushbackReader)
+           (clojure.lang RT)))
+
+(defn purge-string!
+  [rdr]
+  (let [buf (-> rdr .rdr .source_log_frames var-get :buffer)
+        str (.toString buf)]
+    (.delete buf 0 (count str))
+    str))
+
+(defn read-objs
+  "Reads sexps from rdr-or-src and returns them as a {:form :source :line
+  :column} map. Note: this is more that the typical reader gives us."
+  [rdr-or-src]
+  ; FIXME this is hacked...
+  (let [rdr (trt/indexing-push-back-reader (trt/source-logging-push-back-reader rdr-or-src))]
+    (loop [result []]
+      (let [start-line (trt/get-line-number rdr)
+            start-column (trt/get-column-number rdr)]
+        (if-let [o (tr/read rdr false nil)]
+          (let [raw-str (purge-string! rdr)
+                lines (s/split-lines raw-str)
+                no-ws-lines (take-while #(re-find #"^\s*(;.*)?$" %) lines)
+                src-lines (drop (count no-ws-lines) lines)
+                first-line-ws-match (re-matches #"^(\s*)(.*)" (first src-lines))
+                src-lines (assoc (vec src-lines) 0 (nth first-line-ws-match 2))
+                src (s/join "\n" src-lines)
+                line (+ (count no-ws-lines) start-line)
+                column (+ start-column (count (second first-line-ws-match)))]
+            (when (= \newline (trt/peek-char rdr))
+              (trt/read-char rdr)
+              (purge-string! rdr))
+            (recur (conj result {:form o ;(with-meta o (assoc (meta o) :source src))
+                                 :source src
+                                 :line line
+                                 :column column})))
+          result)))))
+
+(defn- read-next-obj
+  "follows the reader while it core/reads an object and returns the string in
+  range of what was read"
+  [rdr]
+  (let [text (StringBuilder.) 
+        pbr (proxy [PushbackReader] [rdr]
+                   (read []
+                         (let [i (proxy-super read)]
+                           (if (> i -1) (.append text (char i)))
+                           i)))]
+    (if (= :unknown *read-eval*)
+      (throw (IllegalStateException. "Unable to read source while *read-eval* is :unknown."))
+      (tr/read (PushbackReader. pbr) false nil))
+    (str text)))
+
+(defn- read-entity-source
+  "goes forward in line numbering reader until line of entity is reached and
+  reads that as an object"
+  [{lrdr :lrdr, sources :sources, :as record} meta-entity]
+  (or (if-let [line (:line meta-entity)]
+        (do
+          (dotimes [_ (dec (- line (.getLineNumber lrdr)))] (.readLine lrdr))
+          (let [new-meta (merge meta-entity {:source (read-next-obj lrdr)})]
+            (update-in record [:sources] conj new-meta))))
+      record))
+
+(defn add-source-to-interns-with-reader
+  "interns are supposed to be meta-data-like maps, at least including :line for
+  the entity to be read"
+  [rdr interns]
+  (let [source-data {:lrdr (LineNumberReader. rdr), :sources []}]
+    (if-let [result (reduce read-entity-source source-data interns)]
+      (:sources result)
+      interns)))
+
+(defn add-source-to-interns
+  "alternative for `source-for-symbol`. Instead of using clojure.repl this
+  functions uses the classloader info provided by system-files to find more
+  recent versions of the source.
+  NOTE: If there are multiple versions a lib on the classpath than it is
+  possible that this function will retrieve code that i not actually in the
+  system! (and the system meta data will clash with the actual file contents)"
+  [ns interns & [ns-file-path]]
+  (if-let [rdr (source-reader-for-ns ns ns-file-path)]
+    (with-open [rdr rdr]
+      (add-source-to-interns-with-reader rdr interns))
+    interns))
+
+(defn add-source-to-interns
+  "alternative for `source-for-symbol`. Instead of using clojure.repl this
+  functions uses the classloader info provided by system-files to find more
+  recent versions of the source.
+  NOTE: If there are multiple versions a lib on the classpath than it is
+  possible that this function will retrieve code that i not actually in the
+  system! (and the system meta data will clash with the actual file contents)"
+  [ns interns & [ns-file-path]]
+  (if-let [rdr (source-reader-for-ns ns ns-file-path)]
+    (with-open [rdr rdr]
+      (add-source-to-interns-with-reader rdr interns))
+    interns))
+
+; -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
+
+; (defn add-source-to-interns-from-repl
+;   "This method uses the RT/baseloader to lookup the files belonging to symbols.
+;   When files get reloaded / defs redefined this can mean that the code being
+;   retrieved is outdated"
+;   [ns intern-meta-data]
+;   (let [ns-string (str (ns-name ns))
+;         sym-fn (partial symbol ns-string)
+;         source-fn #(or (source-for-symbol (sym-fn (-> % :name str))) "")]
+;     (map #(assoc % :source (source-fn %)) intern-meta-data)))

test/rksm/cloxp_source_reader/test.clj

@@ -0,0 +1,73 @@
+(ns rksm.cloxp-source-reader.test
+  (:require [clojure.test :refer :all]
+            [rksm.cloxp-source-reader.core :as src-rdr]
+            [rksm.cloxp-source-reader.ast-reader :as ast-rdr]
+            (rksm.cloxp-source-reader.test dummy-1 dummy-3)))
+
+
+(deftest ast-reader-reading
+
+  (testing "simple read"
+    (is (= [{:form '(ns rksm.cloxp-source-reader.test.dummy-3),
+             :source "(ns rksm.cloxp-source-reader.test.dummy-3)",
+             :line 1,
+             :column 1}
+            {:form '(def x 23), :source "(def x 23)", :line 2, :column 3}]
+         (src-rdr/read-objs "(ns rksm.cloxp-source-reader.test.dummy-3)\n  (def x 23)\n")))))
+
+(deftest ast-reader-parsing
+
+  (testing "parse source"
+    (let [src "(ns rksm.cloxp-source-reader.test.dummy-3)\n  (defmacro b [] `~23)\n(+ 2 3)\n(defn foo [] `~23)\n"
+          expected [{:ns 'rksm.cloxp-source-reader.test.dummy-3,
+                     :name 'foo,
+                     :source "(defn foo [] `~23)",
+                     :line 4}
+                    {:ns 'rksm.cloxp-source-reader.test.dummy-3,
+                     :name 'b,
+                     :source "(defmacro b [] `~23)"
+                     :line 2}]]
+      (is (= expected
+             (map #(select-keys % [:name :ns :source :line])
+                  (ast-rdr/read-and-parse src 'rksm.cloxp-source-reader.test.dummy-3)))))))
+
+; -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
+
+(deftest source-retrieval
+;   (testing "get source for intern"
+;     (is (= "(def x 23)"
+;           (source-for-symbol 'rksm.cloxp-source-reader.test.dummy-1/x))))
+
+  (testing "extract meta entities from source"
+
+    (testing "meta entities match source"
+      (is (= [{:source "(def x 23)" :column 1,:line 1}
+              {:source "(def y 24)" :column 1,:line 2}]
+             (let [entities [{:column 1,:line 1} {:column 1,:line 2}]
+                   source (java.io.StringReader. "(def x 23)\n(def y 24)\n")]
+               (src-rdr/add-source-to-interns-with-reader source entities)))))
+
+    (testing "less meta entities than source"
+      (is (= [{:source "(def x 23)" :column 1,:line 1}
+              {:source "(def y 24)" :column 1,:line 6}]
+             (let [entities [{:column 1,:line 1} {:column 1,:line 6}]
+                   source (java.io.StringReader. "(def x 23)\n(def baz\n\n99)\n\n(def y 24)\n")]
+               (src-rdr/add-source-to-interns-with-reader source entities)))))
+
+    (testing "more meta entities than source"
+      (is (= [{:source "(def x 23)" :column 1,:line 1} {:source "" :column 1,:line 6}]
+             (let [entities [{:column 1,:line 1} {:column 1,:line 6}]
+                   source (java.io.StringReader. "(def x 23)")]
+               (src-rdr/add-source-to-interns-with-reader source entities)))))
+
+    (testing "not entities in source"
+      "this might be kind of unexpected but the reader des not care bout lines"
+      (is (= [{:source "(def y 24)" :column 1,:line 3} {:source "" :column 1,:line 6}]
+             (let [entities [{:column 1,:line 3} {:column 1,:line 6}]
+                   source (java.io.StringReader. "(def x 23)\n\n(def y 24)")]
+               (src-rdr/add-source-to-interns-with-reader source entities)))))))
+
+; -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
+
+(comment
+ (test-ns 'rksm.cloxp-source-reader.test))

test/rksm/cloxp_source_reader/test/dummy_1.clj

@@ -0,0 +1,3 @@
+(ns rksm.cloxp-source-reader.test.dummy-1)
+
+(def x 23)

test/rksm/cloxp_source_reader/test/dummy_3.clj

@@ -0,0 +1,13 @@
+(ns rksm.cloxp-source-reader.test.dummy-3)
+
+(def x 23)
+
+(defonce dummy-atom (atom []))
+
+(defn test-func
+  [y]
+  (swap! dummy-atom conj (+ x y)))
+
+(defmacro foo
+  [x & body]
+  `(foo ~x ~@body))