feat(modules): reload modules and compilation cache

compiler+runtime/include/cpp/jank/c_api.h

@@ -24,6 +24,8 @@ extern "C"
   jank_object_ptr jank_eval(jank_object_ptr s);
   jank_object_ptr jank_read_string(jank_object_ptr s);
 
+  void jank_ns_set_symbol_counter(char const * const ns, uint64_t const count);
+
   jank_object_ptr jank_var_intern(jank_object_ptr ns, jank_object_ptr name);
   jank_object_ptr jank_var_bind_root(jank_object_ptr var, jank_object_ptr val);
   jank_object_ptr jank_var_set_dynamic(jank_object_ptr var, jank_object_ptr dynamic);

compiler+runtime/include/cpp/jank/jit/processor.hpp

@@ -1,5 +1,7 @@
 #pragma once
 
+#include <boost/filesystem/path.hpp>
+#include <fmt/format.h>
 #include <memory>
 
 #include <clang/Interpreter/Interpreter.h>
@@ -34,11 +36,17 @@ namespace jank::jit
     void load_bitcode(native_persistent_string const &module,
                       native_persistent_string_view const &bitcode) const;
 
+    string_result<void> remove_symbol(native_persistent_string const &name) const;
+
     template <typename T>
-    T find_symbol(native_persistent_string const &name) const
+    string_result<T> find_symbol(native_persistent_string const &name) const
     {
-      auto const sym(interpreter->getSymbolAddress(name.c_str()).get());
-      return sym.toPtr<T>();
+      if(auto symbol{ interpreter->getSymbolAddress(name.c_str()) })
+      {
+        return symbol.get().toPtr<T>();
+      }
+
+      return err(fmt::format("Failed to find the symbol: '{}'", name.c_str()));
     }
 
     result<void, native_persistent_string>

compiler+runtime/include/cpp/jank/runtime/context.hpp

@@ -107,10 +107,10 @@ namespace jank::runtime
 
     /* Generates a unique name for use with anything from codgen structs,
      * lifted vars, to shadowed locals. */
-    static native_persistent_string unique_string();
-    static native_persistent_string unique_string(native_persistent_string_view const &prefix);
-    static obj::symbol unique_symbol();
-    static obj::symbol unique_symbol(native_persistent_string_view const &prefix);
+    native_persistent_string unique_string();
+    native_persistent_string unique_string(native_persistent_string_view const &prefix);
+    obj::symbol unique_symbol();
+    obj::symbol unique_symbol(native_persistent_string_view const &prefix);
 
     folly::Synchronized<native_unordered_map<obj::symbol_ptr, ns_ptr>> namespaces;
     folly::Synchronized<native_unordered_map<native_persistent_string, obj::keyword_ptr>> keywords;

compiler+runtime/include/cpp/jank/runtime/core/munge.hpp

@@ -5,6 +5,9 @@
 namespace jank::runtime
 {
   native_persistent_string munge(native_persistent_string const &o);
+  native_persistent_string munge_extra(native_persistent_string const &o,
+                                       native_persistent_string const &search,
+                                       char const * const replace);
   object_ptr munge(object_ptr o);
   native_persistent_string demunge(native_persistent_string const &o);
 }

compiler+runtime/include/cpp/jank/runtime/module/loader.hpp

@@ -95,9 +95,6 @@ namespace jank::runtime::module
 
     loader(context &rt_ctx, native_persistent_string_view const &ps);
 
-    native_bool is_loaded(native_persistent_string_view const &) const;
-    void set_loaded(native_persistent_string_view const &);
-
     string_result<find_result> find(native_persistent_string_view const &module, origin const ori);
     string_result<void> load(native_persistent_string_view const &module, origin const ori);
 
@@ -115,6 +112,5 @@ namespace jank::runtime::module
     /* This maps module strings to entries. Module strings are like fully qualified Java
      * class names. */
     native_unordered_map<native_persistent_string, entry> entries;
-    native_set<native_persistent_string> loaded;
   };
 }

compiler+runtime/include/cpp/jank/runtime/ns.hpp

@@ -50,6 +50,8 @@ namespace jank::runtime
     /* TODO: Benchmark the use of atomics here. That's what Clojure uses. */
     folly::Synchronized<obj::persistent_hash_map_ptr> vars;
     folly::Synchronized<obj::persistent_hash_map_ptr> aliases;
+
+    std::atomic_uint64_t symbol_counter{};
     context &rt_ctx;
   };
 }

compiler+runtime/src/cpp/clojure/core_native.cpp

@@ -78,6 +78,11 @@ namespace clojure::core_native
     return try_object<var>(o)->deref();
   }
 
+  static object_ptr intern_var(object_ptr const sym)
+  {
+    return __rt_ctx->intern_var(try_object<obj::symbol>(sym)).expect_ok();
+  }
+
   static object_ptr var_get_root(object_ptr const o)
   {
     return try_object<var>(o)->get_root();
@@ -255,17 +260,6 @@ namespace clojure::core_native
     return obj::nil::nil_const();
   }
 
-  static object_ptr is_module_loaded(object_ptr const path)
-  {
-    return make_box(__rt_ctx->module_loader.is_loaded(runtime::to_string(path)));
-  }
-
-  static object_ptr set_module_loaded(object_ptr const path)
-  {
-    __rt_ctx->module_loader.set_loaded(runtime::to_string(path));
-    return obj::nil::nil_const();
-  }
-
   static object_ptr compile(object_ptr const path)
   {
     __rt_ctx->compile_module(runtime::to_string(path)).expect_ok();
@@ -421,6 +415,7 @@ jank_object_ptr jank_load_clojure_core_native()
   intern_fn("namespace", &namespace_);
   intern_fn("var?", &core_native::is_var);
   intern_fn("var-get", &core_native::var_get);
+  intern_fn("intern-var", &core_native::intern_var);
   intern_fn("var-get-root", &core_native::var_get_root);
   intern_fn("var-bind-root", &core_native::var_bind_root);
   intern_fn("alter-var-root", &core_native::alter_var_root);
@@ -465,8 +460,6 @@ jank_object_ptr jank_load_clojure_core_native()
   intern_fn("alias", &core_native::alias);
   intern_fn("refer", &core_native::refer);
   intern_fn("load-module", &core_native::load_module);
-  intern_fn("set-module-loaded", &core_native::set_module_loaded);
-  intern_fn("module-loaded?", &core_native::is_module_loaded);
   intern_fn("compile", &core_native::compile);
 
   /* TODO: jank.math? */

compiler+runtime/src/cpp/jank/analyze/local_frame.cpp

@@ -248,7 +248,7 @@ namespace jank::analyze
       return;
     }
 
-    auto const name(context::unique_symbol("const"));
+    auto const name(__rt_ctx->unique_symbol("const"));
     auto const unboxed_name{ visit_number_like(
       [&](auto const) -> option<obj::symbol> {
         return obj::symbol{ name.ns, name.name + "__unboxed" };

compiler+runtime/src/cpp/jank/analyze/processor.cpp

@@ -312,7 +312,7 @@ namespace jank::analyze
           {
             /* C++ doesn't allow multiple params with the same name, so we generate a unique
              * name for shared params. */
-            param = make_box<runtime::obj::symbol>(runtime::context::unique_string("shadowed"));
+            param = make_box<runtime::obj::symbol>(__rt_ctx->unique_string("shadowed"));
             break;
           }
         }
@@ -387,7 +387,7 @@ namespace jank::analyze
     {
       auto const s(runtime::expect_object<runtime::obj::symbol>(first_elem));
       name = s->name;
-      unique_name = runtime::context::unique_string(name);
+      unique_name = __rt_ctx->unique_string(name);
       if(length < 3)
       {
         return err(error{ fmt::format("fn missing forms: {}", full_list->to_string()) });
@@ -397,7 +397,7 @@ namespace jank::analyze
     }
     else
     {
-      name = runtime::context::unique_string("fn");
+      name = __rt_ctx->unique_string("fn");
       unique_name = name;
     }
 

compiler+runtime/src/cpp/jank/c_api.cpp

@@ -75,9 +75,17 @@ extern "C"
     return __rt_ctx->read_string(s_obj->data);
   }
 
+  void jank_ns_set_symbol_counter(char const * const ns, uint64_t const count)
+  {
+    auto const ns_obj(__rt_ctx->intern_ns(ns));
+    ns_obj->symbol_counter.store(count);
+  }
+
   jank_object_ptr jank_var_intern(jank_object_ptr const ns, jank_object_ptr const name)
   {
     auto const ns_obj(try_object<obj::persistent_string>(reinterpret_cast<object *>(ns)));
+    __rt_ctx->intern_ns(ns_obj->data);
+
     auto const name_obj(try_object<obj::persistent_string>(reinterpret_cast<object *>(name)));
     return erase(__rt_ctx->intern_var(ns_obj->data, name_obj->data).expect_ok());
   }

compiler+runtime/src/cpp/jank/codegen/llvm_processor.cpp

@@ -26,9 +26,9 @@ namespace jank::codegen
 
   reusable_context::reusable_context(native_persistent_string const &module_name)
     : module_name{ module_name }
-    , ctor_name{ runtime::munge(runtime::context::unique_string("jank_global_init")) }
+    , ctor_name{ runtime::munge(__rt_ctx->unique_string("jank_global_init")) }
     , llvm_ctx{ std::make_unique<llvm::LLVMContext>() }
-    , module{ std::make_unique<llvm::Module>(runtime::context::unique_string(module_name).c_str(),
+    , module{ std::make_unique<llvm::Module>(__rt_ctx->unique_string(module_name).c_str(),
                                              *llvm_ctx) }
     , builder{ std::make_unique<llvm::IRBuilder<>>(*llvm_ctx) }
     , global_ctor_block{ llvm::BasicBlock::Create(*llvm_ctx, "entry") }
@@ -135,6 +135,29 @@ namespace jank::codegen
     {
       auto const global_ctor_fn(ctx->global_ctor_block->getParent());
       ctx->builder->CreateCall(global_ctor_fn, {});
+
+      /* This dance is performed to keep symbol names unique across all the modules.
+       * Considering LLVM JIT symbols to be global, we need to define them with
+       * unique names to avoid conflicts during JIT recompilation/reloading.
+       *
+       * The approach, right now, is for each namespace, we will keep a counter
+       * and will increase it every time we define a new symbol. When we JIT reload
+       * the same namespace again, we will define new symbols.
+       *
+       * This IR codegen for calling `jank_ns_set_symbol_counter`, is to set the counter
+       * on intial load.
+       */
+      auto const current_ns{ __rt_ctx->current_ns() };
+      auto const fn_type(
+        llvm::FunctionType::get(ctx->builder->getVoidTy(),
+                                { ctx->builder->getPtrTy(), ctx->builder->getInt64Ty() },
+                                false));
+      auto const fn(ctx->module->getOrInsertFunction("jank_ns_set_symbol_counter", fn_type));
+
+      ctx->builder->CreateCall(
+        fn,
+        { gen_c_string(current_ns->name->get_name()),
+          llvm::ConstantInt::get(ctx->builder->getInt64Ty(), current_ns->symbol_counter.load()) });
     }
 
     for(size_t i{}; i < arity.params.size(); ++i)

compiler+runtime/src/cpp/jank/evaluate.cpp

@@ -134,7 +134,7 @@ namespace jank::evaluate
     auto &fn(boost::get<expr::function<expression>>(ret->data));
     expr::function_arity<expression> arity;
     fn.name = name;
-    fn.unique_name = context::unique_string(fn.name);
+    fn.unique_name = __rt_ctx->unique_string(fn.name);
     fn.meta = obj::persistent_hash_map::empty();
 
     auto const &closest_fn_frame(local_frame::find_closest_fn_frame(*expr.frame));
@@ -558,8 +558,10 @@ namespace jank::evaluate
       __rt_ctx->jit_prc.load_ir_module(std::move(cg_prc.ctx->module),
                                        std::move(cg_prc.ctx->llvm_ctx));
 
-      auto const fn(__rt_ctx->jit_prc.find_symbol<object *(*)()>(
-        fmt::format("{}_0", munge(cg_prc.root_fn.unique_name))));
+      auto const fn(
+        __rt_ctx->jit_prc
+          .find_symbol<object *(*)()>(fmt::format("{}_0", munge(cg_prc.root_fn.unique_name)))
+          .expect_ok());
       return fn();
     }
   }

compiler+runtime/src/cpp/jank/jit/processor.cpp

@@ -4,6 +4,7 @@
 #include <clang/Basic/Diagnostic.h>
 #include <clang/Frontend/CompilerInstance.h>
 #include <clang/Frontend/FrontendDiagnostic.h>
+#include <llvm/ExecutionEngine/Orc/Core.h>
 #include <llvm/Support/Signals.h>
 #include <llvm/ExecutionEngine/Orc/LLJIT.h>
 #include <llvm/IRReader/IRReader.h>
@@ -185,6 +186,20 @@ namespace jank::jit
     load_ir_module(std::move(ir_module), std::move(ctx));
   }
 
+  string_result<void> processor::remove_symbol(native_persistent_string const &name) const
+  {
+    auto &ee{ interpreter->getExecutionEngine().get() };
+    llvm::orc::SymbolNameSet to_remove{};
+    to_remove.insert(ee.mangleAndIntern(name.c_str()));
+    auto const error{ ee.getMainJITDylib().remove(to_remove) };
+
+    if(error.isA<llvm::orc::SymbolsCouldNotBeRemoved>())
+    {
+      return err(fmt::format("Failed to remove the symbol: '{}'", name));
+    }
+    return ok();
+  }
+
   option<native_persistent_string>
   processor::find_dynamic_lib(native_persistent_string const &lib) const
   {

compiler+runtime/src/cpp/jank/read/parse.cpp

@@ -914,7 +914,7 @@ namespace jank::read::parse
         auto gensym(get(env, sym));
         if(gensym->type == object_type::nil)
         {
-          gensym = make_box<obj::symbol>(context::unique_symbol(sym->name));
+          gensym = make_box<obj::symbol>(__rt_ctx->unique_symbol(sym->name));
           __rt_ctx->gensym_env_var->set(assoc(env, sym, gensym)).expect_ok();
         }
         sym = expect_object<obj::symbol>(gensym);

compiler+runtime/src/cpp/jank/runtime/context.cpp

@@ -8,13 +8,15 @@
 #include <llvm/TargetParser/Host.h>
 
 #include <fmt/compile.h>
+#include <regex>
 
 #include <jank/native_persistent_string/fmt.hpp>
 #include <jank/read/lex.hpp>
 #include <jank/read/parse.hpp>
 #include <jank/runtime/context.hpp>
 #include <jank/runtime/visit.hpp>
 #include <jank/runtime/core.hpp>
+#include <jank/runtime/core/munge.hpp>
 #include <jank/analyze/processor.hpp>
 #include <jank/evaluate.hpp>
 #include <jank/jit/processor.hpp>
@@ -264,7 +266,7 @@ namespace jank::runtime
                                     std::make_pair(compile_files_var, obj::boolean::true_const()),
                                     std::make_pair(current_module_var, make_box(module))) };
 
-    return load_module(fmt::format("/{}", module), module::origin::source);
+    return load_module(fmt::format("/{}", module), module::origin::latest);
   }
 
   string_result<void>
@@ -285,7 +287,7 @@ namespace jank::runtime
                              module_path.c_str(),
                              file_error.message()));
     }
-    //codegen_ctx->module->print(llvm::outs(), nullptr);
+    // codegen_ctx->module->print(llvm::outs(), nullptr);
 
     auto const target_triple{ llvm::sys::getDefaultTargetTriple() };
     std::string target_error;
@@ -321,8 +323,12 @@ namespace jank::runtime
 
   native_persistent_string context::unique_string(native_persistent_string_view const &prefix)
   {
-    static std::atomic_size_t index{ 1 };
-    return fmt::format(FMT_COMPILE("{}-{}"), prefix.data(), index++);
+    static native_persistent_string const dot{ "\\." };
+    auto const ns{ current_ns() };
+    return fmt::format(FMT_COMPILE("{}-{}-{}"),
+                       runtime::munge_extra(ns->name->get_name(), dot, "_"),
+                       prefix.data(),
+                       ++ns->symbol_counter);
   }
 
   obj::symbol context::unique_symbol()

compiler+runtime/src/cpp/jank/runtime/core.cpp

@@ -448,7 +448,7 @@ namespace jank::runtime
 
   object_ptr gensym(object_ptr const o)
   {
-    return make_box<obj::symbol>(runtime::context::unique_symbol(to_string(o)));
+    return make_box<obj::symbol>(__rt_ctx->unique_symbol(to_string(o)));
   }
 
   object_ptr atom(object_ptr const o)

compiler+runtime/src/cpp/jank/runtime/core/munge.cpp

@@ -1,3 +1,5 @@
+#include <regex>
+
 #include <jank/runtime/core/munge.hpp>
 #include <jank/runtime/obj/persistent_string.hpp>
 #include <jank/runtime/rtti.hpp>
@@ -283,6 +285,15 @@ namespace jank::runtime
     return munged;
   }
 
+  native_persistent_string munge_extra(native_persistent_string const &o,
+                                       native_persistent_string const &search,
+                                       char const * const replace)
+  {
+    native_transient_string const ret{ munge(o) };
+    std::regex const search_regex{ search.c_str() };
+    return std::regex_replace(ret, search_regex, replace);
+  }
+
   /* TODO: Support symbols and other data; Clojure takes in anything and passes it through str. */
   object_ptr munge(object_ptr const o)
   {

compiler+runtime/src/cpp/jank/runtime/core/seq.cpp

@@ -8,6 +8,7 @@
 #include <jank/runtime/behavior/conjable.hpp>
 #include <jank/runtime/behavior/countable.hpp>
 #include <jank/runtime/behavior/seqable.hpp>
+#include <jank/runtime/behavior/set_like.hpp>
 #include <jank/runtime/behavior/sequential.hpp>
 #include <jank/runtime/behavior/collection_like.hpp>
 #include <jank/runtime/behavior/transientable.hpp>
@@ -670,11 +671,7 @@ namespace jank::runtime
       [&](auto const typed_s) -> native_bool {
         using S = typename decltype(typed_s)::value_type;
 
-        if constexpr(behavior::associatively_readable<S>)
-        {
-          return typed_s->contains(key);
-        }
-        if constexpr(std::same_as<S, obj::persistent_hash_set>)
+        if constexpr(behavior::associatively_readable<S> || behavior::set_like<S>)
         {
           return typed_s->contains(key);
         }