nimony: implement build + mimalloc support

.github/workflows/ci.yml

@@ -113,6 +113,10 @@ jobs:
         if: runner.os == 'Windows'
         run: winget install ezwinports.make --disable-interactivity --accept-source-agreements
 
+      - name: System Info (Windows)
+        if: runner.os == 'Windows'
+        run: "gcc -v"
+
       # - name: Restore Nim from cache
       #   if: >
       #     steps.nim-compiler-cache.outputs.cache-hit != 'true' &&

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "vendor/mimalloc"]
+	path = vendor/mimalloc
+	url = https://github.com/nim-lang/mimalloc

lib/std/system/mimalloc.nim

@@ -0,0 +1,23 @@
+{.build("C", "${path}/../../../vendor/mimalloc/src/static.c", "-I${path}/../../../vendor/mimalloc/include").}
+
+type
+  csize_t* {.importc: "size_t", nodecl.} = uint
+
+proc mi_malloc(size: csize_t): pointer {.importc: "mi_malloc".}
+proc mi_calloc(nmemb: csize_t, size: csize_t): pointer {.importc: "mi_calloc".}
+proc mi_realloc(pt: pointer, size: csize_t): pointer {.importc: "mi_realloc".}
+proc mi_free(p: pointer) {.importc: "mi_free".}
+
+proc mi_usable_size(p: pointer): csize_t {.importc: "mi_usable_size".}
+
+proc alloc*(size: int): pointer =
+  result = mi_malloc(size.csize_t)
+
+proc realloc*(p: pointer; size: int): pointer =
+  result = mi_realloc(p, size.csize_t)
+
+proc dealloc*(p: pointer) =
+  mi_free(p)
+
+proc allocatedSize*(p: pointer): int =
+  result = int mi_usable_size(p)

src/hexer/expander.nim

@@ -1108,9 +1108,9 @@ proc traverseStmt(e: var EContext; c: var Cursor; mode = TraverseAll) =
       skip c
     of TypeS:
       traverseTypeDecl e, c
-    of ContinueS, WhenS:
+    of ContinueS, WhenS, ClonerS, TracerS, DisarmerS, MoverS, DtorS:
       error e, "unreachable: ", c
-    of ClonerS, TracerS, DisarmerS, MoverS, DtorS:
+    of PragmasLineS:
       skip c
   else:
     error e, "statement expected, but got: ", c

src/hexer/xelim.nim

@@ -365,7 +365,8 @@ proc trStmt(c: var Context; dest: var TokenBuf; n: var Cursor) =
     trBlock c, dest, n, tar
   of IterS, TemplateS, TypeS, EmitS, BreakS, ContinueS,
      ForS, CmdS, IncludeS, ImportS, FromImportS, ImportExceptS,
-     ExportS, CommentS, ClonerS, TracerS, DisarmerS, MoverS, DtorS:
+     ExportS, CommentS, ClonerS, TracerS, DisarmerS, MoverS, DtorS,
+     PragmasLineS:
     takeTree dest, n
   of ScopeS:
     c.typeCache.openScope()

src/lib/nifindexes.nim

@@ -6,7 +6,7 @@
 
 ## Create an index file for a NIF file.
 
 import std / [os, tables, assertions, syncio, formatfloat, sets]
 import bitabs, lineinfos, nifreader, nifstreams, nifcursors, nifchecksums
 
 #import std / [sha1]
@@ -100,7 +100,8 @@
       inc n
 
 proc createIndex*(infile: string; buildChecksum: bool;
-    hookIndexMap: Table[string, seq[(SymId, SymId)]]) =
+    hookIndexMap: Table[string, seq[(SymId, SymId)]];
+    toBuild: TokenBuf) =
   let PublicT = registerTag "public"
   let PrivateT = registerTag "private"
   let KvT = registerTag "kv"
@@ -177,6 +178,14 @@
     content.add toString(hookSectionBuf)
     content.add "\n"
 
+  let buildT = registerTag "build"
+  var buildBuf = createTokenBuf()
+  buildBuf.addParLe buildT
+  buildBuf.add toBuild
+  buildBuf.addParRi
+  content.add toString(buildBuf)
+  content.add "\n"
+
   if buildChecksum:
     var checksum = newSha1State()
     processForChecksum(checksum, buf)
@@ -189,7 +198,7 @@
     writeFile(indexName, content)
 
 proc createIndex*(infile: string; buildChecksum: bool) =
-  createIndex(infile, buildChecksum, initTable[string, seq[(SymId, SymId)]]())
+  createIndex(infile, buildChecksum, initTable[string, seq[(SymId, SymId)]](), default(TokenBuf))
 
 type
   NifIndexEntry* = object
@@ -198,6 +207,7 @@
   NifIndex* = object
     public*, private*: Table[string, NifIndexEntry]
     hooks*: Table[string, Table[string, NifIndexEntry]]
+    toBuild*: seq[(string, string, string)]
 
 proc readSection(s: var Stream; tab: var Table[string, NifIndexEntry]; useAbsoluteOffset = false) =
   let KvT = registerTag "kv"
@@ -279,6 +289,23 @@
       readSection(s, result.hooks[tagName])
       t = next(s)
 
+    let BuildT = registerTag "build"
+    if t.tag == BuildT:
+      t = next(s)
+      while t.kind != EofToken and t.kind != ParRi:
+        # tup
+        t = next(s)
+        assert t.kind == StringLit
+        let typ = pool.strings[t.litId]
+        t = next(s)
+        assert t.kind == StringLit
+        let path = pool.strings[t.litId]
+        t = next(s)
+        assert t.kind == StringLit
+        let args = pool.strings[t.litId]
+        result.toBuild.add (typ, path, args)
+        t = next(s)
+        t = next(s)
   else:
     assert false, "expected 'index' tag"
 

src/nimony/deps.nim

@@ -15,7 +15,7 @@
 ]#
 
 import std/[os, tables, sets, syncio, assertions, strutils, times]
-import semos, nifconfig, nimony_model
+import semos, nifconfig, nimony_model, nifindexes
 import ".." / gear2 / modnames, semdata
 
 include nifprelude
@@ -247,6 +247,20 @@ const makefileHeader = """
 .SECONDARY:
   """ # don't delete intermediate files
 
+type
+  CFile = tuple
+    name, obj, customArgs: string
+
+proc toBuildList(c: DepContext): seq[CFile] =
+  result = @[]
+  for v in c.nodes:
+    let index = readIndex(mescape(indexFile(v.files[0])))
+    for i in index.toBuild:
+      let path = i[1]
+      let obj = splitFile(path).name & ".o"
+      let customArgs = i[2]
+      result.add (path, obj, customArgs)
+
 proc generateFinalMakefile(c: DepContext): string =
   var s = makefileHeader
   let dest =
@@ -261,11 +275,22 @@ proc generateFinalMakefile(c: DepContext): string =
 
   # The .exe file depends on all .o files:
   if c.cmd in {DoCompile, DoRun}:
+    let buildList = toBuildList(c)
+
     s.add "\n" & mescape(exeFile(c.rootNode.files[0])) & ":"
+
+    for cfile in buildList:
+      s.add " " & mescape("nifcache" / cfile.obj)
+
     for v in c.nodes:
       s.add " " & mescape(objFile(v.files[0]))
     s.add "\n\t$(CC) -o $@ $^"
 
+    for cfile in buildList:
+      s.add "\n" & mescape("nifcache" / cfile.obj) & ": " & mescape(cfile.name) &
+            "\n\t$(CC) -c $(CFLAGS) $(CPPFLAGS) " &
+            mescape(cfile.customArgs) & " $< -o $@"
+
     # The .o files depend on all of their .c files:
     s.add "\n%.o: %.c\n\t$(CC) -c $(CFLAGS) $(CPPFLAGS) $< -o $@"
 
@@ -337,7 +362,6 @@ proc buildGraph*(config: sink NifConfig; project: string; compat, forceRebuild,
   c.processedModules.incl p.modname
   parseDeps c, p, c.rootNode
   let makeFilename = generateFrontendMakefile(c, commandLineArgs)
-  let makeFinalFilename = generateFinalMakefile(c)
   #echo "run with: make -f ", makeFilename
   when defined(windows):
     putEnv("CC", "gcc")
@@ -346,6 +370,8 @@ proc buildGraph*(config: sink NifConfig; project: string; compat, forceRebuild,
     (if forceRebuild: " -B" else: "") &
     " -f "
   exec makeCommand & quoteShell(makeFilename)
+
+  let makeFinalFilename = generateFinalMakefile(c)
   exec makeCommand & quoteShell(makeFinalFilename)
   if cmd == DoRun:
     exec exeFile(c.rootNode.files[0])

src/nimony/nimony.nim

@@ -149,6 +149,7 @@ proc handleCmdLine() =
   of FullProject:
     createDir("nifcache")
     createDir(binDir())
+    exec "git submodule update --init"
     requiresTool "nifler", "src/nifler/nifler.nim", forceRebuild
     requiresTool "nimsem", "src/nimony/nimsem.nim", forceRebuild
     requiresTool "hexer", "src/hexer/hexer.nim", forceRebuild

src/nimony/nimony_model.nim

@@ -54,6 +54,7 @@ type
     DisarmerS = "disarmer"
     MoverS = "mover"
     DtorS = "dtor"
+    PragmasLineS = "pragmas"
 
   SymKind* = enum
     NoSym

src/nimony/sem.nim

@@ -4508,6 +4508,45 @@ template procGuard(c: var SemContext; body: untyped) =
   else:
     c.takeTree it.n
 
+proc semPragmasLine(c: var SemContext; it: var Item) =
+  let info = it.n.info
+  inc it.n
+  while it.n.kind == ParLe and it.n.stmtKind in {CallS, CmdS}:
+    inc it.n
+    if it.n.kind == Ident and pool.strings[it.n.litId] == "build":
+      inc it.n
+      var args = newSeq[string]()
+      while it.n.kind != ParRi:
+        if it.n.kind != StringLit:
+          buildErr c, it.n.info, "expected `string` but got: " & toString(it.n)
+
+        args.add pool.strings[it.n.litId]
+        inc it.n
+
+      skipParRi it.n
+
+      if args.len != 2 and args.len != 3:
+        buildErr c, it.n.info, "build expected 2 or 3 parameters"
+
+      let fileId = getFileId(pool.man, info)
+      let dir = absoluteParentDir(pool.files[fileId])
+
+      let compileType = args[0]
+      let name = args[1].replace("${path}", dir).toAbsolutePath(dir)
+      let customArgs = if args.len == 3: args[2].replace("${path}", dir) else: ""
+
+      if not fileExists2(name):
+        buildErr c, it.n.info, "cannot find: " & name
+
+      c.toBuild.buildTree TupleConstrX, info:
+        c.toBuild.addStrLit compileType, info
+        c.toBuild.addStrLit name, info
+        c.toBuild.addStrLit customArgs, info
+    else:
+      buildErr c, it.n.info, "unsupported pragmas"
+
+  skipParRi it.n
+
 proc semExpr(c: var SemContext; it: var Item; flags: set[SemFlag] = {}) =
   case it.n.kind
   of IntLit:
@@ -4656,6 +4695,8 @@ proc semExpr(c: var SemContext; it: var Item; flags: set[SemFlag] = {}) =
         skip it.n
       of ClonerS, TracerS, DisarmerS, MoverS, DtorS:
         takeTree c, it.n
+      of PragmasLineS:
+        semPragmasLine c, it
     of FalseX, TrueX:
       literalB c, it, c.types.boolType
     of InfX, NegInfX, NanX:
@@ -4796,7 +4837,7 @@ proc writeOutput(c: var SemContext; outfile: string) =
   #b.addRaw toString(c.dest)
   #b.close()
   writeFile outfile, "(.nif24)\n" & toString(c.dest)
-  createIndex outfile, true, c.hookIndexMap
+  createIndex outfile, true, c.hookIndexMap, c.toBuild
 
 proc phaseX(c: var SemContext; n: Cursor; x: SemPhase): TokenBuf =
   assert n == "stmts"

src/nimony/semdata.nim

@@ -97,3 +97,4 @@ type
     genericHooks*: Table[SymId, seq[SymId]]
     hookIndexMap*: Table[string, seq[(SymId, SymId)]]
     freshSyms*: HashSet[SymId] ## symdefs that should count as new for semchecking
+    toBuild*: TokenBuf

src/nimony/semos.nim

@@ -35,6 +35,18 @@ proc binDir*(): string =
 proc toolDir*(f: string): string =
   result = binDir() / f
 
+proc absoluteParentDir*(f: string): string =
+  result = f.absolutePath().parentDir()
+
+proc fileExists2*(f: string): bool =
+  result = os.fileExists(f)
+
+proc toAbsolutePath*(f: string, dir: string): string =
+  if f.isAbsolute:
+    result = f
+  else:
+    result = dir / f
+
 proc findTool*(name: string): string =
   assert not name.isAbsolute
   let exe = name.addFileExt(ExeExt)

tests/nifc/hello.expected.idx.nif

@@ -6,5 +6,6 @@
  (kv MyObject.ptr.object +699)
  (kv MyObject.my.sequence +171)
  (kv MyObject.sequence.base +127))
+(build)
 
 )

tests/nimony/sysbasics/foo.c

@@ -0,0 +1,3 @@
+int myFunc() {
+  return 12;
+}

tests/nimony/sysbasics/foo1.c

@@ -0,0 +1,3 @@
+int myFunc2() {
+  return 12;
+}

tests/nimony/sysbasics/tbuild.nim

@@ -0,0 +1,9 @@
+{.build("C", "foo.c").}
+{.build("C", "foo1.c", "-fno-strict-aliasing").}
+
+
+proc myFunc(): int {.importc: "myFunc".}
+proc myFunc2(): int {.importc: "myFunc2".}
+
+let s = myFunc()
+let s2 = myFunc2()

tests/nimony/sysbasics/tmimalloc.nim

@@ -0,0 +1,23 @@
+{.build("C", "${path}/../../../vendor/mimalloc/src/static.c", "-I${path}/../../../vendor/mimalloc/include").}
+
+type
+  csize_t* {.importc: "size_t", nodecl.} = uint
+
+proc mi_malloc(size: csize_t): pointer {.importc: "mi_malloc".}
+proc mi_calloc(nmemb: csize_t, size: csize_t): pointer {.importc: "mi_calloc".}
+proc mi_realloc(pt: pointer, size: csize_t): pointer {.importc: "mi_realloc".}
+proc mi_free(p: pointer) {.importc: "mi_free".}
+
+proc mi_usable_size(p: pointer): csize_t {.importc: "mi_usable_size".}
+
+proc alloc*(size: int): pointer =
+  result = mi_malloc(size.csize_t)
+
+proc realloc*(p: pointer; size: int): pointer =
+  result = mi_realloc(p, size.csize_t)
+
+proc dealloc*(p: pointer) =
+  mi_free(p)
+
+proc allocatedSize*(p: pointer): int =
+  result = int mi_usable_size(p)