nimony: implements hook index (#338)

* nimony: implements hook index

* fixes tests

* stores offset

* implementgs `readIndex`

src/lib/nifindexes.nim

@@ -6,7 +6,7 @@
 
 ## Create an index file for a NIF file.
 
-import std / [os, tables, assertions, syncio, formatfloat]
+import std / [os, tables, assertions, syncio, formatfloat, sets]
 import bitabs, lineinfos, nifreader, nifstreams, nifcursors, nifchecksums
 
 #import std / [sha1]
@@ -99,7 +99,8 @@ proc processForChecksum(dest: var Sha1State; content: var TokenBuf) =
     else:
       inc n
 
-proc createIndex*(infile: string; buildChecksum: bool) =
+proc createIndex*(infile: string; buildChecksum: bool;
+    hookIndexMap: Table[string, seq[(SymId, SymId)]]) =
   let PublicT = registerTag "public"
   let PrivateT = registerTag "private"
   let KvT = registerTag "kv"
@@ -117,6 +118,7 @@ proc createIndex*(infile: string; buildChecksum: bool) =
   public.addParLe PublicT
   private.addParLe PrivateT
   var buf = createTokenBuf(100)
+  var symToOffsetMap = initTable[SymId, int]()
 
   while true:
     let offs = offset(s.r)
@@ -137,6 +139,7 @@ proc createIndex*(infile: string; buildChecksum: bool) =
             addr(public)
           else:
             addr(private)
+        symToOffsetMap[sym] = target
         let diff = if isPublic: target - previousPublicTarget
                   else: target - previousPrivateTarget
         dest[].buildTree KvT, info:
@@ -155,25 +158,48 @@ proc createIndex*(infile: string; buildChecksum: bool) =
   content.add toString(public)
   content.add "\n"
   content.add toString(private)
+  content.add "\n"
+
+  for (key, values) in hookIndexMap.pairs:
+    var hookSectionBuf = default(TokenBuf)
+    let tag = registerTag(key)
+    hookSectionBuf.addParLe tag
+
+    for value in values:
+      let (obj, sym) = value
+      let offset = symToOffsetMap[sym]
+      hookSectionBuf.buildTree KvT, NoLineInfo:
+        hookSectionBuf.add symToken(obj, NoLineInfo)
+        hookSectionBuf.add intToken(pool.integers.getOrIncl(offset), NoLineInfo)
+
+    hookSectionBuf.addParRi()
+
+    content.add toString(hookSectionBuf)
+    content.add "\n"
+
   if buildChecksum:
     var checksum = newSha1State()
     processForChecksum(checksum, buf)
     let final = SecureHash checksum.finalize()
-    content.add "\n(checksum \"" & $final & "\")"
+    content.add "(checksum \"" & $final & "\")"
   content.add "\n)\n"
   if fileExists(indexName) and readFile(indexName) == content:
     discard "no change"
   else:
     writeFile(indexName, content)
 
+proc createIndex*(infile: string; buildChecksum: bool) =
+  createIndex(infile, buildChecksum, initTable[string, seq[(SymId, SymId)]]())
+
 type
   NifIndexEntry* = object
     offset*: int
     info*: PackedLineInfo
   NifIndex* = object
     public*, private*: Table[string, NifIndexEntry]
+    hooks*: Table[string, Table[string, NifIndexEntry]]
 
-proc readSection(s: var Stream; tab: var Table[string, NifIndexEntry]) =
+proc readSection(s: var Stream; tab: var Table[string, NifIndexEntry]; useAbsoluteOffset = false) =
   let KvT = registerTag "kv"
   var previousOffset = 0
   var t = next(s)
@@ -195,7 +221,8 @@ proc readSection(s: var Stream; tab: var Table[string, NifIndexEntry]) =
         if t.kind == IntLit:
           let offset = pool.integers[t.intId] + previousOffset
           tab[key] = NifIndexEntry(offset: offset, info: info)
-          previousOffset = offset
+          if not useAbsoluteOffset:
+            previousOffset = offset
         else:
           assert false, "invalid (kv) construct: IntLit expected"
         t = next(s) # skip offset
@@ -224,6 +251,14 @@ proc readIndex*(indexName: string): NifIndex =
   let PrivateT = registerTag "private"
   let IndexT = registerTag "index"
 
+  let ClonerT = registerTag "cloner"
+  let TracerT = registerTag "tracer"
+  let DisarmerT = registerTag "disarmer"
+  let MoverT = registerTag "mover"
+  let DtorT = registerTag "dtor"
+
+  let hookSet = toHashSet([ClonerT, TracerT, DisarmerT, MoverT, DtorT])
+
   result = default(NifIndex)
   var t = next(s)
   if t.tag == IndexT:
@@ -237,6 +272,13 @@ proc readIndex*(indexName: string): NifIndex =
       readSection s, result.private
     else:
       assert false, "'private' expected"
+    t = next(s)
+    while t.tag in hookSet:
+      let tagName = pool.tags[t.tag]
+      result.hooks[tagName] = initTable[string, NifIndexEntry]()
+      readSection(s, result.hooks[tagName])
+      t = next(s)
+
   else:
     assert false, "expected 'index' tag"
 

src/nimony/sem.nim

@@ -2598,12 +2598,8 @@ proc getParamsType(c: var SemContext; paramsAt: int): seq[TypeCursor] =
       inc n
       while n.kind != ParRi:
         if n.substructureKind == ParamS:
-          skipToLocalType n
-          var localTypeBuf = createTokenBuf()
-          takeTree localTypeBuf, n
-          swap c.dest, localTypeBuf
-          result.add typeToCursor(c, 0)
-          swap c.dest, localTypeBuf
+          var local = takeLocal(n)
+          result.add local.typ
           skip n
           skipParRi n
         else:
@@ -2612,11 +2608,11 @@ proc getParamsType(c: var SemContext; paramsAt: int): seq[TypeCursor] =
 
 type
   HookOp = enum
-    hookCloner
-    hookTracer
-    hookDisarmer
-    hookMover
-    hookDtor
+    hookCloner = "cloner"
+    hookTracer = "tracer"
+    hookDisarmer = "disarmer"
+    hookMover = "mover"
+    hookDtor = "dtor"
 
 proc getObjSymId(c: var SemContext; obj: TypeCursor): SymId =
   var obj = skipModifier(obj)
@@ -2679,19 +2675,15 @@ proc checkTypeHook(c: var SemContext; params: seq[TypeCursor]; op: HookOp; info:
     of hookMover:
       buildErr c, info, "signature for '=sink' must be proc[T: object](x: var T; y: T)"
 
-proc expandHook(c: var SemContext; dest: var TokenBuf; obj: SymId, symId: SymId, kind: StmtKind, info: PackedLineInfo) =
-  assert kind in {ClonerS, TracerS, DisarmerS, DtorS, MoverS}
-  dest.add parLeToken(pool.tags.getOrIncl($kind), info)
-  dest.add symToken(obj, info)
-  dest.add symToken(symId, info)
-  dest.addParRi()
+proc expandHook(c: var SemContext; obj: SymId, symId: SymId, op: HookOp) =
+  c.hookIndexMap.mgetOrPut($op, @[]).add (obj, symId)
 
 proc getHookName(symId: SymId): string =
   result = pool.syms[symId]
   extractBasename(result)
   result = result.normalize
 
-proc semHook(c: var SemContext; dest: var TokenBuf; name: string; beforeParams: int; symId: SymId, info: PackedLineInfo): TypeCursor =
+proc semHook(c: var SemContext; name: string; beforeParams: int; symId: SymId, info: PackedLineInfo): TypeCursor =
   let params = getParamsType(c, beforeParams)
   case name
   of "=destroy":
@@ -2713,7 +2705,7 @@ proc semHook(c: var SemContext; dest: var TokenBuf; name: string; beforeParams:
     raiseAssert "unreachable"
 
 proc semProc(c: var SemContext; it: var Item; kind: SymKind; pass: PassKind) =
-  const hookTable = toTable({"=destroy": DtorS, "=wasmoved": DisarmerS, "=trace": TracerS, "=copy": ClonerS, "=sink": MoverS})
+  const hookTable = toTable({"=destroy": hookDtor, "=wasmoved": hookDisarmer, "=trace": hookTracer, "=copy": hookCloner, "=sink": hookMover})
   let info = it.n.info
   let declStart = c.dest.len
   takeToken c, it.n
@@ -2733,7 +2725,6 @@ proc semProc(c: var SemContext; it: var Item; kind: SymKind; pass: PassKind) =
     inc c.routine.inLoop
     inc c.routine.inGeneric
 
-  var hookTagBuf = createTokenBuf()
   try:
     c.openScope() # open parameter scope
     semGenericParams c, it.n
@@ -2769,7 +2760,7 @@ proc semProc(c: var SemContext; it: var Item; kind: SymKind; pass: PassKind) =
           let params = getParamsType(c, beforeParams)
           assert params.len >= 1
           let obj = getObjSymId(c, params[0])
-          expandHook(c, hookTagBuf, obj, symId, hookTable[name], info)
+          expandHook(c, obj, symId, hookTable[name])
 
       of checkBody:
         if it.n != "stmts":
@@ -2783,12 +2774,12 @@ proc semProc(c: var SemContext; it: var Item; kind: SymKind; pass: PassKind) =
         addReturnResult c, resId, it.n.info
         let name = getHookName(symId)
         if name in hookTable:
-          let objCursor = semHook(c, hookTagBuf, name, beforeParams, symId, info)
+          let objCursor = semHook(c, name, beforeParams, symId, info)
           let obj = getObjSymId(c, objCursor)
 
           if c.routine.inGeneric == 0:
             # because it's a hook for sure
-            expandHook(c, hookTagBuf, obj, symId, hookTable[name], info)
+            expandHook(c, obj, symId, hookTable[name])
           else:
             if obj in c.genericHooks:
               c.genericHooks[obj].add symId
@@ -2821,8 +2812,6 @@ proc semProc(c: var SemContext; it: var Item; kind: SymKind; pass: PassKind) =
   producesVoid c, info, it.typ
   publish c, symId, declStart
 
-  c.dest.add hookTagBuf
-
 proc semExprSym(c: var SemContext; it: var Item; s: Sym; start: int; flags: set[SemFlag]) =
   it.kind = s.kind
   let expected = it.typ
@@ -4440,7 +4429,7 @@ proc writeOutput(c: var SemContext; outfile: string) =
   #b.addRaw toString(c.dest)
   #b.close()
   writeFile outfile, "(.nif24)\n" & toString(c.dest)
-  createIndex outfile, true
+  createIndex outfile, true, c.hookIndexMap
 
 proc phaseX(c: var SemContext; n: Cursor; x: SemPhase): TokenBuf =
   assert n == "stmts"
@@ -4485,7 +4474,7 @@ proc requestHookInstance(c: var SemContext; decl: Cursor) =
         while typevarsStart.kind != ParRi:
           let name = asTypevar(typevarsStart).name.symId
           inferred[name] = typevarsSeq[counter]
-          skip typevarsStart
+          skip typevarsStart # skip the typevar tree
           inc counter
         discard requestRoutineInstance(c, hook, typeArgs, inferred, info)
 

src/nimony/semdata.nim

@@ -92,3 +92,4 @@ type
     #fieldsCache: Table[SymId, Table[StrId, ObjField]]
     meta*: MetaInfo
     genericHooks*: Table[SymId, seq[SymId]]
+    hookIndexMap*: Table[string, seq[(SymId, SymId)]]

tests/nifc/hello.expected.idx.nif

@@ -6,4 +6,5 @@
  (kv MyObject.ptr.object +699)
  (kv MyObject.my.sequence +171)
  (kv MyObject.sequence.base +127))
+
 )

tests/nimony/sysbasics/thooks.nif

@@ -22,37 +22,32 @@
    (param :p.0 . . ~31,41,lib/std/system.nim
     (pointer) .)) . . . 2,1
   (stmts
-   (discard .))) ,7
- (tracer MyObjectBase.0.tho8gh7q4 =trace.0.tho8gh7q4) ,10
+   (discard .))) ,10
  (proc 5 :=wasMoved.0.tho8gh7q4 . . . 16
   (params 1
    (param :x.1 . . 3
     (mut 4 MyObjectBase.0.tho8gh7q4) .)) . . . 2,1
   (stmts
-   (discard .))) ,10
- (disarmer MyObjectBase.0.tho8gh7q4 =wasMoved.0.tho8gh7q4) ,13
+   (discard .))) ,13
  (proc 5 :=copy.0.tho8gh7q4 . . . 12
   (params 1
    (param :x.2 . . 3
     (mut 4 MyObjectBase.0.tho8gh7q4) .) 22
    (param :y.0 . . 3 MyObjectBase.0.tho8gh7q4 .)) . . . 2,1
   (stmts
-   (discard .))) ,13
- (cloner MyObjectBase.0.tho8gh7q4 =copy.0.tho8gh7q4) ,16
+   (discard .))) ,16
  (proc 5 :=sink.0.tho8gh7q4 . . . 12
   (params 1
    (param :x.3 . . 3
     (mut 4 MyObjectBase.0.tho8gh7q4) .) 22
    (param :y.1 . . 3 MyObjectBase.0.tho8gh7q4 .)) . . . 2,1
   (stmts
-   (discard .))) ,16
- (mover MyObjectBase.0.tho8gh7q4 =sink.0.tho8gh7q4) ,19
+   (discard .))) ,19
  (proc 5 :=destroy.0.tho8gh7q4 . . . 15
   (params 1
    (param :x.4 . . 3 MyObjectBase.0.tho8gh7q4 .)) . . . 2,1
   (stmts
-   (discard .))) ,19
- (dtor MyObjectBase.0.tho8gh7q4 =destroy.0.tho8gh7q4) ,22
+   (discard .))) ,22
  (proc 5 :=wasMoved.1.tho8gh7q4 . . 16
   (typevars 1
    (typevar :T.1.tho8gh7q4 . . . .) 4
@@ -130,8 +125,7 @@
    (param :x.14 . . 11 MyObject.1.tho8gh7q4 .)) ~12,~8 . ~12,~8 . ~12,~8 . ~10,~7
   (stmts 4
    (let :x.15 . . ~2,~27,lib/std/system.nim
-    (i -1) 4 +12))) 12,40
- (dtor MyObject.1.tho8gh7q4 =destroy.2.tho8gh7q4) 12,53
+    (i -1) 4 +12))) 12,53
  (proc :=checkHook.1.tho8gh7q4 . ~12,~4 .
   (at =checkHook.0.tho8gh7q4 ~8,~47,lib/std/system.nim
    (i -1) ~8,~21,lib/std/system.nim
@@ -169,8 +163,7 @@
    (let :x.25 . . ~2,~17,lib/std/system.nim
     (i -1) 4 +12) 4,1
    (let :y.9 . . ~2,~18,lib/std/system.nim
-    (i -1) 4 +4))) ,22
- (disarmer MyObject.1.tho8gh7q4 =wasMoved.2.tho8gh7q4) ,26
+    (i -1) 4 +4))) ,26
  (proc :=copy.2.tho8gh7q4 . .
   (at =copy.1.tho8gh7q4 4,~20,lib/std/system.nim
    (i -1) 4,6,lib/std/system.nim
@@ -180,8 +173,7 @@
     (mut 12 MyObject.1.tho8gh7q4) .) 24
    (param :y.10 . . 11 MyObject.1.tho8gh7q4 .)) . . . 2,1
   (stmts
-   (discard .))) ,26
- (cloner MyObject.1.tho8gh7q4 =copy.2.tho8gh7q4) ,29
+   (discard .))) ,29
  (proc :=sink.2.tho8gh7q4 . .
   (at =sink.1.tho8gh7q4 4,~23,lib/std/system.nim
    (i -1) 4,3,lib/std/system.nim
@@ -191,8 +183,7 @@
     (mut 12 MyObject.1.tho8gh7q4) .) 24
    (param :y.11 . . 11 MyObject.1.tho8gh7q4 .)) . . . 2,1
   (stmts
-   (discard .))) ,29
- (mover MyObject.1.tho8gh7q4 =sink.2.tho8gh7q4) ,22
+   (discard .))) ,22
  (proc :=wasMoved.3.tho8gh7q4 . .
   (at =wasMoved.1.tho8gh7q4 4,10,lib/std/system.nim
    (f +64) 4,~16,lib/std/system.nim
@@ -204,8 +195,7 @@
    (let :x.29 . . ~2,~17,lib/std/system.nim
     (i -1) 4 +12) 4,1
    (let :y.12 . . ~2,~18,lib/std/system.nim
-    (i -1) 4 +4))) ,22
- (disarmer MyObject.2.tho8gh7q4 =wasMoved.3.tho8gh7q4) ,26
+    (i -1) 4 +4))) ,26
  (proc :=copy.3.tho8gh7q4 . .
   (at =copy.1.tho8gh7q4 4,6,lib/std/system.nim
    (f +64) 4,~20,lib/std/system.nim
@@ -215,8 +205,7 @@
     (mut 12 MyObject.2.tho8gh7q4) .) 24
    (param :y.13 . . 11 MyObject.2.tho8gh7q4 .)) . . . 2,1
   (stmts
-   (discard .))) ,26
- (cloner MyObject.2.tho8gh7q4 =copy.3.tho8gh7q4) ,29
+   (discard .))) ,29
  (proc :=sink.3.tho8gh7q4 . .
   (at =sink.1.tho8gh7q4 4,3,lib/std/system.nim
    (f +64) 4,~23,lib/std/system.nim
@@ -226,8 +215,7 @@
     (mut 12 MyObject.2.tho8gh7q4) .) 24
    (param :y.14 . . 11 MyObject.2.tho8gh7q4 .)) . . . 2,1
   (stmts
-   (discard .))) ,29
- (mover MyObject.2.tho8gh7q4 =sink.3.tho8gh7q4) ,32
+   (discard .))) ,32
  (proc :=destroy.3.tho8gh7q4 . .
   (at =destroy.1.tho8gh7q4 4,,lib/std/system.nim
    (f +64) 4,~26,lib/std/system.nim
@@ -236,5 +224,4 @@
    (param :x.32 . . 11 MyObject.2.tho8gh7q4 .)) . . . 2,1
   (stmts 4
    (let :x.33 . . ~2,~27,lib/std/system.nim
-    (i -1) 4 +12))) ,32
- (dtor MyObject.2.tho8gh7q4 =destroy.3.tho8gh7q4))
+    (i -1) 4 +12))))