Convert utf8 column position to utf16 (#264)

* Convert utf8 columns to utf16

* Update hover to send the range info

* Update tests to check utf16 position

ls.nim

@@ -556,6 +556,30 @@ proc uriToStash*(ls: LanguageServer, uri: string): string =
   else:
     ""
 
+proc toUtf16Pos*(
+    ls: LanguageServer, uri: string, line: int, utf8Pos: int
+): Option[int] =
+  if uri in ls.openFiles and line >= 0 and line < ls.openFiles[uri].fingerTable.len:
+    let utf16Pos = ls.openFiles[uri].fingerTable[line].utf8to16(utf8Pos)
+    return some(utf16Pos)
+  else:
+    return none(int)
+
+proc toUtf16Pos*(suggest: Suggest, ls: LanguageServer): Suggest =
+  result = suggest
+  let uri = pathToUri(suggest.filePath)
+  let pos = toUtf16Pos(ls, uri, suggest.line - 1, suggest.column)
+  if pos.isSome:
+    result.column = pos.get()
+
+proc toUtf16Pos*(
+  suggest: SuggestInlayHint, ls: LanguageServer, uri: string
+): SuggestInlayHint =
+  result = suggest
+  let pos = toUtf16Pos(ls, uri, suggest.line - 1, suggest.column)
+  if pos.isSome:
+    result.column = pos.get()
+
 proc range*(startLine, startCharacter, endLine, endCharacter: int): Range =
   return
     Range %* {
@@ -565,35 +589,41 @@ proc range*(startLine, startCharacter, endLine, endCharacter: int): Range =
 
 proc toLabelRange*(suggest: Suggest): Range =
   with suggest:
-    let endColumn = column + qualifiedPath[^1].strip(chars = {'`'}).len
-    return range(line - 1, column, line - 1, endColumn)
+    return range(line - 1, column, line - 1, column + utf16Len(qualifiedPath[^1]))
 
 proc toDiagnostic(suggest: Suggest): Diagnostic =
   with suggest:
     let
-      endColumn = column + doc.rfind('\'') - doc.find('\'') - 2
-      node =
-        %*{
-          "uri": pathToUri(filepath),
-          "range": range(line - 1, column, line - 1, column + endColumn),
-          "severity":
-            case forth
-            of "Error": DiagnosticSeverity.Error.int
-            of "Hint": DiagnosticSeverity.Hint.int
-            of "Warning": DiagnosticSeverity.Warning.int
-            else: DiagnosticSeverity.Error.int
-          ,
-          "message": doc,
-          "source": "nim",
-          "code": "nimsuggest chk",
-        }
+      textStart = doc.find('\'')
+      textEnd = doc.rfind('\'')
+      endColumn =
+        if textStart >= 0 and textEnd >= 0:
+          column + utf16Len(doc[textStart + 1 ..< textEnd])
+        else:
+          column + 1
+
+    let node =
+      %*{
+        "uri": pathToUri(filepath),
+        "range": range(line - 1, column, line - 1, endColumn),
+        "severity":
+          case forth
+          of "Error": DiagnosticSeverity.Error.int
+          of "Hint": DiagnosticSeverity.Hint.int
+          of "Warning": DiagnosticSeverity.Warning.int
+          else: DiagnosticSeverity.Error.int
+        ,
+        "message": doc,
+        "source": "nim",
+        "code": "nimsuggest chk",
+      }
     return node.to(Diagnostic)
 
 proc sendDiagnostics*(ls: LanguageServer, diagnostics: seq[Suggest], path: string) =
   debug "Sending diagnostics", count = diagnostics.len, path = path
   let params =
     PublishDiagnosticsParams %*
-    {"uri": pathToUri(path), "diagnostics": diagnostics.map(toDiagnostic)}
+    {"uri": pathToUri(path), "diagnostics": diagnostics.map(x => x.toUtf16Pos(ls).toDiagnostic)}
   ls.notify("textDocument/publishDiagnostics", %params)
 
   if diagnostics.len != 0:

routes.nim

@@ -176,7 +176,7 @@ proc definition*(
     result = ns.get
       .def(uriToPath(uri), ls.uriToStash(uri), line + 1, ch.get)
       .await()
-      .map(toLocation)
+      .map(x => x.toUtf16Pos(ls).toLocation)
 
 proc declaration*(
     ls: LanguageServer, params: TextDocumentPositionParams, id: int
@@ -192,7 +192,7 @@ proc declaration*(
     result = ns.get
       .declaration(uriToPath(uri), ls.uriToStash(uri), line + 1, ch.get)
       .await()
-      .map(toLocation)
+      .map(x => x.toUtf16Pos(ls).toLocation)
 
 proc expandAll*(
     ls: LanguageServer, params: TextDocumentPositionParams
@@ -310,7 +310,7 @@ proc typeDefinition*(
     result = ns.get
       .`type`(uriToPath(uri), ls.uriToStash(uri), line + 1, ch.get)
       .await()
-      .map(toLocation)
+      .map(x => x.toUtf16Pos(ls).toLocation)
 
 proc toSymbolInformation*(suggest: Suggest): SymbolInformation =
   with suggest:
@@ -329,7 +329,7 @@ proc documentSymbols*(
   let ns = await ls.tryGetNimsuggest(uri)
   if ns.isSome:
     ns.get().outline(uriToPath(uri), ls.uriToStash(uri)).await().map(
-      toSymbolInformation
+      x => x.toUtf16Pos(ls).toSymbolInformation
     )
   else:
     @[]
@@ -385,11 +385,23 @@ proc hover*(
     if ch.isNone:
       return none(Hover)
     let suggestions =
-      await nimsuggest.get().def(uriToPath(uri), ls.uriToStash(uri), line + 1, ch.get)
+      await nimsuggest.get().highlight(uriToPath(uri), ls.uriToStash(uri), line + 1, ch.get)
     if suggestions.len == 0:
-      return none[Hover]()
+      return none(Hover)
+    var suggest = suggestions[0]
+    if suggest.symkind == "skModule": # NOTE: skMoudle always return position (1, 0)
+      return some(Hover(contents: some(%toMarkedStrings(suggest))))
     else:
-      return some(Hover(contents: some(%toMarkedStrings(suggestions[0]))))
+      for s in suggestions:
+        if s.line == line + 1:
+          if s.column <= ch.get:
+            suggest = s
+          else:
+            break
+      return some(Hover(
+        contents: some(%toMarkedStrings(suggest)),
+        range: some(toLabelRange(suggest.toUtf16Pos(ls))),
+      ))
 
 proc references*(
     ls: LanguageServer, params: ReferenceParams
@@ -404,7 +416,7 @@ proc references*(
     let refs =
       await nimsuggest.get.use(uriToPath(uri), ls.uriToStash(uri), line + 1, ch.get)
     result = refs.filter(suggest => suggest.section != ideDef or includeDeclaration).map(
-        toLocation
+        x => x.toUtf16Pos(ls).toLocation
       )
 
 proc prepareRename*(
@@ -536,7 +548,7 @@ proc inlayHint*(
             configuration.parameterHintsEnabled
           )
       )
-      .map(x => x.inlayHintInfo.toInlayHint(configuration))
+      .map(x => x.inlayHintInfo.toUtf16Pos(ls, uri).toInlayHint(configuration))
       .filter(x => x.label != "")
 
 proc codeAction*(
@@ -706,7 +718,7 @@ proc workspaceSymbol*(
     let
       nimsuggest = await ls.lastNimsuggest
       symbols = await nimsuggest.globalSymbols(params.query, "-")
-    return symbols.map(toSymbolInformation)
+    return symbols.map(x => x.toUtf16Pos(ls).toSymbolInformation)
 
 proc toDocumentHighlight(suggest: Suggest): DocumentHighlight =
   return DocumentHighlight %* {"range": toLabelRange(suggest)}
@@ -725,7 +737,7 @@ proc documentHighlight*(
     let suggestLocations = await nimsuggest.get.highlight(
       uriToPath(uri), ls.uriToStash(uri), line + 1, ch.get
     )
-    result = suggestLocations.map(toDocumentHighlight)
+    result = suggestLocations.map(x => x.toUtf16Pos(ls).toDocumentHighlight)
 
 proc extractId(id: JsonNode): int =
   if id.kind == JInt:

tests/projects/hw/hw.nim

@@ -1,5 +1,5 @@
-proc a() = discard
-a()
+proc a안녕() = discard
+#[안녕]#a안녕()
 var bbb = 100
 bbb = 200
 bbb = ""
@@ -32,3 +32,5 @@ type
 proc f(a: Obj) =
   with a:
     field1 = field2
+
+let a안녕bcd = 0

tests/tnimlangserver.nim

@@ -121,19 +121,41 @@ suite "LSP features":
 
   test "Sending hover.":
     let
-      hoverParams = positionParams(fixtureUri("projects/hw/hw.nim"), 1, 0)
+      hoverParams = positionParams(helloWorldUri, 1, 6)
       hover = client.call("textDocument/hover", %hoverParams).waitFor
-    doAssert contains($hover, "hw.a: proc ()")
+      expected = %*{
+        "contents": [{
+          "language": "nim",
+          "value": "hw.a안녕: proc (){.noSideEffect, gcsafe, raises: <inferred> [].}"
+        }],
+        "range": {
+          "start": {
+            "line": 1,
+            "character": 6
+          },
+          "end": {
+            "line": 1,
+            "character": 9
+          }
+        }
+      }
+    check hover == expected
 
   test "Sending hover(no content)":
-    let
-      hoverParams = positionParams(helloWorldUri, 2, 0)
-      hover = client.call("textDocument/hover", %hoverParams).waitFor
-    check hover.kind == JNull
+    block:
+      let
+        hoverParams = positionParams(helloWorldUri, 1, 5)
+        hover = client.call("textDocument/hover", %hoverParams).waitFor
+      check hover.kind == JNull
+    block:
+      let
+        hoverParams = positionParams(helloWorldUri, 2, 0)
+        hover = client.call("textDocument/hover", %hoverParams).waitFor
+      check hover.kind == JNull
 
   test "Definitions.":
     let
-      positionParams = positionParams(helloWorldUri, 1, 0)
+      positionParams = positionParams(helloWorldUri, 1, 6)
       locations = to(waitFor client.call("textDocument/definition", %positionParams),
                      seq[Location])
       expected = seq[Location] %* [{
@@ -145,7 +167,7 @@ suite "LSP features":
           },
           "end": {
             "line": 0,
-            "character": 6
+            "character": 8
           }
         }
       }]
@@ -158,7 +180,7 @@ suite "LSP features":
       },
       "position": {
          "line": 1,
-         "character": 0
+         "character": 6
       },
       "textDocument": {
          "uri": helloWorldUri
@@ -175,19 +197,19 @@ suite "LSP features":
         },
         "end": {
           "line": 0,
-          "character": 6
+          "character": 8
         }
       }
       }, {
       "uri": helloWorldUri,
       "range": {
         "start": {
           "line": 1,
-          "character": 0
+          "character": 6
         },
         "end": {
           "line": 1,
-          "character": 1
+          "character": 9
         }
       }
     }]
@@ -200,7 +222,7 @@ suite "LSP features":
       },
       "position": {
          "line": 1,
-         "character": 0
+         "character": 7
       },
       "textDocument": {
          "uri": helloWorldUri
@@ -214,11 +236,11 @@ suite "LSP features":
       "range": {
         "start": {
           "line": 1,
-          "character": 0
+          "character": 6
         },
         "end": {
           "line": 1,
-          "character": 1
+          "character": 9
         }
       }
     }]

tests/tsuggestapi.nim

@@ -40,12 +40,12 @@ suite "Nimsuggest tests":
     )[]
 
   test "test Nimsuggest.call":
-    let res = waitFor nimSuggest.call("def", helloWorldFile, helloWorldFile, 2, 0)
+    let res = waitFor nimSuggest.call("def", helloWorldFile, helloWorldFile, 2, 10)
     doAssert res.len == 1
     doAssert res[0].forth.contains("noSideEffect")
 
   test "test Nimsuggest.def":
-    let res = waitFor nimSuggest.def(helloWorldFile, helloWorldFile, 2, 0)
+    let res = waitFor nimSuggest.def(helloWorldFile, helloWorldFile, 2, 10)
     doAssert res.len == 1
     doAssert res[0].forth.contains("proc")
 

utils.nim

@@ -35,6 +35,19 @@ proc createUTFMapping*(line: string): FingerTable =
 
   #echo fingerTable
 
+proc utf16Len*(utf8Str: string): int =
+  result = 0
+  for rune in utf8Str.runes:
+    case rune.int32
+    of 0x0000 .. 0x007F,
+       0x0080 .. 0x07FF,
+       0x0800 .. 0xFFFF:
+      result += 1
+    of 0x10000 .. 0x10FFFF:
+      result += 2
+    else:
+      discard
+
 proc utf16to8*(fingerTable: FingerTable, utf16pos: int): int =
   result = utf16pos
   for finger in fingerTable:
@@ -43,10 +56,18 @@ proc utf16to8*(fingerTable: FingerTable, utf16pos: int): int =
     else:
       break
 
+proc utf8to16*(fingerTable: FingerTable, utf8pos: int): int =
+  result = utf8pos
+  for finger in fingerTable:
+    if finger.u16pos < result:
+      result -= finger.offset
+    else:
+      break
+
 when isMainModule:
   import termstyle
   var x = "heållo☀☀wor𐐀𐐀☀ld heållo☀wor𐐀ld heållo☀wor𐐀ld"
-  var fingerTable = populateUTFMapping(x)
+  var fingerTable = createUTFMapping(x)
 
   var corrected = utf16to8(fingerTable, 5)
   for y in x: