From cf85884efd59f5228d79640833b0c5b65d97579f Mon Sep 17 00:00:00 2001 From: doongjohn <40219740+doongjohn@users.noreply.github.com> Date: Sun, 1 Dec 2024 08:32:37 +0900 Subject: [PATCH] Convert utf8 column position to utf16 (#264) * Convert utf8 columns to utf16 * Update hover to send the range info * Update tests to check utf16 position --- ls.nim | 68 +++++++++++++++++++++++++++++----------- routes.nim | 34 +++++++++++++------- tests/projects/hw/hw.nim | 6 ++-- tests/tnimlangserver.nim | 52 +++++++++++++++++++++--------- tests/tsuggestapi.nim | 4 +-- utils.nim | 23 +++++++++++++- 6 files changed, 137 insertions(+), 50 deletions(-) diff --git a/ls.nim b/ls.nim index e685b64..db87ca0 100644 --- a/ls.nim +++ b/ls.nim @@ -556,6 +556,30 @@ proc uriToStash*(ls: LanguageServer, uri: string): string = else: "" +proc toUtf16Pos*( + ls: LanguageServer, uri: string, line: int, utf8Pos: int +): Option[int] = + if uri in ls.openFiles and line >= 0 and line < ls.openFiles[uri].fingerTable.len: + let utf16Pos = ls.openFiles[uri].fingerTable[line].utf8to16(utf8Pos) + return some(utf16Pos) + else: + return none(int) + +proc toUtf16Pos*(suggest: Suggest, ls: LanguageServer): Suggest = + result = suggest + let uri = pathToUri(suggest.filePath) + let pos = toUtf16Pos(ls, uri, suggest.line - 1, suggest.column) + if pos.isSome: + result.column = pos.get() + +proc toUtf16Pos*( + suggest: SuggestInlayHint, ls: LanguageServer, uri: string +): SuggestInlayHint = + result = suggest + let pos = toUtf16Pos(ls, uri, suggest.line - 1, suggest.column) + if pos.isSome: + result.column = pos.get() + proc range*(startLine, startCharacter, endLine, endCharacter: int): Range = return Range %* { @@ -565,35 +589,41 @@ proc range*(startLine, startCharacter, endLine, endCharacter: int): Range = proc toLabelRange*(suggest: Suggest): Range = with suggest: - let endColumn = column + qualifiedPath[^1].strip(chars = {'`'}).len - return range(line - 1, column, line - 1, endColumn) + return range(line - 1, column, line - 1, column + utf16Len(qualifiedPath[^1])) proc toDiagnostic(suggest: Suggest): Diagnostic = with suggest: let - endColumn = column + doc.rfind('\'') - doc.find('\'') - 2 - node = - %*{ - "uri": pathToUri(filepath), - "range": range(line - 1, column, line - 1, column + endColumn), - "severity": - case forth - of "Error": DiagnosticSeverity.Error.int - of "Hint": DiagnosticSeverity.Hint.int - of "Warning": DiagnosticSeverity.Warning.int - else: DiagnosticSeverity.Error.int - , - "message": doc, - "source": "nim", - "code": "nimsuggest chk", - } + textStart = doc.find('\'') + textEnd = doc.rfind('\'') + endColumn = + if textStart >= 0 and textEnd >= 0: + column + utf16Len(doc[textStart + 1 ..< textEnd]) + else: + column + 1 + + let node = + %*{ + "uri": pathToUri(filepath), + "range": range(line - 1, column, line - 1, endColumn), + "severity": + case forth + of "Error": DiagnosticSeverity.Error.int + of "Hint": DiagnosticSeverity.Hint.int + of "Warning": DiagnosticSeverity.Warning.int + else: DiagnosticSeverity.Error.int + , + "message": doc, + "source": "nim", + "code": "nimsuggest chk", + } return node.to(Diagnostic) proc sendDiagnostics*(ls: LanguageServer, diagnostics: seq[Suggest], path: string) = debug "Sending diagnostics", count = diagnostics.len, path = path let params = PublishDiagnosticsParams %* - {"uri": pathToUri(path), "diagnostics": diagnostics.map(toDiagnostic)} + {"uri": pathToUri(path), "diagnostics": diagnostics.map(x => x.toUtf16Pos(ls).toDiagnostic)} ls.notify("textDocument/publishDiagnostics", %params) if diagnostics.len != 0: diff --git a/routes.nim b/routes.nim index 5caae48..859bccc 100644 --- a/routes.nim +++ b/routes.nim @@ -176,7 +176,7 @@ proc definition*( result = ns.get .def(uriToPath(uri), ls.uriToStash(uri), line + 1, ch.get) .await() - .map(toLocation) + .map(x => x.toUtf16Pos(ls).toLocation) proc declaration*( ls: LanguageServer, params: TextDocumentPositionParams, id: int @@ -192,7 +192,7 @@ proc declaration*( result = ns.get .declaration(uriToPath(uri), ls.uriToStash(uri), line + 1, ch.get) .await() - .map(toLocation) + .map(x => x.toUtf16Pos(ls).toLocation) proc expandAll*( ls: LanguageServer, params: TextDocumentPositionParams @@ -310,7 +310,7 @@ proc typeDefinition*( result = ns.get .`type`(uriToPath(uri), ls.uriToStash(uri), line + 1, ch.get) .await() - .map(toLocation) + .map(x => x.toUtf16Pos(ls).toLocation) proc toSymbolInformation*(suggest: Suggest): SymbolInformation = with suggest: @@ -329,7 +329,7 @@ proc documentSymbols*( let ns = await ls.tryGetNimsuggest(uri) if ns.isSome: ns.get().outline(uriToPath(uri), ls.uriToStash(uri)).await().map( - toSymbolInformation + x => x.toUtf16Pos(ls).toSymbolInformation ) else: @[] @@ -385,11 +385,23 @@ proc hover*( if ch.isNone: return none(Hover) let suggestions = - await nimsuggest.get().def(uriToPath(uri), ls.uriToStash(uri), line + 1, ch.get) + await nimsuggest.get().highlight(uriToPath(uri), ls.uriToStash(uri), line + 1, ch.get) if suggestions.len == 0: - return none[Hover]() + return none(Hover) + var suggest = suggestions[0] + if suggest.symkind == "skModule": # NOTE: skMoudle always return position (1, 0) + return some(Hover(contents: some(%toMarkedStrings(suggest)))) else: - return some(Hover(contents: some(%toMarkedStrings(suggestions[0])))) + for s in suggestions: + if s.line == line + 1: + if s.column <= ch.get: + suggest = s + else: + break + return some(Hover( + contents: some(%toMarkedStrings(suggest)), + range: some(toLabelRange(suggest.toUtf16Pos(ls))), + )) proc references*( ls: LanguageServer, params: ReferenceParams @@ -404,7 +416,7 @@ proc references*( let refs = await nimsuggest.get.use(uriToPath(uri), ls.uriToStash(uri), line + 1, ch.get) result = refs.filter(suggest => suggest.section != ideDef or includeDeclaration).map( - toLocation + x => x.toUtf16Pos(ls).toLocation ) proc prepareRename*( @@ -536,7 +548,7 @@ proc inlayHint*( configuration.parameterHintsEnabled ) ) - .map(x => x.inlayHintInfo.toInlayHint(configuration)) + .map(x => x.inlayHintInfo.toUtf16Pos(ls, uri).toInlayHint(configuration)) .filter(x => x.label != "") proc codeAction*( @@ -706,7 +718,7 @@ proc workspaceSymbol*( let nimsuggest = await ls.lastNimsuggest symbols = await nimsuggest.globalSymbols(params.query, "-") - return symbols.map(toSymbolInformation) + return symbols.map(x => x.toUtf16Pos(ls).toSymbolInformation) proc toDocumentHighlight(suggest: Suggest): DocumentHighlight = return DocumentHighlight %* {"range": toLabelRange(suggest)} @@ -725,7 +737,7 @@ proc documentHighlight*( let suggestLocations = await nimsuggest.get.highlight( uriToPath(uri), ls.uriToStash(uri), line + 1, ch.get ) - result = suggestLocations.map(toDocumentHighlight) + result = suggestLocations.map(x => x.toUtf16Pos(ls).toDocumentHighlight) proc extractId(id: JsonNode): int = if id.kind == JInt: diff --git a/tests/projects/hw/hw.nim b/tests/projects/hw/hw.nim index 6ec066b..228d8b3 100644 --- a/tests/projects/hw/hw.nim +++ b/tests/projects/hw/hw.nim @@ -1,5 +1,5 @@ -proc a() = discard -a() +proc a안녕() = discard +#[안녕]#a안녕() var bbb = 100 bbb = 200 bbb = "" @@ -32,3 +32,5 @@ type proc f(a: Obj) = with a: field1 = field2 + +let a안녕bcd = 0 diff --git a/tests/tnimlangserver.nim b/tests/tnimlangserver.nim index 84bcbba..fedddf6 100644 --- a/tests/tnimlangserver.nim +++ b/tests/tnimlangserver.nim @@ -121,19 +121,41 @@ suite "LSP features": test "Sending hover.": let - hoverParams = positionParams(fixtureUri("projects/hw/hw.nim"), 1, 0) + hoverParams = positionParams(helloWorldUri, 1, 6) hover = client.call("textDocument/hover", %hoverParams).waitFor - doAssert contains($hover, "hw.a: proc ()") + expected = %*{ + "contents": [{ + "language": "nim", + "value": "hw.a안녕: proc (){.noSideEffect, gcsafe, raises: [].}" + }], + "range": { + "start": { + "line": 1, + "character": 6 + }, + "end": { + "line": 1, + "character": 9 + } + } + } + check hover == expected test "Sending hover(no content)": - let - hoverParams = positionParams(helloWorldUri, 2, 0) - hover = client.call("textDocument/hover", %hoverParams).waitFor - check hover.kind == JNull + block: + let + hoverParams = positionParams(helloWorldUri, 1, 5) + hover = client.call("textDocument/hover", %hoverParams).waitFor + check hover.kind == JNull + block: + let + hoverParams = positionParams(helloWorldUri, 2, 0) + hover = client.call("textDocument/hover", %hoverParams).waitFor + check hover.kind == JNull test "Definitions.": let - positionParams = positionParams(helloWorldUri, 1, 0) + positionParams = positionParams(helloWorldUri, 1, 6) locations = to(waitFor client.call("textDocument/definition", %positionParams), seq[Location]) expected = seq[Location] %* [{ @@ -145,7 +167,7 @@ suite "LSP features": }, "end": { "line": 0, - "character": 6 + "character": 8 } } }] @@ -158,7 +180,7 @@ suite "LSP features": }, "position": { "line": 1, - "character": 0 + "character": 6 }, "textDocument": { "uri": helloWorldUri @@ -175,7 +197,7 @@ suite "LSP features": }, "end": { "line": 0, - "character": 6 + "character": 8 } } }, { @@ -183,11 +205,11 @@ suite "LSP features": "range": { "start": { "line": 1, - "character": 0 + "character": 6 }, "end": { "line": 1, - "character": 1 + "character": 9 } } }] @@ -200,7 +222,7 @@ suite "LSP features": }, "position": { "line": 1, - "character": 0 + "character": 7 }, "textDocument": { "uri": helloWorldUri @@ -214,11 +236,11 @@ suite "LSP features": "range": { "start": { "line": 1, - "character": 0 + "character": 6 }, "end": { "line": 1, - "character": 1 + "character": 9 } } }] diff --git a/tests/tsuggestapi.nim b/tests/tsuggestapi.nim index 6bc1f8c..1138254 100644 --- a/tests/tsuggestapi.nim +++ b/tests/tsuggestapi.nim @@ -40,12 +40,12 @@ suite "Nimsuggest tests": )[] test "test Nimsuggest.call": - let res = waitFor nimSuggest.call("def", helloWorldFile, helloWorldFile, 2, 0) + let res = waitFor nimSuggest.call("def", helloWorldFile, helloWorldFile, 2, 10) doAssert res.len == 1 doAssert res[0].forth.contains("noSideEffect") test "test Nimsuggest.def": - let res = waitFor nimSuggest.def(helloWorldFile, helloWorldFile, 2, 0) + let res = waitFor nimSuggest.def(helloWorldFile, helloWorldFile, 2, 10) doAssert res.len == 1 doAssert res[0].forth.contains("proc") diff --git a/utils.nim b/utils.nim index bcde0b1..d451386 100644 --- a/utils.nim +++ b/utils.nim @@ -35,6 +35,19 @@ proc createUTFMapping*(line: string): FingerTable = #echo fingerTable +proc utf16Len*(utf8Str: string): int = + result = 0 + for rune in utf8Str.runes: + case rune.int32 + of 0x0000 .. 0x007F, + 0x0080 .. 0x07FF, + 0x0800 .. 0xFFFF: + result += 1 + of 0x10000 .. 0x10FFFF: + result += 2 + else: + discard + proc utf16to8*(fingerTable: FingerTable, utf16pos: int): int = result = utf16pos for finger in fingerTable: @@ -43,10 +56,18 @@ proc utf16to8*(fingerTable: FingerTable, utf16pos: int): int = else: break +proc utf8to16*(fingerTable: FingerTable, utf8pos: int): int = + result = utf8pos + for finger in fingerTable: + if finger.u16pos < result: + result -= finger.offset + else: + break + when isMainModule: import termstyle var x = "heållo☀☀wor𐐀𐐀☀ld heållo☀wor𐐀ld heållo☀wor𐐀ld" - var fingerTable = populateUTFMapping(x) + var fingerTable = createUTFMapping(x) var corrected = utf16to8(fingerTable, 5) for y in x: