diff --git a/core/src/ast/mod.rs b/core/src/ast/mod.rs index cb055c055..65821227a 100644 --- a/core/src/ast/mod.rs +++ b/core/src/ast/mod.rs @@ -19,7 +19,8 @@ pub use enums::Enum; mod types; pub use types::{ - CustomType, LifetimeOrigin, ModSymbol, Mutability, PathType, PrimitiveType, TypeName, + CustomType, LifetimeOrigin, ModSymbol, Mutability, PathType, PrimitiveType, StringEncoding, + TypeName, }; mod lifetimes; diff --git a/core/src/ast/types.rs b/core/src/ast/types.rs index bf3ad6828..74d50aa97 100644 --- a/core/src/ast/types.rs +++ b/core/src/ast/types.rs @@ -382,7 +382,7 @@ pub enum TypeName { Result(Box, Box, bool), Writeable, /// A `&DiplomatStr` type. - StrReference(Lifetime), + StrReference(Lifetime, StringEncoding), /// A `&[T]` type, where `T` is a primitive. PrimitiveSlice(Lifetime, Mutability, PrimitiveType), /// The `()` type. @@ -391,6 +391,13 @@ pub enum TypeName { SelfType(PathType), } +#[derive(Clone, PartialEq, Eq, Hash, Serialize, Deserialize, Debug, Copy)] +#[non_exhaustive] +pub enum StringEncoding { + UnvalidatedUtf8, + UnvalidatedUtf16, +} + impl TypeName { /// Converts the [`TypeName`] back into an AST node that can be spliced into a program. pub fn to_syn(&self) -> syn::Type { @@ -493,11 +500,20 @@ impl TypeName { TypeName::Writeable => syn::parse_quote! { diplomat_runtime::DiplomatWriteable }, - TypeName::StrReference(lifetime) => syn::parse_str(&format!( - "{}DiplomatStr", - ReferenceDisplay(lifetime, &Mutability::Immutable) - )) - .unwrap(), + TypeName::StrReference(lifetime, StringEncoding::UnvalidatedUtf8) => { + syn::parse_str(&format!( + "{}DiplomatStr", + ReferenceDisplay(lifetime, &Mutability::Immutable) + )) + .unwrap() + } + TypeName::StrReference(lifetime, StringEncoding::UnvalidatedUtf16) => { + syn::parse_str(&format!( + "{}DiplomatStr16", + ReferenceDisplay(lifetime, &Mutability::Immutable) + )) + .unwrap() + } TypeName::PrimitiveSlice(lifetime, mutability, name) => { let primitive_name = PRIMITIVE_TO_STRING.get(name).unwrap(); let formatted_str = format!( @@ -532,11 +548,16 @@ impl TypeName { let lifetime = Lifetime::from(&r.lifetime); let mutability = Mutability::from_syn(&r.mutability); - if r.elem.to_token_stream().to_string() == "DiplomatStr" { + let name = r.elem.to_token_stream().to_string(); + if name.starts_with("DiplomatStr") { if mutability.is_mutable() { - panic!("mutable `DiplomatStr` references are disallowed"); + panic!("mutable `DiplomatStr*` references are disallowed"); + } + if name == "DiplomatStr" { + return TypeName::StrReference(lifetime, StringEncoding::UnvalidatedUtf8); + } else if name == "DiplomatStr16" { + return TypeName::StrReference(lifetime, StringEncoding::UnvalidatedUtf16); } - return TypeName::StrReference(lifetime); } if let syn::Type::Slice(slice) = &*r.elem { if let syn::Type::Path(p) = &*slice.elem { @@ -657,7 +678,7 @@ impl TypeName { ok.visit_lifetimes(visit)?; err.visit_lifetimes(visit) } - TypeName::StrReference(lt) => visit(lt, LifetimeOrigin::StrReference), + TypeName::StrReference(lt, ..) => visit(lt, LifetimeOrigin::StrReference), TypeName::PrimitiveSlice(lt, ..) => visit(lt, LifetimeOrigin::PrimitiveSlice), _ => ControlFlow::Continue(()), } @@ -866,7 +887,7 @@ impl TypeName { ok.check_lifetime_elision(full_type, in_path, env, errors); err.check_lifetime_elision(full_type, in_path, env, errors); } - TypeName::StrReference(Lifetime::Anonymous) + TypeName::StrReference(Lifetime::Anonymous, ..) | TypeName::PrimitiveSlice(Lifetime::Anonymous, ..) => { errors.push(ValidityError::LifetimeElisionInReturn { full_type: full_type.clone(), @@ -916,13 +937,20 @@ impl fmt::Display for TypeName { write!(f, "Result<{ok}, {err}>") } TypeName::Writeable => "DiplomatWriteable".fmt(f), - TypeName::StrReference(lifetime) => { + TypeName::StrReference(lifetime, StringEncoding::UnvalidatedUtf8) => { write!( f, "{}DiplomatStr", ReferenceDisplay(lifetime, &Mutability::Immutable) ) } + TypeName::StrReference(lifetime, StringEncoding::UnvalidatedUtf16) => { + write!( + f, + "{}DiplomatStr16", + ReferenceDisplay(lifetime, &Mutability::Immutable) + ) + } TypeName::PrimitiveSlice(lifetime, mutability, typ) => { write!(f, "{}[{typ}]", ReferenceDisplay(lifetime, mutability)) } diff --git a/core/src/hir/lowering.rs b/core/src/hir/lowering.rs index 7dc35af7f..a7431e30c 100644 --- a/core/src/hir/lowering.rs +++ b/core/src/hir/lowering.rs @@ -466,9 +466,10 @@ impl<'ast, 'errors> LoweringContext<'ast, 'errors> { )); None } - ast::TypeName::StrReference(lifetime) => { - Some(Type::Slice(Slice::Str(ltl?.lower_lifetime(lifetime)))) - } + ast::TypeName::StrReference(lifetime, encoding) => Some(Type::Slice(Slice::Str( + ltl?.lower_lifetime(lifetime), + *encoding, + ))), ast::TypeName::PrimitiveSlice(lifetime, mutability, prim) => { let borrow = Borrow::new(ltl?.lower_lifetime(lifetime), *mutability); let prim = PrimitiveType::from_ast(*prim); @@ -658,9 +659,10 @@ impl<'ast, 'errors> LoweringContext<'ast, 'errors> { )); None } - ast::TypeName::StrReference(lifetime) => { - Some(OutType::Slice(Slice::Str(ltl?.lower_lifetime(lifetime)))) - } + ast::TypeName::StrReference(lifetime, encoding) => Some(OutType::Slice(Slice::Str( + ltl?.lower_lifetime(lifetime), + *encoding, + ))), ast::TypeName::PrimitiveSlice(lifetime, mutability, prim) => { let borrow = Borrow::new(ltl?.lower_lifetime(lifetime), *mutability); let prim = PrimitiveType::from_ast(*prim); diff --git a/core/src/hir/snapshots/diplomat_core__hir__elision__tests__simple_mod.snap b/core/src/hir/snapshots/diplomat_core__hir__elision__tests__simple_mod.snap index 50b96b9b9..4d88b964f 100644 --- a/core/src/hir/snapshots/diplomat_core__hir__elision__tests__simple_mod.snap +++ b/core/src/hir/snapshots/diplomat_core__hir__elision__tests__simple_mod.snap @@ -67,6 +67,7 @@ TypeContext { 0, ), ), + UnvalidatedUtf8, ), ), }, @@ -128,6 +129,7 @@ TypeContext { 0, ), ), + UnvalidatedUtf8, ), ), }, @@ -179,6 +181,7 @@ TypeContext { 1, ), ), + UnvalidatedUtf8, ), ), }, @@ -193,6 +196,7 @@ TypeContext { 1, ), ), + UnvalidatedUtf8, ), ), ), diff --git a/core/src/hir/types.rs b/core/src/hir/types.rs index 429cc4573..0bb6eb42a 100644 --- a/core/src/hir/types.rs +++ b/core/src/hir/types.rs @@ -6,6 +6,7 @@ use super::{ }; use crate::ast; pub use ast::Mutability; +pub use ast::StringEncoding; /// Type that can only be used as an output. pub type OutType = Type; @@ -34,7 +35,7 @@ pub enum SelfType { #[non_exhaustive] pub enum Slice { /// A string slice, e.g. `&DiplomatStr`. - Str(MaybeStatic), + Str(MaybeStatic, StringEncoding), /// A primitive slice, e.g. `&mut [u8]`. Primitive(Borrow, PrimitiveType), @@ -89,8 +90,8 @@ impl Slice { /// variant. pub fn lifetime(&self) -> &MaybeStatic { match self { - Slice::Str(lifetime) => lifetime, - Slice::Primitive(reference, _) => &reference.lifetime, + Slice::Str(lifetime, ..) => lifetime, + Slice::Primitive(reference, ..) => &reference.lifetime, } } } diff --git a/example/js/lib/api/ICU4XLocale.js b/example/js/lib/api/ICU4XLocale.js index 07b6c155c..af17436dd 100644 --- a/example/js/lib/api/ICU4XLocale.js +++ b/example/js/lib/api/ICU4XLocale.js @@ -16,7 +16,7 @@ export class ICU4XLocale { } static new(arg_name) { - const buf_arg_name = diplomatRuntime.DiplomatBuf.str(wasm, arg_name); + const buf_arg_name = diplomatRuntime.DiplomatBuf.str8(wasm, arg_name); const diplomat_out = new ICU4XLocale(wasm.ICU4XLocale_new(buf_arg_name.ptr, buf_arg_name.size), true, []); buf_arg_name.free(); return diplomat_out; diff --git a/example/js/lib/api/diplomat-runtime.js b/example/js/lib/api/diplomat-runtime.js index 75317d4a7..5512934d1 100644 --- a/example/js/lib/api/diplomat-runtime.js +++ b/example/js/lib/api/diplomat-runtime.js @@ -1,15 +1,20 @@ -export function readString(wasm, ptr, len) { +export function readString8(wasm, ptr, len) { const buf = new Uint8Array(wasm.memory.buffer, ptr, len); return (new TextDecoder("utf-8")).decode(buf) } +export function readString16(wasm, ptr, len) { + const buf = new Uint16Array(wasm.memory.buffer, ptr, len); + return String.fromCharCode(buf) +} + export function withWriteable(wasm, callback) { const writeable = wasm.diplomat_buffer_writeable_create(0); try { callback(writeable); const outStringPtr = wasm.diplomat_buffer_writeable_get_bytes(writeable); const outStringLen = wasm.diplomat_buffer_writeable_len(writeable); - return readString(wasm, outStringPtr, outStringLen); + return readString8(wasm, outStringPtr, outStringLen); } finally { wasm.diplomat_buffer_writeable_destroy(writeable); } @@ -61,7 +66,7 @@ export function enumDiscriminant(wasm, ptr) { // they can create an edge to this object if they borrow from the str/slice, // or we can manually free the WASM memory if they don't. export class DiplomatBuf { - static str = (wasm, string) => { + static str8 = (wasm, string) => { var utf8_len = 0; for (const codepoint_string of string) { let codepoint = codepoint_string.codePointAt(0); @@ -78,8 +83,16 @@ export class DiplomatBuf { return new DiplomatBuf(wasm, utf8_len, 1, buf => { const result = (new TextEncoder()).encodeInto(string, buf); console.assert(string.length == result.read && utf8_len == result.written, "UTF-8 write error"); - }) -} + }) + } + + static str16 = (wasm, string) => { + return new DiplomatBuf(wasm, string.length, 2, buf => { + for (var i; i < string.length; i++) { + buf[i] = string.codePointAt(i); + } + }) + } static slice = (wasm, slice, align) => { // If the slice is not a Uint8Array, we have to convert to one, as that's the only diff --git a/example/js/lib/api/diplomat-wasm.mjs b/example/js/lib/api/diplomat-wasm.mjs index 2dd3293ba..70ad14546 100644 --- a/example/js/lib/api/diplomat-wasm.mjs +++ b/example/js/lib/api/diplomat-wasm.mjs @@ -1,27 +1,27 @@ import cfg from '../diplomat.config.js'; -import {readString} from './diplomat-runtime.js' +import {readString8} from './diplomat-runtime.js' let wasm; const imports = { env: { diplomat_console_debug_js(ptr, len) { - console.debug(readString(wasm, ptr, len)); + console.debug(readString8(wasm, ptr, len)); }, diplomat_console_error_js(ptr, len) { - console.error(readString(wasm, ptr, len)); + console.error(readString8(wasm, ptr, len)); }, diplomat_console_info_js(ptr, len) { - console.info(readString(wasm, ptr, len)); + console.info(readString8(wasm, ptr, len)); }, diplomat_console_log_js(ptr, len) { - console.log(readString(wasm, ptr, len)); + console.log(readString8(wasm, ptr, len)); }, diplomat_console_warn_js(ptr, len) { - console.warn(readString(wasm, ptr, len)); + console.warn(readString8(wasm, ptr, len)); }, diplomat_throw_error_js(ptr, len) { - throw new Error(readString(wasm, ptr, len)); + throw new Error(readString8(wasm, ptr, len)); } } } diff --git a/feature_tests/c2/include/BorrowedFields.d.h b/feature_tests/c2/include/BorrowedFields.d.h index cb99f725a..c3926bab1 100644 --- a/feature_tests/c2/include/BorrowedFields.d.h +++ b/feature_tests/c2/include/BorrowedFields.d.h @@ -14,7 +14,7 @@ extern "C" { typedef struct BorrowedFields { - struct { const uint16_t* data; size_t len; } a; + struct { const wchar_t* data; size_t len; } a; struct { const char* data; size_t len; } b; } BorrowedFields; diff --git a/feature_tests/cpp2/include/BorrowedFields.d.h b/feature_tests/cpp2/include/BorrowedFields.d.h index cb99f725a..c3926bab1 100644 --- a/feature_tests/cpp2/include/BorrowedFields.d.h +++ b/feature_tests/cpp2/include/BorrowedFields.d.h @@ -14,7 +14,7 @@ extern "C" { typedef struct BorrowedFields { - struct { const uint16_t* data; size_t len; } a; + struct { const wchar_t* data; size_t len; } a; struct { const char* data; size_t len; } b; } BorrowedFields; diff --git a/feature_tests/cpp2/include/BorrowedFields.d.hpp b/feature_tests/cpp2/include/BorrowedFields.d.hpp index 5c23a5b57..403082d3d 100644 --- a/feature_tests/cpp2/include/BorrowedFields.d.hpp +++ b/feature_tests/cpp2/include/BorrowedFields.d.hpp @@ -12,7 +12,7 @@ struct BorrowedFields { - diplomat::span a; + std::wstring_view a; std::string_view b; inline capi::BorrowedFields AsFFI() const; diff --git a/feature_tests/cpp2/include/BorrowedFields.hpp b/feature_tests/cpp2/include/BorrowedFields.hpp index 4eca7cae0..ab15a35c4 100644 --- a/feature_tests/cpp2/include/BorrowedFields.hpp +++ b/feature_tests/cpp2/include/BorrowedFields.hpp @@ -25,7 +25,7 @@ inline capi::BorrowedFields BorrowedFields::AsFFI() const { inline BorrowedFields BorrowedFields::FromFFI(capi::BorrowedFields c_struct) { return BorrowedFields { - .a = diplomat::span(c_struct.a_data, c_struct.a_size), + .a = std::wstring_view(c_struct.a_data, c_struct.a_size), .b = std::string_view(c_struct.b_data, c_struct.b_size), }; } diff --git a/feature_tests/dart/lib/BorrowedFields.g.dart b/feature_tests/dart/lib/BorrowedFields.g.dart index c3928c3ba..4c2d98c95 100644 --- a/feature_tests/dart/lib/BorrowedFields.g.dart +++ b/feature_tests/dart/lib/BorrowedFields.g.dart @@ -6,7 +6,7 @@ part of 'lib.g.dart'; final class _BorrowedFieldsFfi extends ffi.Struct { - external _SliceFfiUint16 a; + external _SliceFfiUtf16 a; external _SliceFfi2Utf8 b; } @@ -23,11 +23,11 @@ final class BorrowedFields { return result; } - Uint16List get a => _underlying.a._asDart; - set a(Uint16List a) { + String get a => _underlying.a._asDart; + set a(String a) { final alloc = ffi2.calloc; alloc.free(_underlying.a._bytes); - final aSlice = _SliceFfiUint16._fromDart(a, alloc); + final aSlice = _SliceFfiUtf16._fromDart(a, alloc); _underlying.a = aSlice; } diff --git a/feature_tests/dart/lib/lib.g.dart b/feature_tests/dart/lib/lib.g.dart index 510de20fa..fc8b5167e 100644 --- a/feature_tests/dart/lib/lib.g.dart +++ b/feature_tests/dart/lib/lib.g.dart @@ -213,8 +213,8 @@ final class _SliceFfiDouble extends ffi.Struct { int get hashCode => _length.hashCode; } -final class _SliceFfiUint16 extends ffi.Struct { - external ffi.Pointer _bytes; +final class _SliceFfiUtf16 extends ffi.Struct { + external ffi.Pointer _bytes; @ffi.Size() external int _length; @@ -222,27 +222,27 @@ final class _SliceFfiUint16 extends ffi.Struct { /// Produces a slice from a Dart object. The Dart object's data is copied into the given allocator /// as it cannot be borrowed directly, and gets freed with the slice object. // ignore: unused_element - static _SliceFfiUint16 _fromDart(Uint16List value, ffi.Allocator allocator) { - final pointer = allocator<_SliceFfiUint16>(); + static _SliceFfiUtf16 _fromDart(String value, ffi.Allocator allocator) { + final pointer = allocator<_SliceFfiUtf16>(); final slice = pointer.ref; slice._length = value.length; - slice._bytes = allocator(slice._length); - slice._bytes.asTypedList(slice._length).setAll(0, value); + slice._bytes = allocator(slice._length).cast(); + slice._bytes.cast().asTypedList(slice._length).setAll(0, value.codeUnits); return slice; } // ignore: unused_element - Uint16List get _asDart => _bytes.asTypedList(_length); + String get _asDart => String.fromCharCodes(_bytes.cast().asTypedList(_length)); // This is expensive @override bool operator ==(Object other) { - if (other is! _SliceFfiUint16 || other._length != _length) { + if (other is! _SliceFfiUtf16 || other._length != _length) { return false; } for (var i = 0; i < _length; i++) { - if (other._bytes[i] != _bytes[i]) { + if (other._bytes.cast()[i] != _bytes.cast()[i]) { return false; } } diff --git a/feature_tests/js/api/BorrowedFields.d.ts b/feature_tests/js/api/BorrowedFields.d.ts index d9d32cb3a..f7f77c0a4 100644 --- a/feature_tests/js/api/BorrowedFields.d.ts +++ b/feature_tests/js/api/BorrowedFields.d.ts @@ -2,6 +2,6 @@ /** */ export class BorrowedFields { - a: Uint16Array; + a: string; b: string; } diff --git a/feature_tests/js/api/BorrowedFields.js b/feature_tests/js/api/BorrowedFields.js index b3fe34ed2..b9869f7fe 100644 --- a/feature_tests/js/api/BorrowedFields.js +++ b/feature_tests/js/api/BorrowedFields.js @@ -5,11 +5,11 @@ export class BorrowedFields { constructor(underlying, edges_a) { this.a = (() => { const [ptr, size] = new Uint32Array(wasm.memory.buffer, underlying, 2); - return new Uint16Array(wasm.memory.buffer, ptr, size); + return diplomatRuntime.readString16(wasm, ptr, size); })(); this.b = (() => { const [ptr, size] = new Uint32Array(wasm.memory.buffer, underlying + 8, 2); - return diplomatRuntime.readString(wasm, ptr, size); + return diplomatRuntime.readString8(wasm, ptr, size); })(); } } diff --git a/feature_tests/js/api/BorrowedFieldsReturning.js b/feature_tests/js/api/BorrowedFieldsReturning.js index 8e27c35be..ca5da0625 100644 --- a/feature_tests/js/api/BorrowedFieldsReturning.js +++ b/feature_tests/js/api/BorrowedFieldsReturning.js @@ -5,7 +5,7 @@ export class BorrowedFieldsReturning { constructor(underlying, edges_a) { this.bytes = (() => { const [ptr, size] = new Uint32Array(wasm.memory.buffer, underlying, 2); - return diplomatRuntime.readString(wasm, ptr, size); + return diplomatRuntime.readString8(wasm, ptr, size); })(); } } diff --git a/feature_tests/js/api/Foo.js b/feature_tests/js/api/Foo.js index dbaa74e66..9179a21e9 100644 --- a/feature_tests/js/api/Foo.js +++ b/feature_tests/js/api/Foo.js @@ -18,7 +18,7 @@ export class Foo { } static new(arg_x) { - const buf_arg_x = diplomatRuntime.DiplomatBuf.str(wasm, arg_x); + const buf_arg_x = diplomatRuntime.DiplomatBuf.str8(wasm, arg_x); return new Foo(wasm.Foo_new(buf_arg_x.ptr, buf_arg_x.size), true, [buf_arg_x]); } @@ -27,7 +27,7 @@ export class Foo { } static new_static(arg_x) { - const buf_arg_x = diplomatRuntime.DiplomatBuf.str(wasm, arg_x); + const buf_arg_x = diplomatRuntime.DiplomatBuf.str8(wasm, arg_x); const diplomat_out = new Foo(wasm.Foo_new_static(buf_arg_x.ptr, buf_arg_x.size), true, []); buf_arg_x.leak(); return diplomat_out; @@ -45,9 +45,9 @@ export class Foo { static extract_from_fields(arg_fields) { const field_a_arg_fields = arg_fields["a"]; - const buf_field_a_arg_fields = diplomatRuntime.DiplomatBuf.slice(wasm, field_a_arg_fields, 2); + const buf_field_a_arg_fields = diplomatRuntime.DiplomatBuf.str16(wasm, field_a_arg_fields, 2); const field_b_arg_fields = arg_fields["b"]; - const buf_field_b_arg_fields = diplomatRuntime.DiplomatBuf.str(wasm, field_b_arg_fields); + const buf_field_b_arg_fields = diplomatRuntime.DiplomatBuf.str8(wasm, field_b_arg_fields); return new Foo(wasm.Foo_extract_from_fields(buf_field_a_arg_fields.ptr, buf_field_a_arg_fields.size, buf_field_b_arg_fields.ptr, buf_field_b_arg_fields.size), true, [buf_field_a_arg_fields, buf_field_b_arg_fields]); } } diff --git a/feature_tests/js/api/MyString.js b/feature_tests/js/api/MyString.js index a21b15453..c2861ff72 100644 --- a/feature_tests/js/api/MyString.js +++ b/feature_tests/js/api/MyString.js @@ -16,14 +16,14 @@ export class MyString { } static new(arg_v) { - const buf_arg_v = diplomatRuntime.DiplomatBuf.str(wasm, arg_v); + const buf_arg_v = diplomatRuntime.DiplomatBuf.str8(wasm, arg_v); const diplomat_out = new MyString(wasm.MyString_new(buf_arg_v.ptr, buf_arg_v.size), true, []); buf_arg_v.free(); return diplomat_out; } set_str(arg_new_str) { - const buf_arg_new_str = diplomatRuntime.DiplomatBuf.str(wasm, arg_new_str); + const buf_arg_new_str = diplomatRuntime.DiplomatBuf.str8(wasm, arg_new_str); wasm.MyString_set_str(this.underlying, buf_arg_new_str.ptr, buf_arg_new_str.size); buf_arg_new_str.free(); } diff --git a/feature_tests/js/api/diplomat-runtime.js b/feature_tests/js/api/diplomat-runtime.js index 75317d4a7..5512934d1 100644 --- a/feature_tests/js/api/diplomat-runtime.js +++ b/feature_tests/js/api/diplomat-runtime.js @@ -1,15 +1,20 @@ -export function readString(wasm, ptr, len) { +export function readString8(wasm, ptr, len) { const buf = new Uint8Array(wasm.memory.buffer, ptr, len); return (new TextDecoder("utf-8")).decode(buf) } +export function readString16(wasm, ptr, len) { + const buf = new Uint16Array(wasm.memory.buffer, ptr, len); + return String.fromCharCode(buf) +} + export function withWriteable(wasm, callback) { const writeable = wasm.diplomat_buffer_writeable_create(0); try { callback(writeable); const outStringPtr = wasm.diplomat_buffer_writeable_get_bytes(writeable); const outStringLen = wasm.diplomat_buffer_writeable_len(writeable); - return readString(wasm, outStringPtr, outStringLen); + return readString8(wasm, outStringPtr, outStringLen); } finally { wasm.diplomat_buffer_writeable_destroy(writeable); } @@ -61,7 +66,7 @@ export function enumDiscriminant(wasm, ptr) { // they can create an edge to this object if they borrow from the str/slice, // or we can manually free the WASM memory if they don't. export class DiplomatBuf { - static str = (wasm, string) => { + static str8 = (wasm, string) => { var utf8_len = 0; for (const codepoint_string of string) { let codepoint = codepoint_string.codePointAt(0); @@ -78,8 +83,16 @@ export class DiplomatBuf { return new DiplomatBuf(wasm, utf8_len, 1, buf => { const result = (new TextEncoder()).encodeInto(string, buf); console.assert(string.length == result.read && utf8_len == result.written, "UTF-8 write error"); - }) -} + }) + } + + static str16 = (wasm, string) => { + return new DiplomatBuf(wasm, string.length, 2, buf => { + for (var i; i < string.length; i++) { + buf[i] = string.codePointAt(i); + } + }) + } static slice = (wasm, slice, align) => { // If the slice is not a Uint8Array, we have to convert to one, as that's the only diff --git a/feature_tests/js/api/diplomat-wasm.mjs b/feature_tests/js/api/diplomat-wasm.mjs index 2dd3293ba..70ad14546 100644 --- a/feature_tests/js/api/diplomat-wasm.mjs +++ b/feature_tests/js/api/diplomat-wasm.mjs @@ -1,27 +1,27 @@ import cfg from '../diplomat.config.js'; -import {readString} from './diplomat-runtime.js' +import {readString8} from './diplomat-runtime.js' let wasm; const imports = { env: { diplomat_console_debug_js(ptr, len) { - console.debug(readString(wasm, ptr, len)); + console.debug(readString8(wasm, ptr, len)); }, diplomat_console_error_js(ptr, len) { - console.error(readString(wasm, ptr, len)); + console.error(readString8(wasm, ptr, len)); }, diplomat_console_info_js(ptr, len) { - console.info(readString(wasm, ptr, len)); + console.info(readString8(wasm, ptr, len)); }, diplomat_console_log_js(ptr, len) { - console.log(readString(wasm, ptr, len)); + console.log(readString8(wasm, ptr, len)); }, diplomat_console_warn_js(ptr, len) { - console.warn(readString(wasm, ptr, len)); + console.warn(readString8(wasm, ptr, len)); }, diplomat_throw_error_js(ptr, len) { - throw new Error(readString(wasm, ptr, len)); + throw new Error(readString8(wasm, ptr, len)); } } } diff --git a/feature_tests/src/lifetimes.rs b/feature_tests/src/lifetimes.rs index ff673082c..410ba106f 100644 --- a/feature_tests/src/lifetimes.rs +++ b/feature_tests/src/lifetimes.rs @@ -8,7 +8,7 @@ pub mod ffi { pub struct Bar<'b, 'a: 'b>(&'b Foo<'a>); pub struct BorrowedFields<'a> { - a: &'a [u16], + a: &'a DiplomatStr16, b: &'a DiplomatStr, } diff --git a/macro/src/lib.rs b/macro/src/lib.rs index 05491d148..7c42ad00a 100644 --- a/macro/src/lib.rs +++ b/macro/src/lib.rs @@ -15,11 +15,23 @@ fn cfgs_to_stream(attrs: &[Attribute]) -> proc_macro2::TokenStream { fn gen_params_at_boundary(param: &ast::Param, expanded_params: &mut Vec) { match ¶m.ty { - ast::TypeName::StrReference(_) | ast::TypeName::PrimitiveSlice(..) => { + ast::TypeName::StrReference( + .., + ast::StringEncoding::UnvalidatedUtf8 | ast::StringEncoding::UnvalidatedUtf16, + ) + | ast::TypeName::PrimitiveSlice(..) => { let data_type = if let ast::TypeName::PrimitiveSlice(.., prim) = ¶m.ty { ast::TypeName::Primitive(*prim).to_syn().to_token_stream() - } else { + } else if let ast::TypeName::StrReference(_, ast::StringEncoding::UnvalidatedUtf8) = + ¶m.ty + { quote! { u8 } + } else if let ast::TypeName::StrReference(_, ast::StringEncoding::UnvalidatedUtf16) = + ¶m.ty + { + quote! { u16 } + } else { + unreachable!() }; expanded_params.push(FnArg::Typed(PatType { attrs: vec![], @@ -82,7 +94,7 @@ fn gen_params_at_boundary(param: &ast::Param, expanded_params: &mut Vec) fn gen_params_invocation(param: &ast::Param, expanded_params: &mut Vec) { match ¶m.ty { - ast::TypeName::StrReference(_) | ast::TypeName::PrimitiveSlice(..) => { + ast::TypeName::StrReference(..) | ast::TypeName::PrimitiveSlice(..) => { let data_ident = Ident::new(&format!("{}_diplomat_data", param.name), Span::call_site()); let len_ident = Ident::new(&format!("{}_diplomat_len", param.name), Span::call_site()); diff --git a/runtime/src/lib.rs b/runtime/src/lib.rs index 87c434184..cec6a9fa9 100644 --- a/runtime/src/lib.rs +++ b/runtime/src/lib.rs @@ -18,6 +18,8 @@ pub type DiplomatChar = u32; pub type DiplomatStr = [u8]; +pub type DiplomatStr16 = [u16]; + /// Allocates a buffer of a given size in Rust's memory. /// /// # Safety diff --git a/tool/src/c/structs.rs b/tool/src/c/structs.rs index 68a960277..8bd25f230 100644 --- a/tool/src/c/structs.rs +++ b/tool/src/c/structs.rs @@ -82,8 +82,12 @@ pub fn gen_method( write!(out, ", ")?; } - if let ast::TypeName::StrReference(_) = ¶m.ty { + if let ast::TypeName::StrReference(_, ast::StringEncoding::UnvalidatedUtf8) = ¶m.ty { write!(out, "const char* {0}_data, size_t {0}_len", param.name)?; + } else if let ast::TypeName::StrReference(_, ast::StringEncoding::UnvalidatedUtf16) = + ¶m.ty + { + write!(out, "const uint16_t* {0}_data, size_t {0}_len", param.name)?; } else if let ast::TypeName::PrimitiveSlice(_, mutability, prim) = ¶m.ty { write!( out, diff --git a/tool/src/c/types.rs b/tool/src/c/types.rs index 5a86c40f0..cac28b6b6 100644 --- a/tool/src/c/types.rs +++ b/tool/src/c/types.rs @@ -53,7 +53,12 @@ pub fn gen_type( } ast::TypeName::Writeable => write!(out, "DiplomatWriteable")?, - ast::TypeName::StrReference(..) => write!(out, "DiplomatStringView")?, + ast::TypeName::StrReference(_, ast::StringEncoding::UnvalidatedUtf8) => { + write!(out, "DiplomatStringView")? + } + ast::TypeName::StrReference(_, ast::StringEncoding::UnvalidatedUtf16) => { + write!(out, "DiplomatU16View")? + } ast::TypeName::PrimitiveSlice(_lt, mutability, prim) => { if mutability.is_mutable() { panic!("Mutable slices in structs not supported"); @@ -97,7 +102,12 @@ pub fn name_for_type(typ: &ast::TypeName) -> ast::Ident { name_for_type(err) )), ast::TypeName::Writeable => ast::Ident::from("writeable"), - ast::TypeName::StrReference(_) => ast::Ident::from("str_ref"), + ast::TypeName::StrReference(_, ast::StringEncoding::UnvalidatedUtf8) => { + ast::Ident::from("str_ref8") + } + ast::TypeName::StrReference(_, ast::StringEncoding::UnvalidatedUtf16) => { + ast::Ident::from("str_ref16") + } ast::TypeName::PrimitiveSlice(_lt, ast::Mutability::Mutable, prim) => { ast::Ident::from(format!("ref_mut_prim_slice_{}", c_type_for_prim(prim))) } diff --git a/tool/src/c2/formatter.rs b/tool/src/c2/formatter.rs index f5a0ef431..ade7a10f6 100644 --- a/tool/src/c2/formatter.rs +++ b/tool/src/c2/formatter.rs @@ -1,7 +1,7 @@ //! This module contains functions for formatting types use super::ty::ResultType; -use diplomat_core::hir::{self, OpaqueOwner, Type, TypeContext, TypeId}; +use diplomat_core::hir::{self, OpaqueOwner, StringEncoding, Type, TypeContext, TypeId}; use std::borrow::Cow; /// This type mediates all formatting @@ -109,7 +109,8 @@ impl<'tcx> CFormatter<'tcx> { } Type::Struct(s) => self.fmt_type_name(P::id_for_path(s)), Type::Enum(e) => self.fmt_type_name(e.tcx_id.into()), - Type::Slice(hir::Slice::Str(_)) => "str_ref".into(), + Type::Slice(hir::Slice::Str(_, StringEncoding::UnvalidatedUtf8)) => "str_ref8".into(), + Type::Slice(hir::Slice::Str(_, StringEncoding::UnvalidatedUtf16)) => "str_ref16".into(), Type::Slice(hir::Slice::Primitive(borrow, p)) => { let constness = borrow.mutability.if_mut_else("", "const_"); let prim = self.fmt_primitive_as_c(*p); diff --git a/tool/src/c2/ty.rs b/tool/src/c2/ty.rs index 5dd6f24a4..b8115ce0b 100644 --- a/tool/src/c2/ty.rs +++ b/tool/src/c2/ty.rs @@ -237,12 +237,20 @@ impl<'ccx, 'tcx: 'ccx, 'header> TyGenContext<'ccx, 'tcx, 'header> { ) -> Vec<(Cow<'ccx, str>, Cow<'a, str>)> { let param_name = self.cx.formatter.fmt_param_name(ident); match ty { - Type::Slice(hir::Slice::Str(..)) if !is_struct => { + Type::Slice(hir::Slice::Str(_, hir::StringEncoding::UnvalidatedUtf8)) if !is_struct => { vec![ ("const char*".into(), format!("{param_name}_data").into()), ("size_t".into(), format!("{param_name}_len").into()), ] } + Type::Slice(hir::Slice::Str(_, hir::StringEncoding::UnvalidatedUtf16)) + if !is_struct => + { + vec![ + ("const wchar_t*".into(), format!("{param_name}_data").into()), + ("size_t".into(), format!("{param_name}_len").into()), + ] + } Type::Slice(hir::Slice::Primitive(b, p)) if !is_struct => { let prim = self.cx.formatter.fmt_primitive_as_c(*p); let ptr_type = self.cx.formatter.fmt_ptr(&prim, b.mutability); @@ -314,7 +322,8 @@ impl<'ccx, 'tcx: 'ccx, 'header> TyGenContext<'ccx, 'tcx, 'header> { } Type::Slice(ref s) => { let ptr_ty = match s { - hir::Slice::Str(..) => "char".into(), + hir::Slice::Str(_, hir::StringEncoding::UnvalidatedUtf8) => "char".into(), + hir::Slice::Str(_, hir::StringEncoding::UnvalidatedUtf16) => "wchar_t".into(), hir::Slice::Primitive(_, prim) => self.cx.formatter.fmt_primitive_as_c(*prim), &_ => unreachable!("unknown AST/HIR variant"), }; diff --git a/tool/src/cpp/conversions.rs b/tool/src/cpp/conversions.rs index 83bdff4ea..9ea3e5696 100644 --- a/tool/src/cpp/conversions.rs +++ b/tool/src/cpp/conversions.rs @@ -203,7 +203,7 @@ pub fn gen_rust_to_cpp( todo!("Returning references from Rust to C++ is not currently supported") } ast::TypeName::Writeable => panic!("Returning writeables is not supported"), - ast::TypeName::StrReference(..) => { + ast::TypeName::StrReference(_, ast::StringEncoding::UnvalidatedUtf8) => { let raw_value_id = format!("diplomat_str_raw_{path}"); writeln!(out, "capi::DiplomatStringView {raw_value_id} = {cpp};").unwrap(); @@ -214,6 +214,18 @@ pub fn gen_rust_to_cpp( .unwrap(); "str".into() } + ast::TypeName::StrReference(_, ast::StringEncoding::UnvalidatedUtf16) => { + let raw_value_id = format!("diplomat_slice_raw_{path}"); + writeln!(out, "capi::DiplomatU16View {raw_value_id} = {cpp};").unwrap(); + + let span = &library_config.span.expr; + writeln!( + out, + "{span} slice({raw_value_id}.data, {raw_value_id}.len);" + ) + .unwrap(); + "slice".into() + } ast::TypeName::PrimitiveSlice(_lt, mutability, prim) => { assert!(mutability.is_immutable()); let raw_value_id = format!("diplomat_slice_raw_{path}"); @@ -354,7 +366,7 @@ pub fn gen_cpp_to_rust( } } ast::TypeName::Primitive(_) => cpp.to_string(), - ast::TypeName::StrReference(_) => { + ast::TypeName::StrReference(..) => { format!("{{ {cpp}.data(), {cpp}.size() }}") } ast::TypeName::PrimitiveSlice(..) => { diff --git a/tool/src/cpp/types.rs b/tool/src/cpp/types.rs index 6f58214c3..abf5a6ab6 100644 --- a/tool/src/cpp/types.rs +++ b/tool/src/cpp/types.rs @@ -155,11 +155,15 @@ fn gen_type_inner( write!(out, "capi::DiplomatWriteable")?; } - ast::TypeName::StrReference(_) => { + ast::TypeName::StrReference(_, ast::StringEncoding::UnvalidatedUtf8) => { let maybe_const = if in_struct { "" } else { "const " }; write!(out, "{maybe_const}{}", library_config.string_view.expr)?; } + ast::TypeName::StrReference(_, ast::StringEncoding::UnvalidatedUtf16) => { + write!(out, "const {}", library_config.span.expr)?; + } + ast::TypeName::PrimitiveSlice(_, ast::Mutability::Mutable, prim) => { write!( out, diff --git a/tool/src/cpp2/formatter.rs b/tool/src/cpp2/formatter.rs index 26e34b79e..430fa225d 100644 --- a/tool/src/cpp2/formatter.rs +++ b/tool/src/cpp2/formatter.rs @@ -123,6 +123,7 @@ impl<'tcx> Cpp2Formatter<'tcx> { ident: &'a str, mutability: hir::Mutability, ) -> Cow<'a, str> { + // TODO: This needs to change if an abstraction other than std::span is used // TODO: Where is the right place to put `const` here? if mutability.is_mutable() { format!("diplomat::span<{ident}>").into() @@ -131,10 +132,16 @@ impl<'tcx> Cpp2Formatter<'tcx> { } } - pub fn fmt_borrowed_str(&self) -> Cow<'static, str> { + pub fn fmt_borrowed_utf8_str(&self) -> Cow<'static, str> { + // TODO: This needs to change if an abstraction other than std::u8string_view is used "std::string_view".into() } + pub fn fmt_borrowed_utf16_str(&self) -> Cow<'static, str> { + // TODO: This needs to change if an abstraction other than std::u16string_view is used + "std::wstring_view".into() + } + pub fn fmt_owned_str(&self) -> Cow<'static, str> { "std::string".into() } diff --git a/tool/src/cpp2/ty.rs b/tool/src/cpp2/ty.rs index 3cda5a95b..8fc240d31 100644 --- a/tool/src/cpp2/ty.rs +++ b/tool/src/cpp2/ty.rs @@ -454,7 +454,12 @@ impl<'ccx, 'tcx: 'ccx, 'header> TyGenContext<'ccx, 'tcx, 'header> { .insert(self.cx.formatter.fmt_impl_header_path(id)); type_name } - Type::Slice(hir::Slice::Str(_lifetime)) => self.cx.formatter.fmt_borrowed_str(), + Type::Slice(hir::Slice::Str(_, hir::StringEncoding::UnvalidatedUtf8)) => { + self.cx.formatter.fmt_borrowed_utf8_str() + } + Type::Slice(hir::Slice::Str(_, hir::StringEncoding::UnvalidatedUtf16)) => { + self.cx.formatter.fmt_borrowed_utf16_str() + } Type::Slice(hir::Slice::Primitive(b, p)) => { let ret = self.cx.formatter.fmt_primitive_as_c(p); let ret = self.cx.formatter.fmt_borrowed_slice(&ret, b.mutability); @@ -537,21 +542,7 @@ impl<'ccx, 'tcx: 'ccx, 'header> TyGenContext<'ccx, 'tcx, 'header> { expression: format!("{cpp_name}.AsFFI()").into(), }] } - Type::Slice(hir::Slice::Str(..)) => { - // TODO: This needs to change if an abstraction other than std::string_view is used - vec![ - PartiallyNamedExpression { - suffix: "_data".into(), - expression: format!("{cpp_name}.data()").into(), - }, - PartiallyNamedExpression { - suffix: "_size".into(), - expression: format!("{cpp_name}.size()").into(), - }, - ] - } - Type::Slice(hir::Slice::Primitive(..)) => { - // TODO: This needs to change if an abstraction other than std::span is used + Type::Slice(hir::Slice::Str(..)) | Type::Slice(hir::Slice::Primitive(..)) => { vec![ PartiallyNamedExpression { suffix: "_data".into(), @@ -653,13 +644,15 @@ impl<'ccx, 'tcx: 'ccx, 'header> TyGenContext<'ccx, 'tcx, 'header> { // Note: The impl file is imported in gen_type_name(). format!("{type_name}::FromFFI({var_name})").into() } - Type::Slice(hir::Slice::Str(..)) => { - // TODO: This needs to change if an abstraction other than std::string_view is used - let string_view = self.cx.formatter.fmt_borrowed_str(); + Type::Slice(hir::Slice::Str(_, hir::StringEncoding::UnvalidatedUtf8)) => { + let string_view = self.cx.formatter.fmt_borrowed_utf8_str(); + format!("{string_view}({var_name}_data, {var_name}_size)").into() + } + Type::Slice(hir::Slice::Str(_, hir::StringEncoding::UnvalidatedUtf16)) => { + let string_view = self.cx.formatter.fmt_borrowed_utf16_str(); format!("{string_view}({var_name}_data, {var_name}_size)").into() } Type::Slice(hir::Slice::Primitive(b, p)) => { - // TODO: This needs to change if an abstraction other than std::span is used let prim_name = self.cx.formatter.fmt_primitive_as_c(p); let span = self .cx diff --git a/tool/src/dart/formatter.rs b/tool/src/dart/formatter.rs index 832e3467b..4e569429c 100644 --- a/tool/src/dart/formatter.rs +++ b/tool/src/dart/formatter.rs @@ -169,6 +169,10 @@ impl<'tcx> DartFormatter<'tcx> { "ffi2.Utf8" } + pub fn fmt_utf16_primitive(&self) -> &'static str { + "ffi2.Utf16" + } + pub fn fmt_void(&self) -> &'static str { "void" } @@ -277,7 +281,11 @@ impl<'tcx> DartFormatter<'tcx> { } } - pub fn fmt_str_slice_type(&self) -> &'static str { + pub fn fmt_utf8_slice_type(&self) -> &'static str { "_SliceFfi2Utf8" } + + pub fn fmt_utf16_slice_type(&self) -> &'static str { + "_SliceFfiUtf16" + } } diff --git a/tool/src/dart/mod.rs b/tool/src/dart/mod.rs index 4d64070ef..f7e53e157 100644 --- a/tool/src/dart/mod.rs +++ b/tool/src/dart/mod.rs @@ -502,7 +502,7 @@ impl<'a, 'cx> TyGenContext<'a, 'cx> { } type_name } - Type::Slice(hir::Slice::Str(_lifetime)) => self.formatter.fmt_string().into(), + Type::Slice(hir::Slice::Str(..)) => self.formatter.fmt_string().into(), Type::Slice(hir::Slice::Primitive(_, p)) => { self.imports .insert(self.formatter.fmt_import("dart:typed_data")); @@ -545,7 +545,12 @@ impl<'a, 'cx> TyGenContext<'a, 'cx> { } self.formatter.fmt_enum_as_ffi(cast).into() } - Type::Slice(hir::Slice::Str(_lifetime)) => self.formatter.fmt_utf8_primitive().into(), + Type::Slice(hir::Slice::Str(_, hir::StringEncoding::UnvalidatedUtf8)) => { + self.formatter.fmt_utf8_primitive().into() + } + Type::Slice(hir::Slice::Str(_, hir::StringEncoding::UnvalidatedUtf16)) => { + self.formatter.fmt_utf16_primitive().into() + } Type::Slice(hir::Slice::Primitive(_, p)) => { self.formatter.fmt_primitive_as_ffi(p, false).into() } @@ -662,36 +667,57 @@ impl<'a, 'cx> TyGenContext<'a, 'cx> { }; let slice_ty = match slice { - hir::Slice::Str(..) => self.formatter.fmt_str_slice_type(), + hir::Slice::Str(_, hir::StringEncoding::UnvalidatedUtf8) => { + self.formatter.fmt_utf8_slice_type() + } + hir::Slice::Str(_, hir::StringEncoding::UnvalidatedUtf16) => { + self.formatter.fmt_utf16_slice_type() + } hir::Slice::Primitive(_, p) => self.formatter.fmt_slice_type(*p), _ => todo!("{slice:?}"), }; let ffi_type = match slice { - hir::Slice::Str(..) => self.formatter.fmt_utf8_primitive(), + hir::Slice::Str(_, hir::StringEncoding::UnvalidatedUtf8) => { + self.formatter.fmt_utf8_primitive() + } + hir::Slice::Str(_, hir::StringEncoding::UnvalidatedUtf16) => { + self.formatter.fmt_utf16_primitive() + } hir::Slice::Primitive(_, p) => self.formatter.fmt_primitive_as_ffi(*p, false), _ => todo!("{slice:?}"), }; let to_dart = match slice { - hir::Slice::Str(..) => { + hir::Slice::Str(_, hir::StringEncoding::UnvalidatedUtf8) => { self.imports .insert(self.formatter.fmt_import("dart:convert")); "Utf8Decoder().convert(_bytes.cast().asTypedList(_length))" } + hir::Slice::Str(_, hir::StringEncoding::UnvalidatedUtf16) => { + self.imports + .insert(self.formatter.fmt_import("dart:convert")); + "String.fromCharCodes(_bytes.cast().asTypedList(_length))" + } // TODO: How to read ffi.Size? hir::Slice::Primitive(_, hir::PrimitiveType::IntSize(_)) => "this", _ => "_bytes.asTypedList(_length)", }; let from_dart = match slice { - hir::Slice::Str(..) => concat!( + hir::Slice::Str(_, hir::StringEncoding::UnvalidatedUtf8) => concat!( "final units = Utf8Encoder().convert(value);\n", "slice._length = units.length;\n", // TODO: Figure out why Pointer cannot be allocated "slice._bytes = allocator(slice._length).cast();\n", "slice._bytes.cast().asTypedList(slice._length).setAll(0, units);" ), + hir::Slice::Str(_, hir::StringEncoding::UnvalidatedUtf16) => concat!( + "slice._length = value.length;\n", + // TODO: Figure out why Pointer cannot be allocated + "slice._bytes = allocator(slice._length).cast();\n", + "slice._bytes.cast().asTypedList(slice._length).setAll(0, value.codeUnits);" + ), hir::Slice::Primitive(_, hir::PrimitiveType::IntSize(_)) => "", _ => concat!( "slice._length = value.length;\n", diff --git a/tool/src/dotnet/raw.rs b/tool/src/dotnet/raw.rs index 052c1cee1..95bb229ef 100644 --- a/tool/src/dotnet/raw.rs +++ b/tool/src/dotnet/raw.rs @@ -385,9 +385,12 @@ fn gen_param( write!(out, "DiplomatWriteable* {name}") } else { match typ { - ast::TypeName::StrReference(..) => { + ast::TypeName::StrReference(_, ast::StringEncoding::UnvalidatedUtf8) => { write!(out, "byte* {name}, nuint {name}Sz") } + ast::TypeName::StrReference(..) => { + write!(out, "ushort* {name}, nuint {name}Sz") + } ast::TypeName::PrimitiveSlice(.., prim) => { write!( out, diff --git a/tool/src/dotnet/types.rs b/tool/src/dotnet/types.rs index 596486950..5feb38b3c 100644 --- a/tool/src/dotnet/types.rs +++ b/tool/src/dotnet/types.rs @@ -49,10 +49,14 @@ pub fn gen_type_name( write!(out, "DiplomatWriteable") } - ast::TypeName::StrReference(..) => { + ast::TypeName::StrReference(_, ast::StringEncoding::UnvalidatedUtf8) => { write!(out, "string") } + ast::TypeName::StrReference(_, ast::StringEncoding::UnvalidatedUtf16) => { + write!(out, "ushort[]") + } + ast::TypeName::PrimitiveSlice(.., prim) => { write!(out, "{}[]", type_name_for_prim(prim)) } @@ -108,7 +112,12 @@ pub fn name_for_type(typ: &ast::TypeName) -> ast::Ident { ast::Ident::from(format!("Result{}{}", name_for_type(ok), name_for_type(err))) } ast::TypeName::Writeable => ast::Ident::from("Writeable"), - ast::TypeName::StrReference(_) => ast::Ident::from("StrRef"), + ast::TypeName::StrReference(_, ast::StringEncoding::UnvalidatedUtf8) => { + ast::Ident::from("StrRef8") + } + ast::TypeName::StrReference(_, ast::StringEncoding::UnvalidatedUtf16) => { + ast::Ident::from("RefMutPrimSliceU16") + } ast::TypeName::PrimitiveSlice(_, ast::Mutability::Mutable, prim) => ast::Ident::from( format!("RefMutPrimSlice{}", prim.to_string().to_upper_camel_case()), ), diff --git a/tool/src/js/conversions.rs b/tool/src/js/conversions.rs index 86c892fbf..8f69aba9f 100644 --- a/tool/src/js/conversions.rs +++ b/tool/src/js/conversions.rs @@ -114,7 +114,7 @@ pub fn gen_value_js_to_rust<'env>( entries: &mut BTreeMap<&'env ast::NamedLifetime, Vec>>, ) { match typ { - ast::TypeName::StrReference(lifetime) | ast::TypeName::PrimitiveSlice(lifetime, ..) => { + ast::TypeName::StrReference(lifetime, ..) | ast::TypeName::PrimitiveSlice(lifetime, ..) => { let param_name_buf = Argument::DiplomatBuf(param_name.clone()); // TODO: turn `gen_value_js_to_rust` into a struct and add a // `display_slice` method so we can use the `SliceKind` type here to @@ -124,9 +124,16 @@ pub fn gen_value_js_to_rust<'env>( "const {param_name_buf} = diplomatRuntime.DiplomatBuf.slice(wasm, {param_name}, {align});", align = layout::primitive_size_alignment(*prim).align() )); + } else if matches!( + typ, + ast::TypeName::StrReference(_, ast::StringEncoding::UnvalidatedUtf16) + ) { + pre_logic.push(format!( + "const {param_name_buf} = diplomatRuntime.DiplomatBuf.str16(wasm, {param_name}, 2);", + )); } else { pre_logic.push(format!( - "const {param_name_buf} = diplomatRuntime.DiplomatBuf.str(wasm, {param_name});" + "const {param_name_buf} = diplomatRuntime.DiplomatBuf.str8(wasm, {param_name});" )); } @@ -506,7 +513,8 @@ impl fmt::Display for InvocationIntoJs<'_> { ReturnTypeForm::Empty => unreachable!(), } } - ast::TypeName::StrReference(..) => self.display_slice(SliceKind::Str).fmt(f), + ast::TypeName::StrReference(_, ast::StringEncoding::UnvalidatedUtf8) => self.display_slice(SliceKind::Str).fmt(f), + ast::TypeName::StrReference(_, ast::StringEncoding::UnvalidatedUtf16) => self.display_slice(SliceKind::Str16).fmt(f), ast::TypeName::PrimitiveSlice(.., prim) => { self.display_slice(SliceKind::Primitive(prim.into())).fmt(f) } @@ -521,13 +529,15 @@ impl fmt::Display for InvocationIntoJs<'_> { /// where the implementations are largely the same. enum SliceKind { Str, + Str16, Primitive(JsPrimitive), } impl SliceKind { fn display<'a>(&'a self, ptr: &'a ast::Ident, size: &'a ast::Ident) -> impl fmt::Display + 'a { display::expr(move |f| match self { - SliceKind::Str => write!(f, "diplomatRuntime.readString(wasm, {ptr}, {size})"), + SliceKind::Str => write!(f, "diplomatRuntime.readString8(wasm, {ptr}, {size})"), + SliceKind::Str16 => write!(f, "diplomatRuntime.readString16(wasm, {ptr}, {size})"), SliceKind::Primitive(prim) => match prim { JsPrimitive::Number(num) => { write!(f, "new {num}Array(wasm.memory.buffer, ptr, size)") @@ -779,7 +789,12 @@ impl fmt::Display for UnderlyingIntoJs<'_> { todo!("Result in a buffer") } ast::TypeName::Writeable => todo!("Writeable in a buffer"), - ast::TypeName::StrReference(..) => self.display_slice(SliceKind::Str).fmt(f), + ast::TypeName::StrReference(_, ast::StringEncoding::UnvalidatedUtf8) => { + self.display_slice(SliceKind::Str).fmt(f) + } + ast::TypeName::StrReference(_, ast::StringEncoding::UnvalidatedUtf16) => { + self.display_slice(SliceKind::Str16).fmt(f) + } ast::TypeName::PrimitiveSlice(.., prim) => { self.display_slice(SliceKind::Primitive(prim.into())).fmt(f) } diff --git a/tool/src/js/runtime.mjs b/tool/src/js/runtime.mjs index 75317d4a7..5512934d1 100644 --- a/tool/src/js/runtime.mjs +++ b/tool/src/js/runtime.mjs @@ -1,15 +1,20 @@ -export function readString(wasm, ptr, len) { +export function readString8(wasm, ptr, len) { const buf = new Uint8Array(wasm.memory.buffer, ptr, len); return (new TextDecoder("utf-8")).decode(buf) } +export function readString16(wasm, ptr, len) { + const buf = new Uint16Array(wasm.memory.buffer, ptr, len); + return String.fromCharCode(buf) +} + export function withWriteable(wasm, callback) { const writeable = wasm.diplomat_buffer_writeable_create(0); try { callback(writeable); const outStringPtr = wasm.diplomat_buffer_writeable_get_bytes(writeable); const outStringLen = wasm.diplomat_buffer_writeable_len(writeable); - return readString(wasm, outStringPtr, outStringLen); + return readString8(wasm, outStringPtr, outStringLen); } finally { wasm.diplomat_buffer_writeable_destroy(writeable); } @@ -61,7 +66,7 @@ export function enumDiscriminant(wasm, ptr) { // they can create an edge to this object if they borrow from the str/slice, // or we can manually free the WASM memory if they don't. export class DiplomatBuf { - static str = (wasm, string) => { + static str8 = (wasm, string) => { var utf8_len = 0; for (const codepoint_string of string) { let codepoint = codepoint_string.codePointAt(0); @@ -78,8 +83,16 @@ export class DiplomatBuf { return new DiplomatBuf(wasm, utf8_len, 1, buf => { const result = (new TextEncoder()).encodeInto(string, buf); console.assert(string.length == result.read && utf8_len == result.written, "UTF-8 write error"); - }) -} + }) + } + + static str16 = (wasm, string) => { + return new DiplomatBuf(wasm, string.length, 2, buf => { + for (var i; i < string.length; i++) { + buf[i] = string.codePointAt(i); + } + }) + } static slice = (wasm, slice, align) => { // If the slice is not a Uint8Array, we have to convert to one, as that's the only diff --git a/tool/src/js/snapshots/diplomat_tool__js__conversions__tests__str_borrowing@MyStruct.js.snap b/tool/src/js/snapshots/diplomat_tool__js__conversions__tests__str_borrowing@MyStruct.js.snap index 113abd8b4..6840f1481 100644 --- a/tool/src/js/snapshots/diplomat_tool__js__conversions__tests__str_borrowing@MyStruct.js.snap +++ b/tool/src/js/snapshots/diplomat_tool__js__conversions__tests__str_borrowing@MyStruct.js.snap @@ -9,12 +9,12 @@ export class MyStruct { constructor(underlying, edges_a) { this.s = (() => { const [ptr, size] = new Uint32Array(wasm.memory.buffer, underlying, 2); - return diplomatRuntime.readString(wasm, ptr, size); + return diplomatRuntime.readString8(wasm, ptr, size); })(); } static new(arg_s) { - const buf_arg_s = diplomatRuntime.DiplomatBuf.str(wasm, arg_s); + const buf_arg_s = diplomatRuntime.DiplomatBuf.str8(wasm, arg_s); return (() => { const diplomat_receive_buffer = wasm.diplomat_alloc(8, 4); wasm.MyStruct_new(diplomat_receive_buffer, buf_arg_s.ptr, buf_arg_s.size); @@ -26,13 +26,13 @@ export class MyStruct { get() { const field_s_this = this["s"]; - const buf_field_s_this = diplomatRuntime.DiplomatBuf.str(wasm, field_s_this); + const buf_field_s_this = diplomatRuntime.DiplomatBuf.str8(wasm, field_s_this); return (() => { const diplomat_receive_buffer = wasm.diplomat_alloc(8, 4); wasm.MyStruct_get(diplomat_receive_buffer, buf_field_s_this.ptr, buf_field_s_this.size); const [ptr, size] = new Uint32Array(wasm.memory.buffer, diplomat_receive_buffer, 2); wasm.diplomat_free(diplomat_receive_buffer, 8, 4); - return diplomatRuntime.readString(wasm, ptr, size); + return diplomatRuntime.readString8(wasm, ptr, size); })(); } } diff --git a/tool/src/js/snapshots/diplomat_tool__js__structs__tests__method_taking_str@MyStruct.js.snap b/tool/src/js/snapshots/diplomat_tool__js__structs__tests__method_taking_str@MyStruct.js.snap index a6d41251d..02efb93df 100644 --- a/tool/src/js/snapshots/diplomat_tool__js__structs__tests__method_taking_str@MyStruct.js.snap +++ b/tool/src/js/snapshots/diplomat_tool__js__structs__tests__method_taking_str@MyStruct.js.snap @@ -20,14 +20,14 @@ export class MyStruct { } static new_str(arg_v) { - const buf_arg_v = diplomatRuntime.DiplomatBuf.str(wasm, arg_v); + const buf_arg_v = diplomatRuntime.DiplomatBuf.str8(wasm, arg_v); const diplomat_out = new MyStruct(wasm.MyStruct_new_str(buf_arg_v.ptr, buf_arg_v.size), true, []); buf_arg_v.free(); return diplomat_out; } set_str(arg_new_str) { - const buf_arg_new_str = diplomatRuntime.DiplomatBuf.str(wasm, arg_new_str); + const buf_arg_new_str = diplomatRuntime.DiplomatBuf.str8(wasm, arg_new_str); wasm.MyStruct_set_str(this.underlying, buf_arg_new_str.ptr, buf_arg_new_str.size); buf_arg_new_str.free(); } diff --git a/tool/src/js/snapshots/diplomat_tool__js__types__tests__string_reference@MyStruct.js.snap b/tool/src/js/snapshots/diplomat_tool__js__types__tests__string_reference@MyStruct.js.snap index 1660ffbbd..d439d9477 100644 --- a/tool/src/js/snapshots/diplomat_tool__js__types__tests__string_reference@MyStruct.js.snap +++ b/tool/src/js/snapshots/diplomat_tool__js__types__tests__string_reference@MyStruct.js.snap @@ -12,7 +12,7 @@ export class MyStruct { } static new(arg_v) { - const buf_arg_v = diplomatRuntime.DiplomatBuf.str(wasm, arg_v); + const buf_arg_v = diplomatRuntime.DiplomatBuf.str8(wasm, arg_v); const diplomat_out = (() => { const diplomat_receive_buffer = wasm.diplomat_alloc(2, 1); wasm.MyStruct_new(diplomat_receive_buffer, buf_arg_v.ptr, buf_arg_v.size); diff --git a/tool/src/js/structs.rs b/tool/src/js/structs.rs index f4c1f5cb0..f33dbcd81 100644 --- a/tool/src/js/structs.rs +++ b/tool/src/js/structs.rs @@ -500,7 +500,10 @@ pub fn gen_ts_type( return Ok(opt); } ast::TypeName::Writeable => unreachable!(), - ast::TypeName::StrReference(_) => out.write_str("string")?, + ast::TypeName::StrReference( + _, + ast::StringEncoding::UnvalidatedUtf8 | ast::StringEncoding::UnvalidatedUtf16, + ) => out.write_str("string")?, ast::TypeName::PrimitiveSlice(.., prim) => match prim { ast::PrimitiveType::i8 => write!(out, "Int8Array")?, ast::PrimitiveType::u8 => write!(out, "Uint8Array")?, diff --git a/tool/src/js/wasm.mjs b/tool/src/js/wasm.mjs index 2dd3293ba..70ad14546 100644 --- a/tool/src/js/wasm.mjs +++ b/tool/src/js/wasm.mjs @@ -1,27 +1,27 @@ import cfg from '../diplomat.config.js'; -import {readString} from './diplomat-runtime.js' +import {readString8} from './diplomat-runtime.js' let wasm; const imports = { env: { diplomat_console_debug_js(ptr, len) { - console.debug(readString(wasm, ptr, len)); + console.debug(readString8(wasm, ptr, len)); }, diplomat_console_error_js(ptr, len) { - console.error(readString(wasm, ptr, len)); + console.error(readString8(wasm, ptr, len)); }, diplomat_console_info_js(ptr, len) { - console.info(readString(wasm, ptr, len)); + console.info(readString8(wasm, ptr, len)); }, diplomat_console_log_js(ptr, len) { - console.log(readString(wasm, ptr, len)); + console.log(readString8(wasm, ptr, len)); }, diplomat_console_warn_js(ptr, len) { - console.warn(readString(wasm, ptr, len)); + console.warn(readString8(wasm, ptr, len)); }, diplomat_throw_error_js(ptr, len) { - throw new Error(readString(wasm, ptr, len)); + throw new Error(readString8(wasm, ptr, len)); } } } diff --git a/tool/templates/dart/slice.dart.jinja b/tool/templates/dart/slice.dart.jinja index 6cf779d68..f10b7fdc2 100644 --- a/tool/templates/dart/slice.dart.jinja +++ b/tool/templates/dart/slice.dart.jinja @@ -30,6 +30,8 @@ final class {{slice_ty}} extends ffi.Struct { for (var i = 0; i < _length; i++) { {%- if ffi_type == "ffi2.Utf8" %} if (other._bytes.cast()[i] != _bytes.cast()[i]) { + {%- else if ffi_type == "ffi2.Utf16" %} + if (other._bytes.cast()[i] != _bytes.cast()[i]) { {%- else %} if (other._bytes[i] != _bytes[i]) { {%- endif %}