diff --git a/compiler/bin-wasm_of_ocaml/compile.ml b/compiler/bin-wasm_of_ocaml/compile.ml index 314254adc..80a03443f 100644 --- a/compiler/bin-wasm_of_ocaml/compile.ml +++ b/compiler/bin-wasm_of_ocaml/compile.ml @@ -240,7 +240,7 @@ let add_source_map sourcemap_don't_inline_content z opt_source_map_file = Zip.add_file z ~name:"source_map.map" ~file; if not sourcemap_don't_inline_content then - let sm = Wa_source_map.load file in + let sm = Source_map.of_file file in Wa_source_map.iter_sources sm (fun i j file -> if Sys.file_exists file && not (Sys.is_directory file) then diff --git a/compiler/lib/source_map.ml b/compiler/lib/source_map.ml index 64576d89d..e2733a521 100644 --- a/compiler/lib/source_map.ml +++ b/compiler/lib/source_map.ml @@ -71,6 +71,8 @@ module Mappings = struct let empty = Uninterpreted "" + let is_empty (Uninterpreted s) = String.equal s "" + let of_string_unsafe : string -> t = fun s -> Uninterpreted s let to_string : t -> string = fun (Uninterpreted s) -> s @@ -289,8 +291,8 @@ let rewrite_path path = let invalid () = invalid_arg "Source_map.of_json" -let string_of_stringlit (`Stringlit s) = - match Yojson.Safe.from_string s with +let string_of_stringlit ?tmp_buf (`Stringlit s) = + match Yojson.Safe.from_string ?buf:tmp_buf s with | `String s -> s | _ -> invalid () @@ -507,11 +509,13 @@ module Standard = struct t.sources))) ) ]) - let of_json (json : Yojson.Raw.t) = + let of_json ?tmp_buf (json : Yojson.Raw.t) = match json with | `Assoc (("version", `Intlit version) :: rest) when version_is_valid (int_of_string version) -> - let string name json = Option.map ~f:string_of_stringlit (stringlit name json) in + let string name json = + Option.map ~f:(fun s -> string_of_stringlit ?tmp_buf s) (stringlit name json) + in let file = string "file" rest in let sourceroot = string "sourceRoot" rest in let names = @@ -641,7 +645,7 @@ module Index = struct | _ -> invalid_arg errmsg | exception Not_found -> invalid_arg errmsg - let section_of_json : Yojson.Raw.t -> section = function + let section_of_json ?tmp_buf : Yojson.Raw.t -> section = function | `Assoc json -> let offset = match List.assoc "offset" json with @@ -671,7 +675,7 @@ module Index = struct "Source_map.Index.of_json: URLs in index maps are not currently supported" | exception Not_found -> ()); let map = - try Standard.of_json (List.assoc "map" json) with + try Standard.of_json ?tmp_buf (List.assoc "map" json) with | Not_found -> invalid_arg "Source_map.Index.of_json: field 'map' absent" | Invalid_argument _ -> invalid_arg "Source_map.Index.of_json: invalid sub-map object" @@ -679,14 +683,14 @@ module Index = struct { offset; map } | _ -> invalid_arg "Source_map.Index.of_json: section of unexpected type" - let of_json = function + let of_json ?tmp_buf = function | `Assoc (("version", `Intlit version) :: fields) when version_is_valid (int_of_string version) -> ( let string name json = Option.map ~f:string_of_stringlit (stringlit name json) in let file = string "file" fields in match List.assoc "sections" fields with | `List sections -> - let sections = List.map ~f:section_of_json sections in + let sections = List.map ~f:(section_of_json ?tmp_buf) sections in { version = int_of_string version; file; sections } | _ -> invalid_arg "Source_map.Index.of_json: `sections` is not an array" | exception Not_found -> @@ -721,16 +725,16 @@ type t = | Standard of Standard.t | Index of Index.t -let of_json = function +let of_json ?tmp_buf = function | `Assoc fields as json -> ( match List.assoc "sections" fields with - | _ -> Index (Index.of_json json) - | exception Not_found -> Standard (Standard.of_json json)) + | _ -> Index (Index.of_json ?tmp_buf json) + | exception Not_found -> Standard (Standard.of_json ?tmp_buf json)) | _ -> invalid_arg "Source_map.of_json: map is not an object" -let of_string s = of_json (Yojson.Raw.from_string s) +let of_string ?tmp_buf s = of_json ?tmp_buf (Yojson.Raw.from_string ?buf:tmp_buf s) -let of_file f = of_json (Yojson.Raw.from_file f) +let of_file ?tmp_buf f = of_json ?tmp_buf (Yojson.Raw.from_file ?buf:tmp_buf f) let to_string = function | Standard m -> Standard.to_string m diff --git a/compiler/lib/source_map.mli b/compiler/lib/source_map.mli index 64954d0fd..f8d01c207 100644 --- a/compiler/lib/source_map.mli +++ b/compiler/lib/source_map.mli @@ -21,6 +21,8 @@ module Source_content : sig type t val create : string -> t + + val of_stringlit : [ `Stringlit of string ] -> t end type map = @@ -60,6 +62,9 @@ module Mappings : sig val empty : t (** The empty mapping. *) + val is_empty : t -> bool + (** Test whether the mapping is empty. *) + val of_string_unsafe : string -> t (** [of_string_unsafe] does not perform any validation of its argument, unlike {!val:decode}. It is guaranteed that @@ -134,9 +139,9 @@ val to_string : t -> string val to_file : t -> string -> unit -val of_string : string -> t +val of_string : ?tmp_buf:Buffer.t -> string -> t -val of_file : string -> t +val of_file : ?tmp_buf:Buffer.t -> string -> t val invariant : t -> unit diff --git a/compiler/lib/vlq64.ml b/compiler/lib/vlq64.ml index 22f71ce5f..4f38940f5 100644 --- a/compiler/lib/vlq64.ml +++ b/compiler/lib/vlq64.ml @@ -22,7 +22,7 @@ open! Stdlib let alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=" let code_rev = - let a = Array.make 255 (-1) in + let a = Array.make 256 (-1) in for i = 0 to String.length alphabet - 1 do a.(Char.code alphabet.[i]) <- i done; @@ -99,3 +99,21 @@ let decode_l s ~pos ~len = aux i (d :: acc) len in aux pos [] len + +type input = + { string : string + ; mutable pos : int + ; len : int + } + +let rec decode' src s pos offset i = + let digit = Array.unsafe_get code_rev (Char.code s.[pos]) in + if digit = -1 then invalid_arg "Vql64.decode'"; + let i = i + ((digit land vlq_base_mask) lsl offset) in + if digit >= vlq_continuation_bit + then decode' src s (pos + 1) (offset + vlq_base_shift) i + else ( + src.pos <- pos + 1; + i) + +let decode src = fromVLQSigned (decode' src src.string src.pos 0 0) diff --git a/compiler/lib/vlq64.mli b/compiler/lib/vlq64.mli index c3accf521..fb4d19f1f 100644 --- a/compiler/lib/vlq64.mli +++ b/compiler/lib/vlq64.mli @@ -19,6 +19,16 @@ val in_alphabet : char -> bool +type input = + { string : string + ; mutable pos : int + ; len : int + } + +val encode : Buffer.t -> int -> unit + val encode_l : Buffer.t -> int list -> unit +val decode : input -> int + val decode_l : string -> pos:int -> len:int -> int list diff --git a/compiler/lib/wasm/wa_link.ml b/compiler/lib/wasm/wa_link.ml index ef9526ab1..b5ed05130 100644 --- a/compiler/lib/wasm/wa_link.ml +++ b/compiler/lib/wasm/wa_link.ml @@ -575,7 +575,7 @@ let source_name i j file = let extract_source_map ~dir ~name z = if Zip.has_entry z ~name:"source_map.map" then ( - let sm = Wa_source_map.parse (Zip.read_entry z ~name:"source_map.map") in + let sm = Source_map.of_string (Zip.read_entry z ~name:"source_map.map") in let sm = let rewrite_path path = if Filename.is_relative path @@ -590,7 +590,7 @@ let extract_source_map ~dir ~name z = if Zip.has_entry z ~name then Some (Zip.read_entry z ~name) else None) in let map_name = name ^ ".wasm.map" in - Wa_source_map.write (Filename.concat dir map_name) sm; + Source_map.to_file sm (Filename.concat dir map_name); Wasm_binary.append_source_map_section ~file:(Filename.concat dir (name ^ ".wasm")) ~url:map_name) @@ -860,7 +860,7 @@ let rec get_source_map_files files src_index = if Zip.has_entry z ~name:"source_map.map" then let data = Zip.read_entry z ~name:"source_map.map" in - let sm = Wa_source_map.parse data in + let sm = Source_map.of_string data in if not (Wa_source_map.is_empty sm) then ( let l = ref [] in @@ -879,7 +879,7 @@ let add_source_map files z opt_source_map_file = Option.iter ~f:(fun file -> Zip.add_file z ~name:"source_map.map" ~file; - let sm = Wa_source_map.load file in + let sm = Source_map.of_file file in let files = Array.of_list files in let src_index = ref 0 in let st = ref None in diff --git a/compiler/lib/wasm/wa_source_map.ml b/compiler/lib/wasm/wa_source_map.ml index 5cbb51f74..06f9b6cad 100644 --- a/compiler/lib/wasm/wa_source_map.ml +++ b/compiler/lib/wasm/wa_source_map.ml @@ -8,84 +8,35 @@ type resize_data = type t = Yojson.Raw.t -type u = - { mappings : string +type input = Vlq64.input = + { string : string ; mutable pos : int + ; len : int } -module Vlq = struct - let code = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=" - - let code_rev = - let a = Array.make 255 (-1) in - for i = 0 to String.length code - 1 do - a.(Char.code code.[i]) <- i - done; - a - - let vlq_base_shift = 5 - - let vlq_base = 1 lsl vlq_base_shift - - let vlq_base_mask = vlq_base - 1 - - let vlq_continuation_bit = vlq_base - - let rec decode' src s pos offset i = - let digit = Array.unsafe_get code_rev (Char.code s.[pos]) in - if digit = -1 then invalid_arg "Vql64.decode'"; - let i = i + ((digit land vlq_base_mask) lsl offset) in - if digit >= vlq_continuation_bit - then decode' src s (pos + 1) (offset + vlq_base_shift) i - else ( - src.pos <- pos + 1; - i) - - let fromVLQSigned v = - let is_neg = v land 1 = 1 in - let shift = v lsr 1 in - if is_neg then -shift else shift - - let toVLQSigned v = if v < 0 then (-v lsl 1) + 1 else v lsl 1 - - let decode src = fromVLQSigned (decode' src src.mappings src.pos 0 0) - - let rec encode' buf i = - let digit = i land vlq_base_mask in - let i = i lsr vlq_base_shift in - if i = 0 - then Buffer.add_char buf (String.unsafe_get code digit) - else ( - Buffer.add_char buf (String.unsafe_get code (digit lor vlq_continuation_bit)); - encode' buf i) - - let encode buf i = encode' buf (toVLQSigned i) -end - let rec next' src mappings pos = + pos < src.len + && match mappings.[pos] with - | '"' -> - src.pos <- pos + 1; - false | ',' -> src.pos <- pos + 1; true | _ -> next' src mappings (pos + 1) -let next src = next' src src.mappings src.pos +let next src = next' src src.string src.pos let flush buf src start pos = - if start < pos then Buffer.add_substring buf src.mappings start (pos - start) + if start < pos then Buffer.add_substring buf src.string start (pos - start) let rec resize_rec buf start src resize_data i col0 delta0 col = let pos = src.pos in - let delta = Vlq.decode src in + let delta = Vlq64.decode src in let col = col + delta in if col < col0 then if next src then resize_rec buf start src resize_data i col0 delta0 col - else flush buf src start (String.length src.mappings) + else flush buf src start (String.length src.string) else let delta = delta + delta0 in adjust buf start src resize_data i col delta pos @@ -99,157 +50,125 @@ and adjust buf start src (resize_data : resize_data) i col delta pos = if col < col0 then ( flush buf src start pos; - Vlq.encode buf delta; + Vlq64.encode buf delta; let start = src.pos in if next src then resize_rec buf start src resize_data (i + 1) col0 delta0 col - else flush buf src start (String.length src.mappings)) + else flush buf src start (String.length src.string)) else let delta = delta + delta0 in adjust buf start src resize_data (i + 1) col delta pos else ( flush buf src start pos; - Vlq.encode buf delta; + Vlq64.encode buf delta; let start = src.pos in - flush buf src start (String.length src.mappings)) + flush buf src start (String.length src.string)) let resize_mappings (resize_data : resize_data) mappings = - if String.equal mappings "\"\"" || resize_data.i = 0 + if String.equal mappings "" || resize_data.i = 0 then mappings else let col0 = resize_data.pos.(0) in let delta0 = resize_data.delta.(0) in let buf = Buffer.create (String.length mappings) in - resize_rec buf 0 { mappings; pos = 1 } resize_data 1 col0 delta0 0; + resize_rec + buf + 0 + { Vlq64.string = mappings; pos = 0; len = String.length mappings } + resize_data + 1 + col0 + delta0 + 0; Buffer.contents buf -let to_raw_string v = - match v with - | `Stringlit s -> s - | _ -> assert false - -let replace_member assoc m v = - `Assoc ((m, v) :: List.remove_assoc m (Yojson.Raw.Util.to_assoc assoc)) - let resize resize_data sm = - let open Yojson.Raw.Util in - let mappings = to_raw_string (member "mappings" sm) in - let mappings = resize_mappings resize_data mappings in - replace_member sm "mappings" (`Stringlit mappings) + match sm with + | Source_map.Index _ -> assert false + | Standard sm -> + let mappings = Source_map.Mappings.to_string sm.mappings in + let mappings = resize_mappings resize_data mappings in + Source_map.Standard + { sm with mappings = Source_map.Mappings.of_string_unsafe mappings } let is_empty sm = - let open Yojson.Raw.Util in - match member "mappings" sm with - | `Stringlit "\"\"" -> true - | _ -> false + match sm with + | Source_map.Standard { mappings; _ } -> Source_map.Mappings.is_empty mappings + | _ -> assert false let concatenate l = - `Assoc - [ "version", `Intlit "3" - ; ( "sections" - , `List - (List.map - ~f:(fun (ofs, sm) -> - `Assoc - [ ( "offset" - , `Assoc [ "line", `Intlit "0"; "column", `Intlit (string_of_int ofs) ] - ) - ; "map", sm - ]) - l) ) - ] - -let parse ?tmp_buf s = Yojson.Raw.from_string ?buf:tmp_buf s - -let load ?tmp_buf name = parse ?tmp_buf (Fs.read_file name) - -let write name sm = Yojson.Raw.to_file name sm - -let string_from_raw_string s = Yojson.Basic.Util.to_string (Yojson.Basic.from_string s) - -let raw_string_from_string s = Yojson.Basic.to_string (`String s) - -let iter_sources' sm i f = - let open Yojson.Raw.Util in - let l = sm |> member "sources" |> to_option to_list |> Option.value ~default:[] in + Source_map.Index + { version = 3 + ; file = None + ; sections = + List.map + ~f:(fun (ofs, sm) -> + match sm with + | Source_map.Index _ -> assert false + | Standard map -> + { Source_map.Index.offset = { gen_line = 0; gen_column = ofs }; map }) + l + } + +let iter_sources' (sm : Source_map.Standard.t) i f = + let l = sm.sources in let single = List.length l = 1 in - List.iteri - ~f:(fun j nm -> - f i (if single then None else Some j) (string_from_raw_string (to_raw_string nm))) - l + List.iteri ~f:(fun j nm -> f i (if single then None else Some j) nm) l let iter_sources sm f = - let open Yojson.Raw.Util in - match to_option to_list (member "sections" sm) with - | None -> iter_sources' sm None f - | Some l -> - let single_map = List.length l = 1 in + match sm with + | Source_map.Standard sm -> iter_sources' sm None f + | Index { sections; _ } -> + let single_map = List.length sections = 1 in List.iteri ~f:(fun i entry -> - iter_sources' (member "map" entry) (if single_map then None else Some i) f) - l + iter_sources' entry.Source_map.Index.map (if single_map then None else Some i) f) + sections let blackbox_filename = "/builtin/blackbox.ml" let blackbox_contents = "(* generated code *)" -let insert_source_contents' ~rewrite_path sm i f = - let rewrite_path path = - raw_string_from_string (rewrite_path (string_from_raw_string path)) - in - let open Yojson.Raw.Util in - let l = sm |> member "sources" |> to_option to_list |> Option.value ~default:[] in +let insert_source_contents' ~rewrite_path (sm : Source_map.Standard.t) i f = + let l = sm.sources in let single = List.length l = 1 in - let ignored = ref (-1) in let contents = List.mapi ~f:(fun j name -> - let name = string_from_raw_string (to_raw_string name) in if String.equal name blackbox_filename - then ( - ignored := j; - `Stringlit (raw_string_from_string blackbox_contents)) + then Some (Source_map.Source_content.create blackbox_contents) else match f i (if single then None else Some j) name with - | Some c -> `Stringlit c - | None -> `Null) + | Some c -> Some (Source_map.Source_content.of_stringlit (`Stringlit c)) + | None -> None) l in - let sm = replace_member sm "sourcesContent" (`List contents) in + let sm = { sm with sources_content = Some contents } in let sm = - if !ignored >= 0 - then replace_member sm "ignoreList" (`List [ `Intlit (string_of_int !ignored) ]) + if List.mem blackbox_filename ~set:sm.sources + then { sm with ignore_list = [ blackbox_filename ] } else sm in - let sm = - replace_member - sm - "sources" - (match member "sources" sm with - | `Null -> `Null - | `List l -> - `List (List.map ~f:(fun s -> `Stringlit (rewrite_path (to_raw_string s))) l) - | _ -> assert false) - in + let sm = { sm with sources = List.map ~f:rewrite_path sm.sources } in sm let insert_source_contents ~rewrite_path sm f = - let open Yojson.Raw.Util in - match to_option to_list (member "sections" sm) with - | None -> insert_source_contents' ~rewrite_path sm None f - | Some l -> - let single_map = List.length l = 1 in + match sm with + | Source_map.Standard sm -> + Source_map.Standard (insert_source_contents' ~rewrite_path sm None f) + | Index ({ sections; _ } as sm) -> + let single_map = List.length sections = 1 in let sections = List.mapi ~f:(fun i entry -> - replace_member - entry - "map" - (insert_source_contents' - ~rewrite_path - (member "map" entry) - (if single_map then None else Some i) - f)) - l + { entry with + Source_map.Index.map = + insert_source_contents' + ~rewrite_path + entry.Source_map.Index.map + (if single_map then None else Some i) + f + }) + sections in - replace_member sm "sections" (`List sections) + Index { sm with sections } diff --git a/compiler/lib/wasm/wa_source_map.mli b/compiler/lib/wasm/wa_source_map.mli index 6845260ed..65ab1f97f 100644 --- a/compiler/lib/wasm/wa_source_map.mli +++ b/compiler/lib/wasm/wa_source_map.mli @@ -1,12 +1,6 @@ type t -val load : ?tmp_buf:Buffer.t -> string -> t - -val parse : ?tmp_buf:Buffer.t -> string -> t - -val write : string -> t -> unit - -val is_empty : t -> bool +val is_empty : Source_map.t -> bool type resize_data = { mutable i : int @@ -14,17 +8,17 @@ type resize_data = ; mutable delta : int array } -val resize : resize_data -> t -> t +val resize : resize_data -> Source_map.t -> Source_map.t -val concatenate : (int * t) list -> t +val concatenate : (int * Source_map.t) list -> Source_map.t -val iter_sources : t -> (int option -> int option -> string -> unit) -> unit +val iter_sources : Source_map.t -> (int option -> int option -> string -> unit) -> unit val insert_source_contents : rewrite_path:(string -> string) - -> t + -> Source_map.t -> (int option -> int option -> string -> string option) - -> t + -> Source_map.t val blackbox_filename : string diff --git a/compiler/lib/wasm/wa_wasm_link.ml b/compiler/lib/wasm/wa_wasm_link.ml index 8b4c2005c..ac11566f5 100644 --- a/compiler/lib/wasm/wa_wasm_link.ml +++ b/compiler/lib/wasm/wa_wasm_link.ml @@ -1497,7 +1497,7 @@ type t = { module_name : string ; file : string ; contents : Read.t - ; source_map_contents : Wa_source_map.t option + ; source_map_contents : Source_map.t option } type import_status = @@ -1877,8 +1877,8 @@ let f files ~output_file ~opt_output_sourcemap_file = Option.map ~f:(fun src -> match src with - | `File file -> Wa_source_map.load ~tmp_buf file - | `Data data -> Wa_source_map.parse ~tmp_buf data) + | `File file -> Source_map.of_file ~tmp_buf file + | `Data data -> Source_map.of_string ~tmp_buf data) opt_source_map }) (Array.of_list files) @@ -2275,12 +2275,12 @@ let f files ~output_file ~opt_output_sourcemap_file = add_section out_ch ~id:10 code_pieces; Option.iter ~f:(fun file -> - Wa_source_map.write - file + Source_map.to_file (Wa_source_map.concatenate (List.map ~f:(fun (pos, sm) -> pos + code_section_offset, sm) - (List.rev !source_maps)))) + (List.rev !source_maps))) + file) opt_output_sourcemap_file; (* 11: data *)