From ca39c4ed1614fe096a3ab9974b216ad871659d6e Mon Sep 17 00:00:00 2001 From: Hugo Heuzard Date: Thu, 16 Jan 2025 12:17:57 +0100 Subject: [PATCH] Make wasm tests run on windows with CRLF --- .gitattributes | 4 - .github/workflows/build-wasm_of_ocaml.yml | 25 ++++-- CHANGES.md | 1 + compiler/lib-wasm/link.ml | 7 +- dune-project | 1 + runtime/wasm/dune | 31 +++----- runtime/wasm/runtime.js | 1 + runtime/wasm/sys.wat | 17 ++-- runtime/wasm/unix.wat | 7 ++ tools/ci_setup.ml | 96 ++++++++++++----------- tools/dune | 6 ++ wasm_of_ocaml-compiler.opam | 1 + 12 files changed, 112 insertions(+), 85 deletions(-) diff --git a/.gitattributes b/.gitattributes index b201a765ab..c7b30a1a32 100644 --- a/.gitattributes +++ b/.gitattributes @@ -3,7 +3,3 @@ *.ml linguist-language=OCaml *.mli linguist-language=OCaml - -# We are pinning wasm_of_ocaml using this file in the CI. This would -# fail on Windows otherwise. -VERSION -text diff --git a/.github/workflows/build-wasm_of_ocaml.yml b/.github/workflows/build-wasm_of_ocaml.yml index 2f1d394e01..3bca5f44c4 100644 --- a/.github/workflows/build-wasm_of_ocaml.yml +++ b/.github/workflows/build-wasm_of_ocaml.yml @@ -33,6 +33,11 @@ jobs: separate_compilation: true jane_street_tests: false all_jane_street_tests: false + - os: windows-latest + ocaml-compiler: "5.2" + separate_compilation: true + jane_street_tests: true + all_jane_street_tests: false - os: ubuntu-latest ocaml-compiler: "5.2" separate_compilation: true @@ -86,16 +91,25 @@ jobs: # It's faster to use a cached version run: opam install --fake binaryen-bin + # Work-around a race between reinstalling mingw-w64-shims + # (because of conf-pkg-config optional dep) and installing other + # packages that implicitly depend on mingw-w64-shims. + - run: opam install conf-pkg-config conf-mingw-w64-gcc-i686 + if: runner.os == 'Windows' + + - name: Pin dune + run: opam pin add -n dune https://github.com/hhugo/dune.git#shorter-path-jsoo + - name: Pin wasm_of_ocaml working-directory: ./wasm_of_ocaml - run: opam pin -n --with-version `< VERSION` . + run: opam pin . -n --with-version dev - name: Checkout Jane Street packages if: matrix.jane_street_tests run: | opam repo add js janestreet/opam-repository opam install opam-format ocamlfind dune graphics cmdliner - opam exec -- ocaml wasm_of_ocaml/tools/ci_setup.ml + opam exec -- dune exec --root wasm_of_ocaml tools/ci_setup.exe - name: Pin Jane Street packages if: ${{ ! matrix.jane_street_tests }} @@ -108,7 +122,8 @@ jobs: - name: Install wasm_of_ocaml and its test dependencies working-directory: ./wasm_of_ocaml run: | - opam install `basename -s .opam *.opam` num cohttp-lwt-unix ppx_expect cstruct + opam install . + opam install num cohttp-lwt-unix ppx_expect cstruct uucp - name: Run tests if: ${{ matrix.separate_compilation }} @@ -121,7 +136,7 @@ jobs: run: opam exec -- dune build @runtest-wasm --profile with-effects - name: Run Base tests - if: matrix.all_jane_street_tests + if: ${{ matrix.all_jane_street_tests || matrix.os == 'windows-latest' }} working-directory: ./janestreet/lib/base run: opam exec -- dune runtest @@ -136,7 +151,7 @@ jobs: run: opam exec -- dune runtest - name: Run Bignum tests - if: matrix.all_jane_street_tests + if: ${{ matrix.all_jane_street_tests || matrix.os == 'windows-latest' }} working-directory: ./janestreet/lib/bignum run: opam exec -- dune runtest diff --git a/CHANGES.md b/CHANGES.md index 8d43b3ef3b..9ac2b2c774 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -12,6 +12,7 @@ * Compiler/Runtime: Make resuming a continuation more efficient in js (#1765) * Lib: Add other textMetrics property (#1784) * Misc: move tests to OCaml 5.3 +* Test: run wasm tests on windows ## Bug fixes * Fix small bug in global data flow analysis (#1768) diff --git a/compiler/lib-wasm/link.ml b/compiler/lib-wasm/link.ml index d043d0833e..b5822d31ea 100644 --- a/compiler/lib-wasm/link.ml +++ b/compiler/lib-wasm/link.ml @@ -688,12 +688,7 @@ let remove_directory path = with Sys_error _ -> () let gen_dir dir f = - let d_tmp = - Filename.temp_file_name - ~temp_dir:(Filename.dirname dir) - (Filename.basename dir) - ".tmp" - in + let d_tmp = Filename.temp_file_name ~temp_dir:(Filename.dirname dir) "assets" ".tmp" in try let res = f d_tmp in remove_directory dir; diff --git a/dune-project b/dune-project index 1cb47d81d2..9a9aa2c405 100644 --- a/dune-project +++ b/dune-project @@ -146,6 +146,7 @@ (ppxlib (>= 0.15.0)) (re :with-test) (cmdliner (>= 1.1.0)) + (opam-format :with-test) (sedlex (>= 2.3)) menhir menhirLib diff --git a/runtime/wasm/dune b/runtime/wasm/dune index 90422c0f9d..3bbc0a5fc6 100644 --- a/runtime/wasm/dune +++ b/runtime/wasm/dune @@ -54,24 +54,19 @@ args (glob_files *.wat)) (action - (progn - (bash - "which wasm-merge > /dev/null || (echo 'Error: Binaryen tools not found in the PATH'; false)") - (bash - "wasm-merge --version | grep -q 'version \\(11[89]\\|1[2-9][0-9]\\)' || (echo 'Error: Binaryen version 118 or greater is currently required'; false)") - (run - wasm-merge - -g - --enable-gc - --enable-exception-handling - --enable-reference-types - --enable-tail-call - --enable-strings - --enable-multivalue - --enable-bulk-memory - %{read-lines:args} - -o - %{target})))) + (run + wasm-merge + -g + --enable-gc + --enable-exception-handling + --enable-reference-types + --enable-tail-call + --enable-strings + --enable-multivalue + --enable-bulk-memory + %{read-lines:args} + -o + %{target}))) (rule (target args) diff --git a/runtime/wasm/runtime.js b/runtime/wasm/runtime.js index 00adc8f4a6..f41982a08e 100644 --- a/runtime/wasm/runtime.js +++ b/runtime/wasm/runtime.js @@ -397,6 +397,7 @@ channel_list, exit: (n) => isNode && process.exit(n), argv: () => (isNode ? process.argv.slice(1) : ["a.out"]), + on_windows: () => isNode && process.platform === "win32", getenv: (n) => (isNode ? process.env[n] : null), system: (c) => { var res = require("node:child_process").spawnSync(c, { diff --git a/runtime/wasm/sys.wat b/runtime/wasm/sys.wat index b397c89a82..dcb7c0b813 100644 --- a/runtime/wasm/sys.wat +++ b/runtime/wasm/sys.wat @@ -36,6 +36,7 @@ (func $caml_raise_sys_error (param (ref eq)))) (import "fail" "caml_raise_not_found" (func $caml_raise_not_found)) (import "bindings" "argv" (func $argv (result (ref extern)))) + (import "bindings" "on_windows" (func $on_windows (result i32))) (import "bindings" "system" (func $system (param anyref) (result (ref eq)))) (import "bindings" "getenv" (func $getenv (param anyref) (result anyref))) (import "bindings" "time" (func $time (result f64))) @@ -135,27 +136,27 @@ (func (export "caml_sys_const_ostype_unix") (param (ref eq)) (result (ref eq)) - ;; ZZZ - (ref.i31 (i32.const 1))) + (ref.i31 (i32.eqz (call $on_windows)))) (func (export "caml_sys_const_ostype_win32") (param (ref eq)) (result (ref eq)) - ;; ZZZ - (ref.i31 (i32.const 0))) + (ref.i31 (call $on_windows))) (func (export "caml_sys_const_ostype_cygwin") (param (ref eq)) (result (ref eq)) - ;; ZZZ (ref.i31 (i32.const 0))) (data $Unix "Unix") + (data $Win32 "Win32") (func (export "caml_sys_get_config") (param (ref eq)) (result (ref eq)) - ;; ZZZ - ;; (call $log_js (string.const "caml_sys_get_config")) (array.new_fixed $block 4 (ref.i31 (i32.const 0)) - (array.new_data $string $Unix (i32.const 0) (i32.const 4)) + (if (result (ref eq)) (call $on_windows) + (then + (array.new_data $string $Win32 (i32.const 0) (i32.const 5))) + (else + (array.new_data $string $Unix (i32.const 0) (i32.const 4)))) (ref.i31 (i32.const 32)) (ref.i31 (i32.const 0)))) diff --git a/runtime/wasm/unix.wat b/runtime/wasm/unix.wat index 797a3b3f3f..4740b129ee 100644 --- a/runtime/wasm/unix.wat +++ b/runtime/wasm/unix.wat @@ -99,4 +99,11 @@ (func $unix_inet_addr_of_string (export "unix_inet_addr_of_string") (param (ref eq)) (result (ref eq)) (ref.i31 (i32.const 0))) + (export "caml_unix_filedescr_of_fd" (func $unix_filedescr_of_fd)) + (func $unix_filedescr_of_fd (export "unix_filedescr_of_fd") + (param (ref eq)) (result (ref eq)) + (local.get 0)) + (func $unix_cleanup (export "caml_unix_cleanup") + (param (ref eq)) (result (ref eq)) + (ref.i31 (i32.const 0))) ) diff --git a/tools/ci_setup.ml b/tools/ci_setup.ml index e35336f6ad..185d5a5126 100644 --- a/tools/ci_setup.ml +++ b/tools/ci_setup.ml @@ -1,20 +1,19 @@ -#use "topfind" - -#require "opam-format" - -#require "unix" - -#require "str" - module StringSet = Set.Make (String) (****) -let repo = "janestreet/opam-repository/packages" +let jane_root, wasmoo_root = + match Sys.argv with + | [| _; jane_root; wasmoo_root |] -> jane_root, wasmoo_root + | _ -> "janestreet", "wasm_of_ocaml" + +let repo = Filename.concat jane_root "opam-repository/packages" let roots = [ "bonsai_web_components"; "string_dict"; "ppx_html" ] -let omitted_others = StringSet.of_list [ "cohttp-async"; "cohttp"; "uri"; "uri-sexp" ] +let omitted_others = + StringSet.of_list + [ "cohttp-async"; "cohttp"; "uri"; "uri-sexp"; "cstruct"; "uucp"; "odoc-parser" ] let omitted_js = StringSet.of_list [ "sexplib0" ] @@ -137,7 +136,7 @@ let read_opam_file filename = ~pos:{ filename; start = 0, 0; stop = 0, 0 } (OpamParser.FullPos.file (Filename.concat (Filename.concat repo filename) "opam")) -let dependencies (_, { OpamFile.OPAM.depends }) = +let dependencies (_, { OpamFile.OPAM.depends; _ }) = let open OpamFormula in depends |> map (fun (nm, _) -> Atom (nm, None)) @@ -165,21 +164,18 @@ let rec traverse visited p = let is_forked p = StringSet.mem p forked_packages -let exec_async ~delay cmd = - let p = - Unix.open_process_out (Printf.sprintf "sleep %f; %s" (float delay /. 10.) cmd) - in +let exec_async cmd = + let p = Unix.open_process_out cmd in fun () -> ignore (Unix.close_process_out p) let ( let* ) (f : unit -> 'a) (g : 'a -> unit -> 'b) : unit -> 'b = fun () -> g (f ()) () let sync_exec f l = - let l = List.mapi f l in + let l = List.map f l in List.iter (fun f -> f ()) l -let pin delay nm = +let pin nm = exec_async - ~delay (Printf.sprintf "opam pin add -n %s https://github.com/ocaml-wasm/%s.git#wasm-v0.18" nm @@ -191,45 +187,42 @@ let install_others others = let others = StringSet.elements (StringSet.diff others omitted_others) in ignore (Sys.command ("opam install -y " ^ String.concat " " others)) -let clone delay ?branch ?(depth = 1) nm src = +let clone ?branch ?(depth = 1) nm src = exec_async - ~delay (Printf.sprintf - "git clone -q --depth %d %s%s janestreet/lib/%s" + "git clone -q --depth %d %s%s %s/lib/%s" depth (match branch with | None -> "" | Some b -> Printf.sprintf "-b %s " b) src + jane_root nm) -let clone' delay ?branch ?commit nm src = +let clone' ?branch ?commit nm src = match commit with - | None -> clone delay ?branch nm src + | None -> clone ?branch nm src | Some commit -> - let* () = clone delay ?branch ~depth:100 nm src in + let* () = clone ?branch ~depth:100 nm src in exec_async - ~delay:0 - (Printf.sprintf "cd janestreet/lib/%s && git checkout -b wasm %s" nm commit) + (Printf.sprintf "cd %s/lib/%s && git checkout -b wasm %s" jane_root nm commit) let () = let write f contents = - Out_channel.( - with_open_bin (Filename.concat "janestreet" f) - @@ fun ch -> output_string ch contents) + Out_channel.(with_open_bin f @@ fun ch -> output_string ch contents) in let copy f f' = - let contents = - In_channel.(with_open_bin (Filename.concat "wasm_of_ocaml" f) @@ input_all) - in - Out_channel.( - with_open_bin (Filename.concat "janestreet" f') - @@ fun ch -> output_string ch contents) + let contents = In_channel.(with_open_bin f @@ input_all) in + Out_channel.(with_open_bin f' @@ fun ch -> output_string ch contents) in - write "dune-workspace" dune_workspace; - Unix.mkdir "janestreet/node_wrapper" 0o755; - List.iter (fun (f, contents) -> write f contents) node_wrapper; - copy "tools/node_wrapper.ml" "node_wrapper/node_wrapper.ml" + write (Filename.concat jane_root "dune-workspace") dune_workspace; + Unix.mkdir (Filename.concat jane_root "node_wrapper") 0o755; + List.iter + (fun (f, contents) -> write (Filename.concat jane_root f) contents) + node_wrapper; + copy + (Filename.concat wasmoo_root "tools/node_wrapper.ml") + (Filename.concat jane_root "node_wrapper/node_wrapper.ml") let () = let js, others = @@ -238,9 +231,9 @@ let () = in pin_packages (); install_others others; - sync_exec (fun i () -> clone i "ocaml-uri" "https://github.com/mirage/ocaml-uri") [ () ]; + sync_exec (fun () -> clone "ocaml-uri" "https://github.com/mirage/ocaml-uri") [ () ]; sync_exec - (fun i nm -> + (fun nm -> let branch = if is_forked nm then Some "wasm-v0.18" else None in let commit = if is_forked nm @@ -253,7 +246,6 @@ let () = String.sub tar_file 0 (String.index tar_file '.')) in clone' - i ?branch ?commit nm @@ -266,9 +258,25 @@ let () = let () = List.iter (fun (dir, patch) -> + let p = if Sys.win32 then "patch --binary" else "patch" in let ch = - Unix.open_process_out (Printf.sprintf "cd janestreet/lib/%s && patch -p 1" dir) + Unix.open_process_out + (Printf.sprintf "cd %s/lib/%s && %s -p 1 --" jane_root dir p) + in + let patch = + if Sys.win32 + then String.concat "\r\n" (String.split_on_char '\n' patch) + else patch in output_string ch patch; - ignore (Unix.close_process_out ch)) + match Unix.close_process_out ch with + | WEXITED 0 -> () + | e -> + let name, i = + match e with + | WEXITED n -> "exit", n + | WSIGNALED n -> "signal", n + | WSTOPPED n -> "stop", n + in + failwith (Printf.sprintf "%s %d while patching %s" name i dir)) patches diff --git a/tools/dune b/tools/dune index 7532203a08..f5589b29cc 100644 --- a/tools/dune +++ b/tools/dune @@ -1,3 +1,9 @@ (executable (name node_wrapper) + (modules node_wrapper) (libraries unix)) + +(executable + (name ci_setup) + (modules ci_setup) + (libraries opam-format unix)) diff --git a/wasm_of_ocaml-compiler.opam b/wasm_of_ocaml-compiler.opam index b145d310e0..bcb3f21403 100644 --- a/wasm_of_ocaml-compiler.opam +++ b/wasm_of_ocaml-compiler.opam @@ -20,6 +20,7 @@ depends: [ "ppxlib" {>= "0.15.0"} "re" {with-test} "cmdliner" {>= "1.1.0"} + "opam-format" {with-test} "sedlex" {>= "2.3"} "menhir" "menhirLib"