diff --git a/doc/changes/10644.md b/doc/changes/10644.md new file mode 100644 index 00000000000..6b131abb95d --- /dev/null +++ b/doc/changes/10644.md @@ -0,0 +1,4 @@ +- Add support for the -H flag (introduced in OCaml compiler 5.2) in dune + (requires lang versions 3.17). This adaptation gives + the correct semantics for `(implicit_transitive_deps false)`. + (#10644, fixes #9333, ocsigen/tyxml#274, #2733, #4963, @MA0100) \ No newline at end of file diff --git a/src/dune_rules/compilation_context.ml b/src/dune_rules/compilation_context.ml index 3fdfad25023..8bb1be96cd0 100644 --- a/src/dune_rules/compilation_context.ml +++ b/src/dune_rules/compilation_context.ml @@ -3,28 +3,34 @@ open Import module Includes = struct type t = Command.Args.without_targets Command.Args.t Lib_mode.Cm_kind.Map.t - let make ~project ~opaque ~requires : _ Lib_mode.Cm_kind.Map.t = + let make ~project ~opaque ~direct_requires ~hidden_requires : _ Lib_mode.Cm_kind.Map.t = (* TODO : some of the requires can filtered out using [ocamldep] info *) let open Resolve.Memo.O in - let iflags libs mode = Lib_flags.L.include_flags ~project libs mode in + let iflags direct_libs hidden_libs mode = + Lib_flags.L.include_flags ~project ~direct_libs ~hidden_libs mode + in let make_includes_args ~mode groups = Command.Args.memo (Resolve.Memo.args - (let+ libs = requires in + (let+ direct_libs = direct_requires + and+ hidden_libs = hidden_requires in Command.Args.S - [ iflags libs mode; Hidden_deps (Lib_file_deps.deps libs ~groups) ])) + [ iflags direct_libs hidden_libs mode + ; Hidden_deps (Lib_file_deps.deps (direct_libs @ hidden_libs) ~groups) + ])) in let cmi_includes = make_includes_args ~mode:(Ocaml Byte) [ Ocaml Cmi ] in let cmx_includes = Command.Args.memo (Resolve.Memo.args - (let+ libs = requires in + (let+ direct_libs = direct_requires + and+ hidden_libs = hidden_requires in Command.Args.S - [ iflags libs (Ocaml Native) + [ iflags direct_libs hidden_libs (Ocaml Native) ; Hidden_deps (if opaque then - List.map libs ~f:(fun lib -> + List.map (direct_libs @ hidden_libs) ~f:(fun lib -> ( lib , if Lib.is_local lib then [ Lib_file_deps.Group.Ocaml Cmi ] @@ -32,7 +38,7 @@ module Includes = struct |> Lib_file_deps.deps_with_exts else Lib_file_deps.deps - libs + (direct_libs @ hidden_libs) ~groups:[ Lib_file_deps.Group.Ocaml Cmi; Ocaml Cmx ]) ])) in @@ -74,6 +80,7 @@ type t = ; modules : modules ; flags : Ocaml_flags.t ; requires_compile : Lib.t list Resolve.Memo.t + ; requires_hidden : Lib.t list Resolve.Memo.t ; requires_link : Lib.t list Resolve.t Memo.Lazy.t ; includes : Includes.t ; preprocessing : Pp_spec.t @@ -99,6 +106,7 @@ let obj_dir t = t.obj_dir let modules t = t.modules.modules let flags t = t.flags let requires_compile t = t.requires_compile +let requires_hidden t = t.requires_hidden let requires_link t = Memo.Lazy.force t.requires_link let includes t = t.includes let preprocessing t = t.preprocessing @@ -139,10 +147,24 @@ let create = let open Memo.O in let project = Scope.project scope in - let requires_compile = + let context = Super_context.context super_context in + let* ocaml = Context.ocaml context in + let direct_requires, hidden_requires = if Dune_project.implicit_transitive_deps project - then Memo.Lazy.force requires_link - else requires_compile + then Memo.Lazy.force requires_link, Resolve.Memo.return [] + else if Version.supports_hidden_includes ocaml.version + && Dune_project.dune_version project >= (3, 17) + then ( + let requires_hidden = + let open Resolve.Memo.O in + let+ requires_compile = requires_compile + and+ requires_link = Memo.Lazy.force requires_link in + let requires_table = Table.create (module Lib) 5 in + List.iter ~f:(fun lib -> Table.set requires_table lib ()) requires_compile; + List.filter requires_link ~f:(fun l -> not (Table.mem requires_table l)) + in + requires_compile, requires_hidden) + else requires_compile, Resolve.Memo.return [] in let sandbox = Sandbox_config.no_special_requirements in let modes = @@ -153,8 +175,6 @@ let create in Option.value ~default modes |> Lib_mode.Map.map ~f:Option.is_some in - let context = Super_context.context super_context in - let* ocaml = Context.ocaml context in let opaque = let profile = Context.profile context in eval_opaque ocaml profile opaque @@ -180,9 +200,10 @@ let create ; obj_dir ; modules = { modules; dep_graphs } ; flags - ; requires_compile + ; requires_compile = direct_requires + ; requires_hidden = hidden_requires ; requires_link - ; includes = Includes.make ~project ~opaque ~requires:requires_compile + ; includes = Includes.make ~project ~opaque ~direct_requires ~hidden_requires ; preprocessing ; opaque ; stdlib @@ -263,8 +284,16 @@ let for_module_generated_at_link_time cctx ~requires ~module_ = their implementation must also be compiled with -opaque *) Ocaml.Version.supports_opaque_for_mli cctx.ocaml.version in + let direct_requires = requires in + let hidden_requires = Resolve.Memo.return [] in let modules = singleton_modules module_ in - let includes = Includes.make ~project:(Scope.project cctx.scope) ~opaque ~requires in + let includes = + Includes.make + ~project:(Scope.project cctx.scope) + ~opaque + ~direct_requires + ~hidden_requires + in { cctx with opaque ; flags = Ocaml_flags.empty diff --git a/src/dune_rules/compilation_context.mli b/src/dune_rules/compilation_context.mli index 643466e623a..9924ea719a6 100644 --- a/src/dune_rules/compilation_context.mli +++ b/src/dune_rules/compilation_context.mli @@ -55,6 +55,7 @@ val obj_dir : t -> Path.Build.t Obj_dir.t val modules : t -> Modules.With_vlib.t val flags : t -> Ocaml_flags.t val requires_link : t -> Lib.t list Resolve.Memo.t +val requires_hidden : t -> Lib.t list Resolve.Memo.t val requires_compile : t -> Lib.t list Resolve.Memo.t val includes : t -> Command.Args.without_targets Command.Args.t Lib_mode.Cm_kind.Map.t val preprocessing : t -> Pp_spec.t diff --git a/src/dune_rules/lib_flags.ml b/src/dune_rules/lib_flags.ml index 3eab4dd2fe8..d8e7dabc509 100644 --- a/src/dune_rules/lib_flags.ml +++ b/src/dune_rules/lib_flags.ml @@ -85,13 +85,16 @@ let link_deps sctx t mode = module L = struct type nonrec t = Lib.t list - let to_iflags dirs = + let to_flags flag dirs = Command.Args.S (Path.Set.fold dirs ~init:[] ~f:(fun dir acc -> - Command.Args.Path dir :: A "-I" :: acc) + Command.Args.Path dir :: A flag :: acc) |> List.rev) ;; + let to_iflags dir = to_flags "-I" dir + let to_hflags dir = to_flags "-H" dir + let remove_stdlib dirs libs = match libs with | [] -> dirs @@ -155,8 +158,13 @@ module L = struct remove_stdlib dirs ts ;; - let include_flags ?project ts mode = - to_iflags (include_paths ?project ts { lib_mode = mode; melange_emit = false }) + let include_flags ?project ~direct_libs ~hidden_libs mode = + let include_paths ts = + include_paths ?project ts { lib_mode = mode; melange_emit = false } + in + let hidden_includes = to_hflags (include_paths hidden_libs) in + let direct_includes = to_iflags (include_paths direct_libs) in + Command.Args.S [ direct_includes; hidden_includes ] ;; let melange_emission_include_flags ?project ts = diff --git a/src/dune_rules/lib_flags.mli b/src/dune_rules/lib_flags.mli index 1faa6dee7b7..8a65f55a084 100644 --- a/src/dune_rules/lib_flags.mli +++ b/src/dune_rules/lib_flags.mli @@ -24,7 +24,14 @@ module L : sig val to_iflags : Path.Set.t -> _ Command.Args.t val include_paths : ?project:Dune_project.t -> t -> Lib_mode.t -> Path.Set.t - val include_flags : ?project:Dune_project.t -> t -> Lib_mode.t -> _ Command.Args.t + + val include_flags + : ?project:Dune_project.t + -> direct_libs:t + -> hidden_libs:t + -> Lib_mode.t + -> _ Command.Args.t + val melange_emission_include_flags : ?project:Dune_project.t -> t -> _ Command.Args.t val c_include_flags : t -> Super_context.t -> _ Command.Args.t val toplevel_ld_paths : t -> Path.Set.t diff --git a/src/dune_rules/merlin/ocaml_index.ml b/src/dune_rules/merlin/ocaml_index.ml index 561f6a88099..7941191f70e 100644 --- a/src/dune_rules/merlin/ocaml_index.ml +++ b/src/dune_rules/merlin/ocaml_index.ml @@ -24,19 +24,22 @@ let cctx_rules cctx = let obj_dir = Compilation_context.obj_dir cctx in let fn = index_path_in_obj_dir obj_dir in let additional_libs = - let open Resolve.Memo.O in - let+ non_compile_libs = - (* The indexer relies on the load_path of cmt files. When - [implicit_transitive_deps] is set to [false] some necessary paths will - be missing.These are passed to the indexer with the `-I` flag. - - The implicit transitive libs correspond to the set: - (requires_link \ req_compile) *) - let* req_link = Compilation_context.requires_link cctx in - let+ req_compile = Compilation_context.requires_compile cctx in - List.filter req_link ~f:(fun l -> not (List.exists req_compile ~f:(Lib.equal l))) - in - Lib_flags.L.include_flags non_compile_libs (Lib_mode.Ocaml Byte) + let scope = Compilation_context.scope cctx in + (* Dune language >= 3.17 correctly passes the `-H` flag to the compiler. *) + if Dune_project.dune_version (Scope.project scope) < (3, 17) + then + let open Resolve.Memo.O in + let+ non_compile_libs = + let* req_link = Compilation_context.requires_link cctx in + let+ req_compile = Compilation_context.requires_compile cctx in + List.filter req_link ~f:(fun l -> + not (List.exists req_compile ~f:(Lib.equal l))) + in + Lib_flags.L.include_flags + ~direct_libs:non_compile_libs + ~hidden_libs:[] + (Lib_mode.Ocaml Byte) + else Resolve.Memo.return Command.Args.empty in (* Indexing depends (recursively) on [required_compile] libs: - These libs's cmt files should be built before indexing starts diff --git a/src/ocaml/version.ml b/src/ocaml/version.ml index fba7b9e1a25..3caf3c7c444 100644 --- a/src/ocaml/version.ml +++ b/src/ocaml/version.ml @@ -29,3 +29,4 @@ let supports_alerts version = version >= (4, 8, 0) let has_sandboxed_otherlibs version = version >= (5, 0, 0) let has_META_files version = version >= (5, 0, 0) let supports_bin_annot_occurrences version = version >= (5, 2, 0) +let supports_hidden_includes version = version >= (5, 2, 0) diff --git a/src/ocaml/version.mli b/src/ocaml/version.mli index f7da69ad2b1..dc79429144e 100644 --- a/src/ocaml/version.mli +++ b/src/ocaml/version.mli @@ -76,3 +76,6 @@ val has_META_files : t -> bool (** Whether the compiler supports occurrences indexation *) val supports_bin_annot_occurrences : t -> bool + +(** Whether the compiler supports the -H flag *) +val supports_hidden_includes : t -> bool diff --git a/test/blackbox-tests/test-cases/dune b/test/blackbox-tests/test-cases/dune index 5eb82ae6938..0934807c370 100644 --- a/test/blackbox-tests/test-cases/dune +++ b/test/blackbox-tests/test-cases/dune @@ -164,3 +164,13 @@ (cram (applies_to sandboxing-stale-directory-target) (deps %{bin:bash})) + +(cram + (applies_to hidden-deps-supported) + (enabled_if + (>= %{ocaml_version} 5.2.0))) + +(cram + (applies_to hidden-deps-unsupported) + (enabled_if + (< %{ocaml_version} 5.2.0))) diff --git a/test/blackbox-tests/test-cases/hidden-deps-supported.t/bar.ml b/test/blackbox-tests/test-cases/hidden-deps-supported.t/bar.ml new file mode 100644 index 00000000000..98146d7098e --- /dev/null +++ b/test/blackbox-tests/test-cases/hidden-deps-supported.t/bar.ml @@ -0,0 +1,2 @@ +let x = 5 +let y = Foo.v diff --git a/test/blackbox-tests/test-cases/hidden-deps-supported.t/dune b/test/blackbox-tests/test-cases/hidden-deps-supported.t/dune new file mode 100644 index 00000000000..1807d44beba --- /dev/null +++ b/test/blackbox-tests/test-cases/hidden-deps-supported.t/dune @@ -0,0 +1,18 @@ +(library + (name foo) + (modules foo)) + +(library + (name bar) + (modules bar) + (libraries foo)) + +(executable + (name run) + (modules run) + (libraries bar)) + + (executable + (name runf) + (modules runf) + (libraries bar)) diff --git a/test/blackbox-tests/test-cases/hidden-deps-supported.t/foo.ml b/test/blackbox-tests/test-cases/hidden-deps-supported.t/foo.ml new file mode 100644 index 00000000000..dfa3cba9d50 --- /dev/null +++ b/test/blackbox-tests/test-cases/hidden-deps-supported.t/foo.ml @@ -0,0 +1,2 @@ +let v = 9 +let w = 4 diff --git a/test/blackbox-tests/test-cases/hidden-deps-supported.t/run.ml b/test/blackbox-tests/test-cases/hidden-deps-supported.t/run.ml new file mode 100644 index 00000000000..a956be2d6c3 --- /dev/null +++ b/test/blackbox-tests/test-cases/hidden-deps-supported.t/run.ml @@ -0,0 +1 @@ +let _ = Bar.y diff --git a/test/blackbox-tests/test-cases/hidden-deps-supported.t/run.t b/test/blackbox-tests/test-cases/hidden-deps-supported.t/run.t new file mode 100644 index 00000000000..37c46fa06c4 --- /dev/null +++ b/test/blackbox-tests/test-cases/hidden-deps-supported.t/run.t @@ -0,0 +1,57 @@ +This test is guarded by ocaml version >= 5.2, so it should include foo with -H when +implicit_transitive_deps is set to false. + + $ getincludes () { + > dune build --verbose ./run.exe 2>&1 | grep run.ml | grep -Eo '\-[IH] [a-z/.]+' | sort + > } + + $ cat >dune-project < (lang dune 3.17) + > (implicit_transitive_deps true) + > EOF + + $ getincludes + -I .bar.objs/byte + -I .bar.objs/byte + -I .bar.objs/native + -I .foo.objs/byte + -I .foo.objs/byte + -I .foo.objs/native + -I .run.eobjs/byte + -I .run.eobjs/byte + -I .run.eobjs/native + + $ cat >dune-project < (lang dune 3.17) + > (implicit_transitive_deps false) + > EOF + + $ getincludes + -H .foo.objs/byte + -H .foo.objs/byte + -H .foo.objs/native + -I .bar.objs/byte + -I .bar.objs/byte + -I .bar.objs/native + -I .run.eobjs/byte + -I .run.eobjs/byte + -I .run.eobjs/native + +Test transitive deps can not be directly accessed, both for compiler versions supporting -H or not: + + $ cat >dune-project < (lang dune 3.17) + > (implicit_transitive_deps false) + > EOF + + $ dune build ./runf.exe 2>&1 | grep -v ocamlc + File "runf.ml", line 1, characters 16-21: + 1 | let a = Bar.y + Foo.v + ^^^^^ + Error: Unbound module Foo + +Test if #274 is fixed: + + $ dune build --root=./tyxml + Entering directory 'tyxml' + Leaving directory 'tyxml' diff --git a/test/blackbox-tests/test-cases/hidden-deps-supported.t/runf.ml b/test/blackbox-tests/test-cases/hidden-deps-supported.t/runf.ml new file mode 100644 index 00000000000..0d3f1db0e17 --- /dev/null +++ b/test/blackbox-tests/test-cases/hidden-deps-supported.t/runf.ml @@ -0,0 +1 @@ +let a = Bar.y + Foo.v diff --git a/test/blackbox-tests/test-cases/hidden-deps-supported.t/tyxml/dune b/test/blackbox-tests/test-cases/hidden-deps-supported.t/tyxml/dune new file mode 100644 index 00000000000..14cd546b25b --- /dev/null +++ b/test/blackbox-tests/test-cases/hidden-deps-supported.t/tyxml/dune @@ -0,0 +1,4 @@ +(executable + (name run) + (libraries tyxml)) + \ No newline at end of file diff --git a/test/blackbox-tests/test-cases/hidden-deps-supported.t/tyxml/dune-project b/test/blackbox-tests/test-cases/hidden-deps-supported.t/tyxml/dune-project new file mode 100644 index 00000000000..fff9b78b842 --- /dev/null +++ b/test/blackbox-tests/test-cases/hidden-deps-supported.t/tyxml/dune-project @@ -0,0 +1,2 @@ +(lang dune 3.17) +(implicit_transitive_deps false) diff --git a/test/blackbox-tests/test-cases/hidden-deps-supported.t/tyxml/run.ml b/test/blackbox-tests/test-cases/hidden-deps-supported.t/tyxml/run.ml new file mode 100644 index 00000000000..66f0f01edf5 --- /dev/null +++ b/test/blackbox-tests/test-cases/hidden-deps-supported.t/tyxml/run.ml @@ -0,0 +1,2 @@ +open Tyxml.Html +let _ = p [ a [ txt "a" ] ] diff --git a/test/blackbox-tests/test-cases/hidden-deps-unsupported.t/bar.ml b/test/blackbox-tests/test-cases/hidden-deps-unsupported.t/bar.ml new file mode 100644 index 00000000000..98146d7098e --- /dev/null +++ b/test/blackbox-tests/test-cases/hidden-deps-unsupported.t/bar.ml @@ -0,0 +1,2 @@ +let x = 5 +let y = Foo.v diff --git a/test/blackbox-tests/test-cases/hidden-deps-unsupported.t/dune b/test/blackbox-tests/test-cases/hidden-deps-unsupported.t/dune new file mode 100644 index 00000000000..c5481663927 --- /dev/null +++ b/test/blackbox-tests/test-cases/hidden-deps-unsupported.t/dune @@ -0,0 +1,14 @@ +(library + (name foo) + (modules foo)) + +(library + (name bar) + (modules bar) + (libraries foo)) + +(executable + (name run) + (modules run) + (libraries bar)) + \ No newline at end of file diff --git a/test/blackbox-tests/test-cases/hidden-deps-unsupported.t/foo.ml b/test/blackbox-tests/test-cases/hidden-deps-unsupported.t/foo.ml new file mode 100644 index 00000000000..dfa3cba9d50 --- /dev/null +++ b/test/blackbox-tests/test-cases/hidden-deps-unsupported.t/foo.ml @@ -0,0 +1,2 @@ +let v = 9 +let w = 4 diff --git a/test/blackbox-tests/test-cases/hidden-deps-unsupported.t/run.ml b/test/blackbox-tests/test-cases/hidden-deps-unsupported.t/run.ml new file mode 100644 index 00000000000..436b88a6649 --- /dev/null +++ b/test/blackbox-tests/test-cases/hidden-deps-unsupported.t/run.ml @@ -0,0 +1,2 @@ +let _ = Bar.y +let _ = print_endline "yes" diff --git a/test/blackbox-tests/test-cases/hidden-deps-unsupported.t/run.t b/test/blackbox-tests/test-cases/hidden-deps-unsupported.t/run.t new file mode 100644 index 00000000000..c51b4249eae --- /dev/null +++ b/test/blackbox-tests/test-cases/hidden-deps-unsupported.t/run.t @@ -0,0 +1,36 @@ +This test is guarded by ocaml version <= 5.1, so it should not include foo +when implicit_transitive_deps is set to false, i.e. testing backward compatibility of +the new -H feature added. + + $ getincludes () { + > dune build --verbose ./run.exe 2>&1 | grep run.ml | grep -Eo '\-[IH] [a-z/.]+' | sort + > } + + $ cat >dune-project < (lang dune 3.17) + > (implicit_transitive_deps true) + > EOF + + $ getincludes + -I .bar.objs/byte + -I .bar.objs/byte + -I .bar.objs/native + -I .foo.objs/byte + -I .foo.objs/byte + -I .foo.objs/native + -I .run.eobjs/byte + -I .run.eobjs/byte + -I .run.eobjs/native + + $ cat >dune-project < (lang dune 3.17) + > (implicit_transitive_deps false) + > EOF + + $ getincludes + -I .bar.objs/byte + -I .bar.objs/byte + -I .bar.objs/native + -I .run.eobjs/byte + -I .run.eobjs/byte + -I .run.eobjs/native