diff --git a/lib/build.ml b/lib/build.ml index caf66985..5d6cbfe7 100644 --- a/lib/build.ml +++ b/lib/build.ml @@ -24,10 +24,12 @@ module Spec = struct revdep : package option; with_tests : bool; lower_bounds : bool; + local : bool; opam_version : [`V2_0 | `V2_1 | `Dev]; } [@@deriving to_yojson] type list_revdeps = { + local : bool; opam_version : [`V2_0 | `V2_1 | `Dev]; } [@@deriving to_yojson] @@ -40,8 +42,8 @@ module Spec = struct ty : ty; } - let opam ?revdep ~platform ~lower_bounds ~with_tests ~opam_version pkg = - let ty = `Opam (`Build { revdep; lower_bounds; with_tests; opam_version }, pkg) in + let opam ?revdep ~platform ~lower_bounds ~with_tests ~local ~opam_version pkg = + let ty = `Opam (`Build { revdep; lower_bounds; with_tests; local; opam_version }, pkg) in { platform; ty } let pp_pkg ?revdep f pkg = @@ -55,14 +57,16 @@ module Spec = struct | `Dev -> "dev" let pp_ty f = function - | `Opam (`List_revdeps {opam_version}, pkg) -> - Fmt.pf f "list revdeps of %s, using opam %s" (OpamPackage.to_string pkg) + | `Opam (`List_revdeps {local; opam_version}, pkg) -> + Fmt.pf f "list revdeps of %s, using opam %s%s" (OpamPackage.to_string pkg) (pp_opam_version opam_version) - | `Opam (`Build { revdep; lower_bounds; with_tests; opam_version }, pkg) -> + (if local then ", using a local switch" else "") + | `Opam (`Build { revdep; lower_bounds; with_tests; local; opam_version }, pkg) -> let action = if with_tests then "test" else "build" in - Fmt.pf f "%s %a%s, using opam %s" action (pp_pkg ?revdep) pkg + Fmt.pf f "%s %a%s, using opam %s%s" action (pp_pkg ?revdep) pkg (if lower_bounds then ", lower-bounds" else "") (pp_opam_version opam_version) + (if local then ", using a local switch" else "") end type t = { @@ -143,8 +147,8 @@ module Op = struct let build_spec ~for_docker = let base = base_to_string base in match ty with - | `Opam (`List_revdeps { opam_version }, pkg) -> Opam_build.revdeps ~for_docker ~opam_version ~base ~variant ~pkg - | `Opam (`Build { revdep; lower_bounds; with_tests; opam_version }, pkg) -> Opam_build.spec ~for_docker ~opam_version ~base ~variant ~revdep ~lower_bounds ~with_tests ~pkg + | `Opam (`List_revdeps { local; opam_version }, pkg) -> Opam_build.revdeps ~for_docker ~local ~opam_version ~base ~variant ~pkg + | `Opam (`Build { revdep; lower_bounds; with_tests; local; opam_version }, pkg) -> Opam_build.spec ~for_docker ~local ~opam_version ~base ~variant ~revdep ~lower_bounds ~with_tests ~pkg in Current.Job.write job (Fmt.str "@.\ @@ -218,7 +222,8 @@ let v t ~label ~spec ~base ~master ~urgent commit = BC.run t { Op.Key.pool; commit; variant; ty } () |> Current.Primitive.map_result (Result.map ignore) (* TODO: Create a separate type of cache that doesn't parse the output *) -let list_revdeps t ~platform ~opam_version ~pkgopt ~base ~master ~after commit = +(* TODO: Remove the duplicated opam_version parameter (already contained in t) *) +let list_revdeps t ~platform ~local ~opam_version ~pkgopt ~base ~master ~after commit = Current.component "list revdeps" |> let> {PackageOpt.pkg; urgent; has_tests = _} = pkgopt and> base = base @@ -227,7 +232,7 @@ let list_revdeps t ~platform ~opam_version ~pkgopt ~base ~master ~after commit = and> () = after in let t = { Op.config = t; master; urgent; base } in let { Platform.pool; variant; label = _ } = platform in - let ty = `Opam (`List_revdeps {Spec.opam_version}, pkg) in + let ty = `Opam (`List_revdeps {Spec.local; opam_version}, pkg) in BC.run t { Op.Key.pool; commit; variant; ty } () |> Current.Primitive.map_result (Result.map (fun output -> String.split_on_char '\n' output |> diff --git a/lib/build.mli b/lib/build.mli index fd24ac2a..0d36c30b 100644 --- a/lib/build.mli +++ b/lib/build.mli @@ -6,6 +6,7 @@ module Spec : sig platform:Platform.t -> lower_bounds:bool -> with_tests:bool -> + local:bool -> opam_version:[`V2_0 | `V2_1 | `Dev] -> OpamPackage.t -> t @@ -32,6 +33,7 @@ val v : val list_revdeps : t -> platform:Platform.t -> + local:bool -> opam_version:[`V2_0 | `V2_1 | `Dev] -> pkgopt:PackageOpt.t Current.t -> base:base Current.t -> diff --git a/lib/opam_build.ml b/lib/opam_build.ml index 602428b3..3fe2461c 100644 --- a/lib/opam_build.ml +++ b/lib/opam_build.ml @@ -55,7 +55,7 @@ let opam_install ~variant ~opam_version ~pin ~lower_bounds ~with_tests ~pkg = pkg ] -let setup_repository ~variant ~for_docker ~opam_version = +let setup_repository ~variant ~for_docker ~local ~opam_version = let open Obuilder_spec in let home_dir = match Variant.os variant with | `macOS -> None @@ -99,6 +99,21 @@ let setup_repository ~variant ~for_docker ~opam_version = env "OPAMERRLOGLEN" "0" :: (* Show the whole log if it fails *) env "OPAMSOLVERTIMEOUT" "500" :: (* Increase timeout. Poor mccs is doing its best *) env "OPAMPRECISETRACKING" "1" :: (* Mitigate https://github.com/ocaml/opam/issues/3997 *) + (if local then + (* TODO: Check global switch parent directory with spaces on Windows *) + (* NOTE: Check that the package supports directories with spaces *) + let dirname = "directory with space" in + let packages = Variant.packages variant in + [ + run "opam switch remove \"$(opam switch show)\""; + run "mkdir '%s' && opam switch create './%s' --packages '%s'" + dirname dirname packages; + (* NOTE: Check that the package supports being inside a dune project by + having a dune-project file in the parent directory and having a required dune version + too high so the call to dune would fail. *) + run "echo '(lang dune 9999.0)' > '%s'/dune-project" dirname; + ] + else []) @ [ run "rm -rf opam-repository/"; copy ["."] ~dst:"opam-repository/"; @@ -111,7 +126,7 @@ let set_personality ~variant = else [] -let spec ~for_docker ~opam_version ~base ~variant ~revdep ~lower_bounds ~with_tests ~pkg = +let spec ~for_docker ~local ~opam_version ~base ~variant ~revdep ~lower_bounds ~with_tests ~pkg = let opam_install = opam_install ~variant ~opam_version in let revdep = match revdep with | None -> [] @@ -126,18 +141,18 @@ let spec ~for_docker ~opam_version ~base ~variant ~revdep ~lower_bounds ~with_te in Obuilder_spec.stage ~from:base ( set_personality ~variant - @ setup_repository ~variant ~for_docker ~opam_version + @ setup_repository ~variant ~for_docker ~local ~opam_version @ opam_install ~pin:true ~lower_bounds:false ~with_tests:false ~pkg @ lower_bounds @ revdep @ tests ) -let revdeps ~for_docker ~opam_version ~base ~variant ~pkg = +let revdeps ~for_docker ~local ~opam_version ~base ~variant ~pkg = let open Obuilder_spec in let pkg = Filename.quote (OpamPackage.to_string pkg) in Obuilder_spec.stage ~from:base ( - setup_repository ~variant ~for_docker ~opam_version + setup_repository ~variant ~for_docker ~local ~opam_version @ [ run "echo '@@@OUTPUT' && \ opam list -s --color=never --depends-on %s --coinstallable-with %s --installable --all-versions --recursive --depopts && \ diff --git a/lib/opam_build.mli b/lib/opam_build.mli index e6619ac0..c0af6754 100644 --- a/lib/opam_build.mli +++ b/lib/opam_build.mli @@ -1,5 +1,6 @@ val spec : for_docker:bool -> + local:bool -> opam_version:[`V2_0 | `V2_1 | `Dev] -> base:string -> variant:Variant.t -> @@ -11,6 +12,7 @@ val spec : val revdeps : for_docker:bool -> + local:bool -> opam_version:[`V2_0 | `V2_1 | `Dev] -> base:string -> variant:Variant.t -> diff --git a/lib/variant.ml b/lib/variant.ml index 4b1bc886..5faa97dc 100644 --- a/lib/variant.ml +++ b/lib/variant.ml @@ -21,6 +21,16 @@ let docker_tag t = t.distribution^"-ocaml-"^pp_ocaml_version t let distribution t = t.distribution +let packages t = + let version = Ocaml_version.with_patch (Ocaml_version.of_string_exn t.ocaml_version) (Some 0) in + let comp = + let name, version = Ocaml_version.Opam.V2.package version in + name^"."^version + in + match Ocaml_version.Opam.V2.additional_packages version with + | [] -> comp + | extras -> comp^","^String.concat "," extras + let pp f t = Fmt.pf f "%s/%s" (docker_tag t) (Ocaml_version.string_of_arch t.arch) let macos_homebrew = "macos-homebrew" diff --git a/lib/variant.mli b/lib/variant.mli index f45d14fb..a2fd33a6 100644 --- a/lib/variant.mli +++ b/lib/variant.mli @@ -11,6 +11,8 @@ val docker_tag : t -> string val distribution : t -> string val os : t -> [`linux | `macOS] +val packages : t -> string + val pp : t Fmt.t val macos_homebrew : string diff --git a/service/pipeline.ml b/service/pipeline.ml index 702edf50..c3baa4d4 100644 --- a/service/pipeline.ml +++ b/service/pipeline.ml @@ -58,29 +58,29 @@ let with_label l t = let> v = t in Current.Primitive.const v -let build_spec ~platform ~opam_version pkg = +let build_spec ~platform ~local ~opam_version pkg = let+ pkg = pkg in - Build.Spec.opam ~platform ~lower_bounds:false ~with_tests:false ~opam_version pkg + Build.Spec.opam ~platform ~lower_bounds:false ~with_tests:false ~local ~opam_version pkg -let test_spec ~platform ~opam_version pkg = +let test_spec ~platform ~local ~opam_version pkg = let+ pkg = pkg in - Build.Spec.opam ~platform ~lower_bounds:false ~with_tests:true ~opam_version pkg + Build.Spec.opam ~platform ~lower_bounds:false ~with_tests:true ~local ~opam_version pkg -let lower_bounds_spec ~platform ~opam_version pkg = +let lower_bounds_spec ~platform ~local ~opam_version pkg = let+ pkg = pkg in - Build.Spec.opam ~platform ~lower_bounds:true ~with_tests:false ~opam_version pkg + Build.Spec.opam ~platform ~lower_bounds:true ~with_tests:false ~local ~opam_version pkg -let revdep_spec ~platform ~opam_version ~revdep pkg = +let revdep_spec ~platform ~local ~opam_version ~revdep pkg = let+ revdep = revdep and+ pkg = pkg in - Build.Spec.opam ~platform ~lower_bounds:false ~with_tests:true ~revdep ~opam_version pkg + Build.Spec.opam ~platform ~lower_bounds:false ~with_tests:true ~revdep ~local ~opam_version pkg (* List the revdeps of [pkg] (using [builder] and [image]) and test each one (using [spec] and [base], merging [source] into [master]). *) -let test_revdeps ~ocluster ~opam_version ~master ~base ~platform ~pkgopt ~after source = +let test_revdeps ~ocluster ~local ~opam_version ~master ~base ~platform ~pkgopt ~after source = let revdeps = - Build.list_revdeps ~opam_version ~base ocluster ~platform ~pkgopt ~master ~after source |> + Build.list_revdeps ~local ~opam_version ~base ocluster ~platform ~pkgopt ~master ~after source |> Current.map OpamPackage.Set.elements in let pkg = Current.map (fun {PackageOpt.pkg = pkg; urgent = _; has_tests = _} -> pkg) pkgopt in @@ -89,7 +89,7 @@ let test_revdeps ~ocluster ~opam_version ~master ~base ~platform ~pkgopt ~after revdeps |> Node.list_map (module OpamPackage) (fun revdep -> let image = - let spec = revdep_spec ~platform ~opam_version ~revdep pkg in + let spec = revdep_spec ~platform ~local ~opam_version ~revdep pkg in Build.v ocluster ~label:"build" ~base ~spec ~master ~urgent source in let label = Current.map OpamPackage.to_string revdep @@ -109,7 +109,7 @@ let get_significant_available_pkg = function let build_with_cluster ~ocluster ~analysis ~lint ~master source = let pkgs = Current.map Analyse.Analysis.packages analysis in let pkgs = Current.map (List.filter_map get_significant_available_pkg) pkgs in - let build ~opam_version ~lower_bounds ~revdeps label variant = + let build ?(local=false) ~opam_version ~lower_bounds ~revdeps label variant = let arch = Variant.arch variant in let pool = Conf.pool_of_arch variant in let platform = {Platform.label; pool; variant} in @@ -138,13 +138,13 @@ let build_with_cluster ~ocluster ~analysis ~lint ~master source = Build.Docker (Current_docker.Raw.Image.of_hash repo_id) in let image = - let spec = build_spec ~platform ~opam_version pkg in + let spec = build_spec ~platform ~local ~opam_version pkg in Build.v ocluster ~label:"build" ~base ~spec ~master ~urgent source in let build = Node.action `Built image and tests = Node.bool_map (fun () -> let action = - let spec = test_spec ~platform ~opam_version pkg in + let spec = test_spec ~platform ~local ~opam_version pkg in Build.v ocluster ~label:"test" ~base ~spec ~master ~urgent source in let action = Node.action `Built action in @@ -153,7 +153,7 @@ let build_with_cluster ~ocluster ~analysis ~lint ~master source = and lower_bounds_check = if lower_bounds then let action = - let spec = lower_bounds_spec ~platform ~opam_version pkg in + let spec = lower_bounds_spec ~platform ~local ~opam_version pkg in Build.v ocluster ~label:"lower-bounds" ~base ~spec ~master ~urgent source in let action = Node.action `Built action in @@ -161,7 +161,7 @@ let build_with_cluster ~ocluster ~analysis ~lint ~master source = else Node.empty and revdeps = - if revdeps then test_revdeps ~ocluster ~opam_version ~master ~base ~platform ~pkgopt source ~after:image + if revdeps then test_revdeps ~ocluster ~local ~opam_version ~master ~base ~platform ~pkgopt source ~after:image else Node.empty in let label = Current.map OpamPackage.to_string pkg in @@ -222,17 +222,18 @@ let build_with_cluster ~ocluster ~analysis ~lint ~master source = let analysis = Node.action `Analysed analysis and lint = Node.action `Linted lint and extras = - let build ~opam_version ~distro ~arch ~compiler label = + let build ?local ~opam_version ~distro ~arch ~compiler label = let variant = Variant.v ~arch ~distro ~compiler in let label = if String.equal label "" then "" else label^"-" in let label = Fmt.str "%socaml-%s" label (Variant.pp_ocaml_version variant) in - build ~opam_version ~lower_bounds:false ~revdeps:false label variant + build ?local ~opam_version ~lower_bounds:false ~revdeps:false label variant in let master_distro = Distro.tag_of_distro master_distro in List.fold_left (fun acc comp_full -> let comp = Ocaml_version.to_string (Ocaml_version.with_just_major_and_minor comp_full) in build ~opam_version:`V2_0 ~arch:`X86_64 ~distro:master_distro ~compiler:(comp, None) "opam-2.0" :: build ~opam_version:`V2_1 ~arch:`X86_64 ~distro:master_distro ~compiler:(comp, None) "opam-2.1" :: + build ~local:true ~opam_version ~arch:`X86_64 ~distro:master_distro ~compiler:(comp, None) "local-switch" :: List.filter_map (fun v -> match Ocaml_version.extra v with | None -> None