Driver: Split compile and link phases

src/driver/compile.ml

@@ -58,6 +58,7 @@ type partial =
   * Odoc_unit.intf Odoc_unit.unit list Util.StringMap.t
 
 let unmarshal filename : partial =
+  Format.eprintf "unmarshalling %s\n%!" (Fpath.to_string filename);
   let ic = open_in_bin (Fpath.to_string filename) in
   Fun.protect
     ~finally:(fun () -> close_in ic)
@@ -66,6 +67,17 @@ let unmarshal filename : partial =
 let marshal (v : partial) filename =
   let _ = OS.Dir.create (Fpath.parent filename) |> Result.get_ok in
   let oc = open_out_bin (Fpath.to_string filename) in
+  let keys_fst = List.map fst (fst v) |> List.sort String.compare in
+  let keys_snd =
+    Util.StringMap.bindings (snd v) |> List.map fst |> List.sort String.compare
+  in
+  Format.eprintf "marshal %s\n%!" (Fpath.to_string filename);
+  Format.eprintf "keys_fst: (%d) [%a]\n" (List.length keys_fst)
+    Fmt.(list ~sep:comma string)
+    keys_fst;
+  Format.eprintf "keys_snd: (%d) [%a]\n" (List.length keys_snd)
+    Fmt.(list ~sep:comma string)
+    keys_snd;
   Fun.protect
     ~finally:(fun () -> close_out oc)
     (fun () -> Marshal.to_channel oc v [])

src/driver/compile.mli

@@ -1,4 +1,4 @@
-type compiled
+type compiled = Odoc_unit.t
 
 val init_stats : Odoc_unit.t list -> unit
 

src/driver/library_names.ml

@@ -30,6 +30,7 @@ let read_libraries_from_pkg_defs ~library_name pkg_defs =
 
     let deps_str = Fl_metascanner.lookup "requires" [] pkg_defs in
     let deps = Astring.String.fields deps_str in
+    let deps = List.filter (fun x -> String.length x > 0) deps in
     let dir =
       List.find_opt (fun d -> d.Fl_metascanner.def_var = "directory") pkg_defs
     in

src/driver/odoc_driver.ml

@@ -553,7 +553,7 @@ let remap_virtual_interfaces duplicate_hashes pkgs =
 
 let run libs verbose packages_dir odoc_dir odocl_dir html_dir stats nb_workers
     odoc_bin voodoo package_name blessed dune_style compile_grep link_grep
-    generate_grep =
+    generate_grep compile_only no_compile =
   Option.iter (fun odoc_bin -> Odoc.odoc := Bos.Cmd.v odoc_bin) odoc_bin;
   let _ = Voodoo.find_universe_and_version "foo" in
   Eio_main.run @@ fun env ->
@@ -563,21 +563,21 @@ let run libs verbose packages_dir odoc_dir odocl_dir html_dir stats nb_workers
   Stats.init_nprocs nb_workers;
   let () = Worker_pool.start_workers env sw nb_workers in
 
-  let all, extra_libs_paths =
+  let all, extra_paths =
     match (voodoo, package_name, dune_style, packages_dir) with
     | true, Some p, None, None ->
         let all = Voodoo.of_voodoo p ~blessed in
-        let extra_libs_paths = Voodoo.extra_libs_paths odoc_dir in
+        let extra_libs_paths = Voodoo.extra_paths odoc_dir in
         (all, extra_libs_paths)
     | false, None, Some dir, None ->
-        (Dune_style.of_dune_build dir, Util.StringMap.empty)
+        (Dune_style.of_dune_build dir, Util.StringMap.(empty, empty))
     | false, None, None, packages_dir ->
         let libs = if libs = [] then Ocamlfind.all () else libs in
         let libs =
           List.map Ocamlfind.sub_libraries libs
           |> List.fold_left Util.StringSet.union Util.StringSet.empty
         in
-        (Packages.of_libs ~packages_dir libs, Util.StringMap.empty)
+        (Packages.of_libs ~packages_dir libs, Util.StringMap.(empty, empty))
     | true, None, _, _ -> failwith "--voodoo requires --package"
     | false, Some _, _, _ -> failwith "--package requires --voodoo"
     | true, _, _, Some _ | false, _, Some _, Some _ ->
@@ -626,7 +626,7 @@ let run libs verbose packages_dir odoc_dir odocl_dir html_dir stats nb_workers
           let all = Util.StringMap.bindings all |> List.map snd in
           let internal =
             Odoc_unit.of_packages ~output_dir:odoc_dir ~linked_dir:odocl_dir
-              ~index_dir:None ~extra_libs_paths all
+              ~index_dir:None ~extra_paths all
           in
           let external_ =
             let mld_dir = odoc_dir in
@@ -638,13 +638,17 @@ let run libs verbose packages_dir odoc_dir odocl_dir html_dir stats nb_workers
         in
         Compile.init_stats all;
         let compiled =
-          Compile.compile ?partial ~partial_dir:odoc_dir ?linked_dir:odocl_dir
-            all
+          if no_compile then all
+          else
+            Compile.compile ?partial ~partial_dir:odoc_dir ?linked_dir:odocl_dir
+              all
         in
-        let linked = Compile.link compiled in
-        let () = Compile.html_generate html_dir linked in
-        let _ = Odoc.support_files html_dir in
-        ())
+        if compile_only then ()
+        else
+          let linked = Compile.link compiled in
+          let () = Compile.html_generate html_dir linked in
+          let _ = Odoc.support_files html_dir in
+          ())
       (fun () -> render_stats env nb_workers)
   in
 
@@ -732,14 +736,23 @@ let generate_grep =
   let doc = "Show html-generate commands containing the string" in
   Arg.(value & opt (some string) None & info [ "html-grep" ] ~doc)
 
+let compile_only =
+  let doc = "Only compile the odoc files, don't link" in
+  Arg.(value & flag & info [ "compile-only" ] ~doc)
+
+let no_compile =
+  let doc = "Don't compile" in
+  Arg.(value & flag & info [ "no-compile" ] ~doc)
+
 let cmd =
   let doc = "Generate odoc documentation" in
   let info = Cmd.info "odoc_driver" ~doc in
   Cmd.v info
     Term.(
       const run $ packages $ verbose $ packages_dir $ odoc_dir $ odocl_dir
       $ html_dir $ stats $ nb_workers $ odoc_bin $ voodoo $ package_name
-      $ blessed $ dune_style $ compile_grep $ link_grep $ generate_grep)
+      $ blessed $ dune_style $ compile_grep $ link_grep $ generate_grep
+      $ compile_only $ no_compile)
 
 (* let map = Ocamlfind.package_to_dir_map () in
    let _dirs = List.map (fun lib -> List.assoc lib map) deps in

src/driver/odoc_unit.ml

@@ -95,8 +95,9 @@ and pp : all_kinds unit Fmt.t =
     (Fmt.option pp_index) x.index pp_kind
     (x.kind :> all_kinds)
 
-let of_packages ~output_dir ~linked_dir ~index_dir ~extra_libs_paths
-    (pkgs : Packages.t list) : t list =
+let of_packages ~output_dir ~linked_dir ~index_dir
+    ~extra_paths:(extra_pkg_paths, extra_libs_paths) (pkgs : Packages.t list) :
+    t list =
   let linked_dir =
     match linked_dir with None -> output_dir | Some dir -> dir
   in
@@ -129,22 +130,24 @@ let of_packages ~output_dir ~linked_dir ~index_dir ~extra_libs_paths
           (h, lds) pkg.libraries)
       (h, lds) pkgs
   in
-  let pkg_map =
-    Util.StringMap.of_list (List.map (fun pkg -> (pkg.Packages.name, pkg)) pkgs)
+  let pkg_paths =
+    List.fold_left
+      (fun acc pkg -> Util.StringMap.add pkg.Packages.name (doc_dir pkg) acc)
+      extra_pkg_paths pkgs
   in
 
-  let dash_p pkg = (pkg.Packages.name, doc_dir pkg) in
+  let dash_p pkgname path = (pkgname, path) in
 
   let dash_l lib_name =
     match Util.StringMap.find_opt lib_name lib_dirs with
     | Some dir -> [ (lib_name, dir) ]
     | None ->
-        Logs.err (fun m -> m "Library %s not found" lib_name);
+        Logs.err (fun m -> m "Library '%s' not found" lib_name);
         []
   in
   (* Given a pkg,  *)
   let base_args pkg lib_deps : pkg_args =
-    let own_page = dash_p pkg in
+    let own_page = dash_p pkg.Packages.name (doc_dir pkg) in
     let own_libs = List.concat_map dash_l (Util.StringSet.to_list lib_deps) in
     let map_rel dir = List.map (fun (a, b) -> (a, Fpath.(dir // b))) in
     let pages = map_rel output_dir [ own_page ] in
@@ -158,9 +161,11 @@ let of_packages ~output_dir ~linked_dir ~index_dir ~extra_libs_paths
     let pages_rel =
       List.filter_map
         (fun pkgname ->
-          match Util.StringMap.find_opt pkgname pkg_map with
-          | None -> None
-          | Some pkg -> Some (dash_p pkg))
+          match Util.StringMap.find_opt pkgname pkg_paths with
+          | None ->
+              Logs.debug (fun m -> m "Package '%s' not found" pkgname);
+              None
+          | Some path -> Some (dash_p pkgname path))
         packages
     in
     let libs_rel = List.concat_map dash_l libraries in

src/driver/odoc_unit.mli

@@ -44,6 +44,6 @@ val of_packages :
   output_dir:Fpath.t ->
   linked_dir:Fpath.t option ->
   index_dir:Fpath.t option ->
-  extra_libs_paths:Fpath.t Util.StringMap.t ->
+  extra_paths:Fpath.t Util.StringMap.t * Fpath.t Util.StringMap.t ->
   Packages.t list ->
   t list

src/driver/packages.ml

@@ -74,7 +74,8 @@ let pp_libty fmt l =
      }@]"
     l.lib_name Fpath.pp l.dir
     (Fmt.Dump.option Fmt.string)
-    l.archive_name (Fmt.Dump.list Fmt.string)
+    l.archive_name
+    (Fmt.list ~sep:Fmt.comma Fmt.string)
     (Util.StringSet.elements l.lib_deps)
     (Fmt.Dump.list pp_modulety)
     l.modules

src/driver/voodoo.ml

@@ -127,15 +127,30 @@ let process_package pkg =
       Util.StringMap.empty metas
   in
 
+  let ss_pp fmt ss = Format.fprintf fmt "[%d]" (Util.StringSet.cardinal ss) in
+  Logs.debug (fun m ->
+      m "all_lib_deps: %a\n%!"
+        Fmt.(list ~sep:comma (pair ~sep:comma string ss_pp))
+        (Util.StringMap.bindings all_lib_deps));
+
   let assets, mlds = assets_and_mlds_of_pkg pkg_path pkg in
 
   let config =
-    let config_file = Fpath.(v "doc" / pkg.name / "odoc-config.sexp") in
+    let config_file =
+      Fpath.(pkg_path / "doc" / pkg.name / "odoc-config.sexp")
+    in
     match Bos.OS.File.read config_file with
-    | Error _ -> Global_config.empty
-    | Ok s -> Global_config.parse s
+    | Error (`Msg msg) ->
+        Logs.debug (fun m ->
+            m "No config file found: %a\n%s\n%!" Fpath.pp config_file msg);
+        Global_config.empty
+    | Ok s ->
+        Logs.debug (fun m -> m "Config file: %a\n%!" Fpath.pp config_file);
+        Global_config.parse s
   in
 
+  Logs.debug (fun m ->
+      m "Config.packages: %s\n%!" (String.concat ", " config.deps.packages));
   let meta_libraries : Packages.libty list =
     metas
     |> List.filter_map (fun meta_file ->
@@ -259,24 +274,34 @@ let of_voodoo pkg_name ~blessed =
       in
       let packages = List.filter_map (fun x -> x) (last :: packages) in
       let packages = List.map process_package packages in
+      let pkg = List.hd packages in
+      Logs.debug (fun m -> m "Package: %a\n%!" Packages.pp pkg);
       Util.StringMap.singleton pkg_name (List.hd packages)
 
-let extra_libs_paths compile_dir =
+let extra_paths compile_dir =
   let contents =
     Bos.OS.Dir.fold_contents ~dotfiles:true
       (fun p acc -> p :: acc)
       [] compile_dir
   in
   match contents with
-  | Error _ -> Util.StringMap.empty
+  | Error _ -> (Util.StringMap.empty, Util.StringMap.empty)
   | Ok c ->
       List.fold_left
-        (fun acc abs_path ->
+        (fun (pkgs, libs) abs_path ->
           let path = Fpath.rem_prefix compile_dir abs_path |> Option.get in
-          match Fpath.segs path with
-          | [ "p"; _pkg; _version; "lib"; libname ] ->
-              Util.StringMap.add libname path acc
-          | [ "u"; _universe; _pkg; _version; "lib"; libname ] ->
-              Util.StringMap.add libname path acc
-          | _ -> acc)
-        Util.StringMap.empty c
+          let pkgs', libs' =
+            match Fpath.segs path with
+            | [ "p"; _pkg; _version; "lib"; libname ] ->
+                (pkgs, Util.StringMap.add libname path libs)
+            | [ "p"; pkg; _version; "doc" ] ->
+                (Util.StringMap.add pkg path pkgs, libs)
+            | [ "u"; _universe; _pkg; _version; "lib"; libname ] ->
+                (pkgs, Util.StringMap.add libname path libs)
+            | [ "u"; _universe; pkg; _version; "doc" ] ->
+                (Util.StringMap.add pkg path pkgs, libs)
+            | _ -> (pkgs, libs)
+          in
+          (pkgs', libs'))
+        (Util.StringMap.empty, Util.StringMap.empty)
+        c

src/driver/voodoo.mli

@@ -3,4 +3,4 @@ val find_universe_and_version :
 
 val of_voodoo : string -> blessed:bool -> Packages.set
 
-val extra_libs_paths : Fpath.t -> Fpath.t Util.StringMap.t
+val extra_paths : Fpath.t -> Fpath.t Util.StringMap.t * Fpath.t Util.StringMap.t