Experiment: use GADT for stat

Co-authored-by: Thomas Leonard <talex5@gmail.com>
Co-authored-by: Patrick Ferris <patrick@sirref.org>

.gitignore

@@ -1,3 +1,4 @@
 _build
 .*.swp
 *.install
+.vscode

bench/bench_fstat.ml

@@ -6,7 +6,7 @@ let n_stat = 100000
 
 let run_fiber file =
   for _ = 1 to n_stat do
-    let info = (Eio.File.stat file).kind in
+    let info = (Eio.File.stat file [Kind]) Fun.id in
     assert (info = `Regular_file)
   done
 

bench/bench_stat.ml

@@ -0,0 +1,129 @@
+open Eio
+
+let () = Random.init 3
+
+let ( / ) = Eio.Path.( / )
+
+module Bench_dir = struct
+  type t = 
+    | Dir of { name : string; perm : int; children : t list }
+    | File of { name : string; size : int64; perm : int; }
+
+  let get_name = function Dir { name; _ } | File { name; _ } -> name
+
+  let get_children = function
+    | Dir { children; _ } -> children
+    | File _ -> invalid_arg "Files don't have children"
+
+  let compare a b = String.compare (get_name a) (get_name b)
+
+  let rec sort = function
+    | Dir ({ children; _ } as v) ->
+      let c = List.map sort children in
+      let c = List.stable_sort compare c in
+      Dir { v with children = c }
+    | File _ as f -> f
+
+  let rec size = function
+    | Dir { children; _ } ->
+      List.fold_left (fun acc v -> acc + size v) 0 children
+    | File _ -> 1
+
+  let rec pp ppf = function
+    | Dir { name; perm; children } ->
+      if children = [] then Fmt.pf ppf "dir %s (0o%o)" name perm else
+      Fmt.pf ppf "@[<v2>dir %s (0o%o)@ %a@]" name perm Fmt.(list ~sep:Fmt.cut pp) children
+    | File { name; size; perm } ->
+      Fmt.pf ppf "file %s (0o%o) %Lu" name perm size
+
+  let rec make random fs = function
+  | Dir { name; perm; children } ->
+    let dir = fs / name in
+    Path.mkdir ~perm dir;
+    Fiber.List.iter (make random dir) children
+  | File { name; size; perm } ->
+    Path.with_open_out ~create:(`If_missing perm) (fs / name) @@ fun oc ->
+    let buf = Cstruct.create (Int64.to_int size) in
+    Flow.read_exact random buf;
+    Flow.copy (Flow.cstruct_source [ buf ]) oc
+end
+
+let with_bench_dir ~random ~fs t fn =
+  let dir = Filename.temp_dir "eio-bench-" "-stat" in
+  let root = fs / dir in
+  Bench_dir.make random root t;
+  Fun.protect ~finally:(fun () -> assert (0 = Sys.command ("rm -r " ^ (Path.native_exn root)))) (fun () -> fn root)
+
+let rec bench_stat dir =
+  let kind, perm, size =
+    Path.stat ~follow:false dir File.[ Kind; Perm; Size ] @@ fun kind perm size ->
+    (kind, perm, size)
+  in
+  match kind with
+  | `Directory ->
+    let children = Path.read_dir dir |> Fiber.List.map (fun f -> bench_stat (dir / f)) in
+    let name = Path.native_exn dir |> Filename.basename in
+    Bench_dir.Dir { name; perm; children }
+  | `Regular_file ->
+    let name = Path.native_exn dir |> Filename.basename in
+    File { name; perm; size }
+  | _ -> assert false
+
+let file name = Bench_dir.File { name; perm = 0o644; size = 128L }
+let dir name children = Bench_dir.Dir { name; perm = 0o700; children }
+
+let random_bench_dir ~n ~levels =
+  if levels < 0 then invalid_arg "Levels should be > 0";
+  let rec loop root = function
+    | 1 -> (
+      match root with
+      | Bench_dir.Dir d ->
+        let leaf_files = List.init n (fun i -> file (Fmt.str "test-file-%i-%i" 1 i)) in
+        Bench_dir.Dir { d with children = leaf_files }
+      | _ -> failwith "Root is always expected to be a directory"
+    )
+    | level ->
+      match root with
+      | Bench_dir.Dir d ->
+        let files = List.init n (fun i -> file (Fmt.str "test-file-%i-%i" level i)) in
+        let dirs = List.init n (fun i -> dir (Fmt.str "test-dir-%i-%i" level i) []) in 
+        let dirs = List.map (fun dir -> loop dir (level - 1)) dirs in
+        Bench_dir.Dir { d with children = dirs @ files }
+      | _ -> failwith "Root is always expected to be directory"
+  in
+  loop (dir "root" []) levels
+
+let run_bench ~n ~levels ~random ~fs ~clock =
+  let dir = random_bench_dir ~levels ~n |> Bench_dir.sort in
+  Eio.traceln "Going to create %i files and directories" (Bench_dir.size dir);
+  let bench () =
+    with_bench_dir ~random ~fs dir @@ fun root ->
+    Eio.traceln "Created %i files and directories" (Bench_dir.size dir);
+    Gc.full_major ();
+    let stat0 = Gc.stat () in
+    let t0 = Eio.Time.now clock in
+    let res = bench_stat root in
+    let t1 = Eio.Time.now clock in
+    let stat1 = Gc.stat () in
+    match Bench_dir.sort res with
+    | Dir { children = [ dir' ]; _ } ->
+      assert (dir = dir');
+      let time_total = t1 -. t0 in
+      let minor_total = stat1.minor_words -. stat0.minor_words in
+      let major_total = stat1.major_words -. stat0.major_words in
+      time_total, minor_total, major_total
+    | _ -> failwith "Stat not the same as the spec"
+  in
+  let time, minor, major = bench () in
+  [ 
+    Metric.create "stat-time" (`Float (1e3 *. time)) "ms" (Fmt.str "Time to stat %i files and directories" (Bench_dir.size dir));
+    Metric.create "stat-minor" (`Float (1e-3 *. minor)) "kwords" (Fmt.str "Minor words allocated to stat %i files and directories" (Bench_dir.size dir));
+    Metric.create "stat-major" (`Float (1e-3 *. major)) "kwords" (Fmt.str "Major words allocated  %i files and directories" (Bench_dir.size dir))
+  ]
+
+let run env =
+  let fs = Stdenv.fs env in
+  let random = Stdenv.secure_random env in
+  let clock = Stdenv.clock env in
+  run_bench ~n:20 ~levels:4 ~fs ~random ~clock
+

bench/main.ml

@@ -10,6 +10,7 @@ let benchmarks = [
   "HTTP", Bench_http.run;
   "Eio_unix.Fd", Bench_fd.run;
   "File.stat", Bench_fstat.run;
+  "Path.Stat", Bench_stat.run;
 ]
 
 let usage_error () =

lib_eio/file.ml

@@ -4,59 +4,44 @@ module Unix_perm = struct
   type t = int
 end
 
-module Stat = struct
-  type kind = [
-    | `Unknown
-    | `Fifo
-    | `Character_special
-    | `Directory
-    | `Block_device
-    | `Regular_file
-    | `Symbolic_link
-    | `Socket
-  ]
-
-  let pp_kind ppf = function
-    | `Unknown -> Fmt.string ppf "unknown"
-    | `Fifo -> Fmt.string ppf "fifo"
-    | `Character_special -> Fmt.string ppf "character special file"
-    | `Directory -> Fmt.string ppf "directory"
-    | `Block_device -> Fmt.string ppf "block device"
-    | `Regular_file -> Fmt.string ppf "regular file"
-    | `Symbolic_link -> Fmt.string ppf "symbolic link"
-    | `Socket -> Fmt.string ppf "socket"
-
-  type t = {
-    dev : Int64.t;
-    ino : Int64.t;
-    kind : kind;
-    perm : Unix_perm.t;
-    nlink : Int64.t;
-    uid : Int64.t;
-    gid : Int64.t;
-    rdev : Int64.t;
-    size : Optint.Int63.t;
-    atime : float;
-    mtime : float;
-    ctime : float;
-  }
-
-  let pp ppf t =
-    Fmt.record [
-      Fmt.field "dev" (fun t -> t.dev) Fmt.int64;
-      Fmt.field "ino" (fun t -> t.ino) Fmt.int64;
-      Fmt.field "kind" (fun t -> t.kind) pp_kind;
-      Fmt.field "perm" (fun t -> t.perm) (fun ppf i -> Fmt.pf ppf "0o%o" i);
-      Fmt.field "nlink" (fun t -> t.nlink) Fmt.int64;
-      Fmt.field "uid" (fun t -> t.uid) Fmt.int64;
-      Fmt.field "gid" (fun t -> t.gid) Fmt.int64;
-      Fmt.field "rdev" (fun t -> t.rdev) Fmt.int64;
-      Fmt.field "size" (fun t -> t.size) Optint.Int63.pp;
-      Fmt.field "atime" (fun t -> t.atime) Fmt.float;
-      Fmt.field "mtime" (fun t -> t.mtime) Fmt.float;
-      Fmt.field "ctime" (fun t -> t.ctime) Fmt.float;
-    ] ppf t 
-end
+type kind = [
+  | `Unknown
+  | `Fifo
+  | `Character_special
+  | `Directory
+  | `Block_device
+  | `Regular_file
+  | `Symbolic_link
+  | `Socket
+]
+
+let pp_kind ppf = function
+  | `Unknown -> Fmt.string ppf "unknown"
+  | `Fifo -> Fmt.string ppf "fifo"
+  | `Character_special -> Fmt.string ppf "character special file"
+  | `Directory -> Fmt.string ppf "directory"
+  | `Block_device -> Fmt.string ppf "block device"
+  | `Regular_file -> Fmt.string ppf "regular file"
+  | `Symbolic_link -> Fmt.string ppf "symbolic link"
+  | `Socket -> Fmt.string ppf "socket"
+
+type 'a stat =
+  | Dev : int64 stat
+  | Ino : int64 stat
+  | Kind : kind stat
+  | Perm : int stat
+  | Nlink : int64 stat
+  | Uid : int64 stat
+  | Gid : int64 stat
+  | Rdev : int64 stat
+  | Size : int64 stat
+  | Atime : float stat
+  | Ctime : float stat
+  | Mtime : float stat
+
+type ('a, 'ty) stats =
+  | [] : ('ty, 'ty) stats
+  | (::) : 'a stat * ('b, 'ty) stats -> ('a -> 'b, 'ty) stats
 
 type ro_ty = [`File | Flow.source_ty | Resource.close_ty]
 
@@ -71,7 +56,7 @@ module Pi = struct
     include Flow.Pi.SOURCE
 
     val pread : t -> file_offset:Optint.Int63.t -> Cstruct.t list -> int
-    val stat : t -> Stat.t
+    val stat : 'a 'b . t -> ('a, 'b) stats -> 'a -> 'b
     val close : t -> unit
   end
 
@@ -105,7 +90,14 @@ let stat (Resource.T (t, ops)) =
   let module X = (val (Resource.get ops Pi.Read)) in
   X.stat t
 
-let size t = (stat t).size
+let kind t  = stat t [Kind] Fun.id
+let perm t  = stat t [Perm] Fun.id
+let uid t   = stat t [Uid] Fun.id
+let gid t   = stat t [Gid] Fun.id
+let size t  = stat t [Size] (fun s -> Optint.Int63.of_int64 s)
+let atime t = stat t [Atime] Fun.id
+let ctime t = stat t [Ctime] Fun.id
+let mtime t = stat t [Mtime] Fun.id
 
 let pread (Resource.T (t, ops)) ~file_offset bufs =
   let module X = (val (Resource.get ops Pi.Read)) in