From b0f3832f199032f6953cce8c705c874377f26f72 Mon Sep 17 00:00:00 2001 From: Patrick Ferris Date: Sat, 25 Mar 2023 21:28:14 +0000 Subject: [PATCH] Initial cross-platform subprocess support This also splits Eio_unix into separate modules. Co-authored-by: Thomas Leonard --- README.md | 51 +++++++++++ doc/prelude.ml | 14 +-- lib_eio/eio.ml | 2 + lib_eio/eio.mli | 11 +++ lib_eio/process.ml | 110 ++++++++++++++++++++++++ lib_eio/process.mli | 148 ++++++++++++++++++++++++++++++++ lib_eio/unix/eio_unix.ml | 41 ++------- lib_eio/unix/eio_unix.mli | 4 + lib_eio/unix/fork_action.mli | 4 +- lib_eio/unix/private.ml | 20 +++++ lib_eio/unix/process.ml | 104 +++++++++++++++++++++++ lib_eio/unix/process.mli | 47 +++++++++++ lib_eio/unix/resource.ml | 6 ++ lib_eio/unix/types.ml | 3 + lib_eio_linux/eio_linux.ml | 47 +++++++++++ lib_eio_posix/eio_posix.ml | 1 + lib_eio_posix/process.ml | 38 +++++++++ tests/process.md | 159 +++++++++++++++++++++++++++++++++++ 18 files changed, 771 insertions(+), 39 deletions(-) create mode 100644 lib_eio/process.ml create mode 100644 lib_eio/process.mli create mode 100644 lib_eio/unix/private.ml create mode 100644 lib_eio/unix/process.ml create mode 100644 lib_eio/unix/process.mli create mode 100644 lib_eio/unix/resource.ml create mode 100644 lib_eio/unix/types.ml create mode 100644 lib_eio_posix/process.ml create mode 100644 tests/process.md diff --git a/README.md b/README.md index 135694092..8232fbfda 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,7 @@ Eio replaces existing concurrency libraries such as Lwt * [Buffered Writing](#buffered-writing) * [Error Handling](#error-handling) * [Filesystem Access](#filesystem-access) +* [Subprocesses](#subprocesses) * [Time](#time) * [Multicore Support](#multicore-support) * [Synchronisation Tools](#synchronisation-tools) @@ -876,6 +877,55 @@ A program that operates on the current directory will probably want to use `cwd` whereas a program that accepts a path from the user will probably want to use `fs`, perhaps with `open_dir` to constrain all access to be within that directory. +## Subprocesses + +Spawning subprocesses can be done using the [Eio.Process][] module: + +```ocaml +# Eio_main.run @@ fun env -> + let proc_mgr = Eio.Stdenv.process_mgr env in + Eio.Process.run proc_mgr ["echo"; "hello"];; +hello +- : unit = () +``` + +There are various optional arguments for setting the process's current directory or connecting up the standard streams. +For example, we can use `tr` to convert some text to upper-case: + +```ocaml +# Eio_main.run @@ fun env -> + let proc_mgr = Eio.Stdenv.process_mgr env in + Eio.Process.run proc_mgr ["tr"; "a-z"; "A-Z"] + ~stdin:(Eio.Flow.string_source "One two three\n");; +ONE TWO THREE +- : unit = () +``` + +If you want to capture the output of a process, you can provide a suitable `Eio.Flow.sink` as the `stdout` argument, +or use the `parse_out` convenience wrapper: + +```ocaml +# Eio_main.run @@ fun env -> + let proc_mgr = Eio.Stdenv.process_mgr env in + Eio.Process.parse_out proc_mgr Eio.Buf_read.line ["echo"; "hello"];; +- : string = "hello" +``` + +All process functions either return the exit status or check that it was zero (success): + +```ocaml +# Eio_main.run @@ fun env -> + let proc_mgr = Eio.Stdenv.process_mgr env in + Eio.Process.parse_out proc_mgr Eio.Buf_read.take_all ["sh"; "-c"; "exit 3"];; +Exception: +Eio.Io Process Child_error Exited 3, + running command: sh -c "exit 3" +``` + +`Process.spawn` and `Process.await` give more control over the process's lifetime +and exit status, and `Eio_unix.Process` gives more control over passing file +descriptors (on systems that support them). + ## Time The standard environment provides a [clock][Eio.Time] with the usual POSIX time: @@ -1825,3 +1875,4 @@ Some background about the effects system can be found in: [kcas]: https://github.com/ocaml-multicore/kcas [Meio]: https://github.com/tarides/meio [Lambda Capabilities]: https://roscidus.com/blog/blog/2023/04/26/lambda-capabilities/ +[Eio.Process]: https://github.com/ocaml-multicore/eio/blob/main/lib_eio/process.ml diff --git a/doc/prelude.ml b/doc/prelude.ml index 602d196de..7f39e3d38 100644 --- a/doc/prelude.ml +++ b/doc/prelude.ml @@ -32,12 +32,14 @@ module Eio_main = struct let run fn = Eio_main.run @@ fun env -> fn @@ object - method net = env#net - method stdin = env#stdin - method stdout = env#stdout - method cwd = env#cwd - method domain_mgr = fake_domain_mgr - method clock = fake_clock env#clock + method net = env#net + method stdin = env#stdin + method stdout = env#stdout + method stderr = env#stderr + method cwd = env#cwd + method process_mgr = env#process_mgr + method domain_mgr = fake_domain_mgr + method clock = fake_clock env#clock end end diff --git a/lib_eio/eio.ml b/lib_eio/eio.ml index 79a858928..edf5a041f 100644 --- a/lib_eio/eio.ml +++ b/lib_eio/eio.ml @@ -23,6 +23,7 @@ module Flow = Flow module Buf_read = Buf_read module Buf_write = Buf_write module Net = Net +module Process = Process module Domain_manager = Domain_manager module Time = Time module File = File @@ -34,6 +35,7 @@ module Stdenv = struct let stdout (t : ) = t#stdout let stderr (t : ) = t#stderr let net (t : ) = t#net + let process_mgr (t : ) = t#process_mgr let domain_mgr (t : ) = t#domain_mgr let clock (t : ) = t#clock let mono_clock (t : ) = t#mono_clock diff --git a/lib_eio/eio.mli b/lib_eio/eio.mli index ea5ef365b..d53e35dc8 100644 --- a/lib_eio/eio.mli +++ b/lib_eio/eio.mli @@ -84,6 +84,9 @@ module Buf_write = Buf_write (** Networking. *) module Net = Net +(** Subprocesses. *) +module Process = Process + (** Parallel computation across multiple CPU cores. *) module Domain_manager : sig class virtual t : object @@ -208,6 +211,14 @@ module Stdenv : sig val net : -> 'a (** [net t] gives access to the process's network namespace. *) + (** {1 Processes } + + To use this, see {!Process}. + *) + + val process_mgr : -> 'a + (** [process_mgr t] allows you to run subprocesses. *) + (** {1 Domains (using multiple CPU cores)} To use this, see {!Domain_manager}. diff --git a/lib_eio/process.ml b/lib_eio/process.ml new file mode 100644 index 000000000..e3566f00f --- /dev/null +++ b/lib_eio/process.ml @@ -0,0 +1,110 @@ +type exit_status = [ + | `Exited of int + | `Signaled of int +] + +type status = [ exit_status | `Stopped of int ] + +let pp_status ppf = function + | `Exited i -> Format.fprintf ppf "Exited %i" i + | `Signaled i -> Format.fprintf ppf "Signalled %i" i + | `Stopped i -> Format.fprintf ppf "Stopped %i" i + +type error = + | Executable_not_found of string + | Child_error of exit_status + +type Exn.err += E of error + +let err e = Exn.create (E e) + +let () = + Exn.register_pp (fun f -> function + | E e -> + Fmt.string f "Process "; + begin match e with + | Executable_not_found e -> Fmt.pf f "Executable %S not found" e; + | Child_error e -> Fmt.pf f "Child_error %a" pp_status e; + end; + true + | _ -> false + ) + +class virtual t = object + method virtual pid : int + method virtual await : exit_status + method virtual signal : int -> unit +end + +let pid proc = proc#pid +let await proc = proc#await + +let await_exn proc = + match proc#await with + | `Exited 0 -> () + | status -> raise (err (Child_error status)) + +let signal proc = proc#signal + +class virtual mgr = object + method virtual pipe : + sw:Switch.t -> + * + + method virtual spawn : + sw:Switch.t -> + ?cwd:Fs.dir Path.t -> + ?stdin:Flow.source -> + ?stdout:Flow.sink -> + ?stderr:Flow.sink -> + ?env:string array -> + ?executable:string -> + string list -> + t +end + +let bad_char = function + | ' ' | '"' | '\'' | '\\' -> true + | c -> + let c = Char.code c in + c <= 32 || c >= 127 + +let pp_arg f x = + if x = "" || String.exists bad_char x then Fmt.(quote string) f x + else Fmt.string f x + +let pp_args = Fmt.hbox (Fmt.list ~sep:Fmt.sp pp_arg) + +let spawn ~sw (t:#mgr) ?cwd ?stdin ?stdout ?stderr ?env ?executable args = + t#spawn ~sw + ?cwd:(cwd :> Fs.dir Path.t option) + ?env + ?executable args + ?stdin:(stdin :> Flow.source option) + ?stdout:(stdout :> Flow.sink option) + ?stderr:(stderr :> Flow.sink option) + +let run (t:#mgr) ?cwd ?stdin ?stdout ?stderr ?env ?executable args = + Switch.run @@ fun sw -> + let child = spawn ~sw t ?cwd ?stdin ?stdout ?stderr ?env ?executable args in + match await child with + | `Exited 0 -> () + | status -> + let ex = err (Child_error status) in + raise (Exn.add_context ex "running command: %a" pp_args args) + +let pipe ~sw (t:#mgr) = t#pipe ~sw + +let parse_out (t:#mgr) parse ?cwd ?stdin ?stderr ?env ?executable args = + Switch.run @@ fun sw -> + let r, w = pipe t ~sw in + try + let child = spawn ~sw t ?cwd ?stdin ~stdout:w ?stderr ?env ?executable args in + Flow.close w; + let output = Buf_read.parse_exn parse r ~max_size:max_int in + Flow.close r; + await_exn child; + output + with Exn.Io _ as ex -> + let bt = Printexc.get_raw_backtrace () in + Exn.reraise_with_context ex bt "running command: %a" pp_args args diff --git a/lib_eio/process.mli b/lib_eio/process.mli new file mode 100644 index 000000000..2d23f7b36 --- /dev/null +++ b/lib_eio/process.mli @@ -0,0 +1,148 @@ +(** Example: + {[ + # Eio_main.run @@ fun env -> + let proc_mgr = Eio.Stdenv.process_mgr env in + Eio.Process.parse_out proc_mgr Eio.Buf_read.line ["echo"; "hello"] + ]} + *) + +(** {2 Status and error types} *) + +type exit_status = [ + | `Exited of int (** Process exited with the given return code. *) + | `Signaled of int (** Process was killed by the given signal. *) +] + +type status = [ + | exit_status + | `Stopped of int (** Process was stopped (paused) by the given signal. *) +] + +val pp_status : status Fmt.t + +type error = + | Executable_not_found of string (** The requested executable does not exist. *) + | Child_error of exit_status (** The child exited with an error status. *) + +type Exn.err += E of error + +val err : error -> exn +(** [err e] is [Eio.Exn.create (E e)] *) + +val pp_args : string list Fmt.t +(** Formats a list of arguments, quoting any that might cause confusion to the reader. + + This is intended for use in error messages and logging.*) + +(** {2 Processes} *) + +(** A child process. *) +class virtual t : object + method virtual pid : int + method virtual await : exit_status + method virtual signal : int -> unit +end + +val pid : #t -> int +(** [pid t] is the process ID of [t]. *) + +val await : #t -> exit_status +(** [await t] waits for subprocess [t] to exit and then reports the status. *) + +val await_exn : #t -> unit +(** Like {! await} except an exception is raised if the status is not [`Exited 0]. *) + +val signal : #t -> int -> unit +(** [signal t i] sends the signal [i] to process [t]. + + If the process has already exited then this does nothing + (it will not signal a different process, even if the PID has been reused). + + See {!Sys} for the signal numbers. *) + +(** {2 Spawning processes} *) + +class virtual mgr : object + method virtual pipe : + sw:Switch.t -> + * + + method virtual spawn : + sw:Switch.t -> + ?cwd:Fs.dir Path.t -> + ?stdin:Flow.source -> + ?stdout:Flow.sink -> + ?stderr:Flow.sink -> + ?env:string array -> + ?executable:string -> + string list -> + t +end +(** A process manager capable of spawning new processes. *) + +val spawn : + sw:Switch.t -> + #mgr -> + ?cwd:#Fs.dir Path.t -> + ?stdin:#Flow.source -> + ?stdout:#Flow.sink -> + ?stderr:#Flow.sink -> + ?env:string array -> + ?executable:string -> + string list -> t +(** [spawn ~sw mgr args] creates a new subprocess that is connected to the switch [sw]. + + The subprocess will be sent {! Sys.sigkill} when the switch is released. + + If the flows [stdin], [stdout] and [stderr] are not backed by file descriptors then + this also creates pipes and spawns fibers to copy the data as necessary. + If you need more control over file descriptors, see {!Eio_unix.Process}. + + @param cwd The current working directory of the process (default: same as parent process). + @param stdin The flow to attach to the process's standard input (default: same as parent process). + @param stdout A flow that the child process's standard output goes to (default: same as parent process). + @param stderr A flow that the child process's standard error goes to (default: same as parent process). + @param env The environment for the child (default: same as parent process). + @param executable The path of the executable to run. + If not given then the first item in [args] is used, + searching $PATH for it if necessary. *) + +val run : + #mgr -> + ?cwd:#Fs.dir Path.t -> + ?stdin:#Flow.source -> + ?stdout:#Flow.sink -> + ?stderr:#Flow.sink -> + ?env:string array -> + ?executable:string -> + string list -> unit +(** [run] does {!spawn} followed by {!await_exn}, with the advantage that if the process fails then + the error message includes the command that failed. + + Note: If [spawn] needed to create extra fibers to copy [stdin], etc, then it also waits for those to finish. *) + +val parse_out : + #mgr -> + 'a Buf_read.parser -> + ?cwd:#Fs.dir Path.t -> + ?stdin:#Flow.source -> + ?stderr:#Flow.sink -> + ?env:string array -> + ?executable:string -> + string list -> 'a +(** [parse_out mgr parser args] runs [args] and parses the child's stdout with [parser]. + + It also waits for the process to finish and checks its exit status is zero. + + Note that [parser] must consume the entire output of the process (like {!Buf_read.parse}). + + This is a convenience wrapper around {!run}, + and the optional arguments have the same meanings. *) + +(** {2 Pipes} *) + +val pipe : sw:Switch.t -> #mgr -> * +(** [pipe ~sw mgr] creates a pipe backed by the OS. + + The flows can be used by {!spawn} without the need for extra fibers to copy the data. + This can be used to connect multiple processes together. *) diff --git a/lib_eio/unix/eio_unix.ml b/lib_eio/unix/eio_unix.ml index da486a5eb..9243999e4 100644 --- a/lib_eio/unix/eio_unix.ml +++ b/lib_eio/unix/eio_unix.ml @@ -1,17 +1,14 @@ [@@@alert "-unstable"] -open Eio.Std - module Fd = Fd +module Resource = Resource +module Private = Private -module Resource = struct - type t = < fd : Fd.t > - - type _ Eio.Generic.ty += FD : Fd.t Eio.Generic.ty +include Types - let fd t = t#fd - let fd_opt t = Eio.Generic.probe t FD -end +let await_readable = Private.await_readable +let await_writable = Private.await_writable +let pipe = Private.pipe type Eio.Exn.Backend.t += Unix_error of Unix.error * string * string let () = @@ -20,27 +17,6 @@ let () = | _ -> false ) -type source = < Eio.Flow.source; Resource.t; Eio.Flow.close > -type sink = < Eio.Flow.sink; Resource.t; Eio.Flow.close > -type socket = < Eio.Flow.two_way; Resource.t; Eio.Flow.close > - -module Private = struct - type _ Effect.t += - | Await_readable : Unix.file_descr -> unit Effect.t - | Await_writable : Unix.file_descr -> unit Effect.t - | Get_monotonic_clock : Eio.Time.Mono.t Effect.t - | Socket_of_fd : Switch.t * bool * Unix.file_descr -> socket Effect.t - | Socketpair : Switch.t * Unix.socket_domain * Unix.socket_type * int -> (socket * socket) Effect.t - | Pipe : Switch.t -> (source * sink) Effect.t - - module Rcfd = Rcfd - - module Fork_action = Fork_action -end - -let await_readable fd = Effect.perform (Private.Await_readable fd) -let await_writable fd = Effect.perform (Private.Await_writable fd) - let sleep d = Eio.Time.Mono.sleep (Effect.perform Private.Get_monotonic_clock) d @@ -78,8 +54,6 @@ end let socketpair ~sw ?(domain=Unix.PF_UNIX) ?(ty=Unix.SOCK_STREAM) ?(protocol=0) () = Effect.perform (Private.Socketpair (sw, domain, ty, protocol)) -let pipe sw = Effect.perform (Private.Pipe sw) - module Ipaddr = struct let to_unix : _ Eio.Net.Ipaddr.t -> Unix.inet_addr = Obj.magic let of_unix : Unix.inet_addr -> _ Eio.Net.Ipaddr.t = Obj.magic @@ -98,6 +72,8 @@ let getnameinfo (sockaddr : Eio.Net.Sockaddr.t) = let Unix.{ni_hostname; ni_service} = Unix.getnameinfo sockaddr options in (ni_hostname, ni_service)) +module Process = Process + module Stdenv = struct type base = < stdin : source; @@ -105,6 +81,7 @@ module Stdenv = struct stderr : sink; net : Eio.Net.t; domain_mgr : Eio.Domain_manager.t; + process_mgr : Process.mgr; clock : Eio.Time.clock; mono_clock : Eio.Time.Mono.t; fs : Eio.Fs.dir Eio.Path.t; diff --git a/lib_eio/unix/eio_unix.mli b/lib_eio/unix/eio_unix.mli index 1a3b07049..65c852508 100644 --- a/lib_eio/unix/eio_unix.mli +++ b/lib_eio/unix/eio_unix.mli @@ -102,6 +102,9 @@ val pipe : Switch.t -> source * sink can be read from [src]. Note that, like all FDs created by Eio, they are both marked as close-on-exec by default. *) +module Process = Process +(** Spawning sub-processes with extra control. *) + (** The set of resources provided to a process on a Unix-compatible system. *) module Stdenv : sig type base = < @@ -110,6 +113,7 @@ module Stdenv : sig stderr : sink; net : Eio.Net.t; domain_mgr : Eio.Domain_manager.t; + process_mgr : Process.mgr; clock : Eio.Time.clock; mono_clock : Eio.Time.Mono.t; fs : Eio.Fs.dir Eio.Path.t; diff --git a/lib_eio/unix/fork_action.mli b/lib_eio/unix/fork_action.mli index 2c2615063..fc9357d4a 100644 --- a/lib_eio/unix/fork_action.mli +++ b/lib_eio/unix/fork_action.mli @@ -12,7 +12,9 @@ Therefore, the fork call and all child actions need to be written in C. This module provides some support code for doing that. Individual backends will wrap these actions with higher-level APIs and - can also add their own platform-specific actions *) + can also add their own platform-specific actions. + + @canonical Eio_unix.Private.Fork_action *) type fork_fn (** A C function, as defined in "include/fork_action.h". *) diff --git a/lib_eio/unix/private.ml b/lib_eio/unix/private.ml new file mode 100644 index 000000000..e403b3402 --- /dev/null +++ b/lib_eio/unix/private.ml @@ -0,0 +1,20 @@ +[@@@alert "-unstable"] + +open Eio.Std +open Types + +type _ Effect.t += + | Await_readable : Unix.file_descr -> unit Effect.t + | Await_writable : Unix.file_descr -> unit Effect.t + | Get_monotonic_clock : Eio.Time.Mono.t Effect.t + | Socket_of_fd : Switch.t * bool * Unix.file_descr -> socket Effect.t + | Socketpair : Switch.t * Unix.socket_domain * Unix.socket_type * int -> (socket * socket) Effect.t + | Pipe : Switch.t -> (source * sink) Effect.t + +let await_readable fd = Effect.perform (Await_readable fd) +let await_writable fd = Effect.perform (Await_writable fd) + +let pipe sw = Effect.perform (Pipe sw) + +module Rcfd = Rcfd +module Fork_action = Fork_action diff --git a/lib_eio/unix/process.ml b/lib_eio/unix/process.ml new file mode 100644 index 000000000..0e722465d --- /dev/null +++ b/lib_eio/unix/process.ml @@ -0,0 +1,104 @@ +open Eio.Std + +let resolve_program name = + if Filename.is_implicit name then ( + Sys.getenv_opt "PATH" + |> Option.value ~default:"/bin:/usr/bin" + |> String.split_on_char ':' + |> List.find_map (fun dir -> + let p = Filename.concat dir name in + if Sys.file_exists p then Some p else None + ) + ) else if Sys.file_exists name then ( + Some name + ) else None + +let read_of_fd ~sw ~default ~to_close = function + | None -> default + | Some f -> + match Resource.fd_opt f with + | Some fd -> fd + | None -> + let r, w = Private.pipe sw in + Fiber.fork ~sw (fun () -> + Eio.Flow.copy f w; + Eio.Flow.close w + ); + let r = Resource.fd r in + to_close := r :: !to_close; + r + +let write_of_fd ~sw ~default ~to_close = function + | None -> default + | Some f -> + match Resource.fd_opt f with + | Some fd -> fd + | None -> + let r, w = Private.pipe sw in + Fiber.fork ~sw (fun () -> + Eio.Flow.copy r f; + Eio.Flow.close r + ); + let w = Resource.fd w in + to_close := w :: !to_close; + w + +let with_close_list fn = + let to_close = ref [] in + let close () = + List.iter Fd.close !to_close + in + match fn to_close with + | x -> close (); x + | exception ex -> + let bt = Printexc.get_raw_backtrace () in + close (); + Printexc.raise_with_backtrace ex bt + +let get_executable ~args = function + | Some exe -> exe + | None -> + match args with + | [] -> invalid_arg "Arguments list is empty and no executable given!" + | (x :: _) -> + match resolve_program x with + | Some x -> x + | None -> raise (Eio.Process.err (Executable_not_found x)) + +let get_env = function + | Some e -> e + | None -> Unix.environment () + +class virtual mgr = object (self) + inherit Eio.Process.mgr + + method pipe ~sw = (Private.pipe sw :> * ) + + method virtual spawn_unix : + sw:Switch.t -> + ?cwd:Eio.Fs.dir Eio.Path.t -> + env:string array -> + fds:(int * Fd.t * Fork_action.blocking) list -> + executable:string -> + string list -> + Eio.Process.t + + method spawn ~sw ?cwd ?stdin ?stdout ?stderr ?env ?executable args = + let executable = get_executable executable ~args in + let env = get_env env in + with_close_list @@ fun to_close -> + let stdin_fd = read_of_fd ~sw stdin ~default:Fd.stdin ~to_close in + let stdout_fd = write_of_fd ~sw stdout ~default:Fd.stdout ~to_close in + let stderr_fd = write_of_fd ~sw stderr ~default:Fd.stderr ~to_close in + let fds = [ + 0, stdin_fd, `Blocking; + 1, stdout_fd, `Blocking; + 2, stderr_fd, `Blocking; + ] in + self#spawn_unix ~sw ?cwd ~env ~fds ~executable args +end + +let spawn_unix ~sw (mgr:#mgr) ?cwd ~fds ?env ?executable args = + let executable = get_executable executable ~args in + let env = get_env env in + mgr#spawn_unix ~sw ?cwd ~fds ~env ~executable args diff --git a/lib_eio/unix/process.mli b/lib_eio/unix/process.mli new file mode 100644 index 000000000..caeba014e --- /dev/null +++ b/lib_eio/unix/process.mli @@ -0,0 +1,47 @@ +(** This extends the {!Eio.Process} API with more control over file-descriptors. *) + +open Eio.Std + +class virtual mgr : object + inherit Eio.Process.mgr + + method pipe : + sw:Switch.t -> + * + + method virtual spawn_unix : + sw:Switch.t -> + ?cwd:Eio.Fs.dir Eio.Path.t -> + env:string array -> + fds:(int * Fd.t * Fork_action.blocking) list -> + executable:string -> + string list -> + Eio.Process.t + + method spawn : + sw:Switch.t -> + ?cwd:Eio.Fs.dir Eio.Path.t -> + ?stdin:Eio.Flow.source -> + ?stdout:Eio.Flow.sink -> + ?stderr:Eio.Flow.sink -> + ?env:string array -> + ?executable:string -> + string list -> + Eio.Process.t + (** The default implementation uses {!spawn_unix}. *) +end + +val spawn_unix : + sw:Switch.t -> + #mgr -> + ?cwd:Eio.Fs.dir Eio.Path.t -> + fds:(int * Fd.t * Fork_action.blocking) list -> + ?env:string array -> + ?executable:string -> + string list -> + Eio.Process.t +(** [spawn_unix ~sw mgr ~fds args] spawns a sub-process running the command [args]. + + The arguments are as for {!Eio.Process.spawn}, + except that it takes a list of FD mappings for {!Fork_action.inherit_fds} + directly, rather than just flows for the standard streams. *) diff --git a/lib_eio/unix/resource.ml b/lib_eio/unix/resource.ml new file mode 100644 index 000000000..1839596f6 --- /dev/null +++ b/lib_eio/unix/resource.ml @@ -0,0 +1,6 @@ +type t = < fd : Fd.t > + +type _ Eio.Generic.ty += FD : Fd.t Eio.Generic.ty + +let fd t = t#fd +let fd_opt t = Eio.Generic.probe t FD diff --git a/lib_eio/unix/types.ml b/lib_eio/unix/types.ml new file mode 100644 index 000000000..581313a71 --- /dev/null +++ b/lib_eio/unix/types.ml @@ -0,0 +1,3 @@ +type source = < Eio.Flow.source; Resource.t; Eio.Flow.close > +type sink = < Eio.Flow.sink; Resource.t; Eio.Flow.close > +type socket = < Eio.Flow.two_way; Resource.t; Eio.Flow.close > diff --git a/lib_eio_linux/eio_linux.ml b/lib_eio_linux/eio_linux.ml index 0799aec82..523056405 100644 --- a/lib_eio_linux/eio_linux.ml +++ b/lib_eio_linux/eio_linux.ml @@ -283,6 +283,52 @@ end type stdenv = Eio_unix.Stdenv.base +module Process = Low_level.Process + +let process proc : Eio.Process.t = object + method pid = Process.pid proc + + method await = + match Eio.Promise.await @@ Process.exit_status proc with + | Unix.WEXITED i -> `Exited i + | Unix.WSIGNALED i -> `Signaled i + | Unix.WSTOPPED _ -> assert false + + method signal i = Process.signal proc i +end + +(* fchdir wants just a directory FD, not an FD and a path like the *at functions. *) +let with_dir dir_fd path fn = + Switch.run @@ fun sw -> + Low_level.openat ~sw + ~seekable:false + ~access:`R + ~perm:0 + ~flags:Uring.Open_flags.(cloexec + path + directory) + dir_fd (if path = "" then "." else path) + |> fn + +let process_mgr = object + inherit Eio_unix.Process.mgr + + method spawn_unix ~sw ?cwd ~env ~fds ~executable args = + let actions = Process.Fork_action.[ + Eio_unix.Private.Fork_action.inherit_fds fds; + execve executable ~argv:(Array.of_list args) ~env + ] in + let with_actions cwd fn = match cwd with + | None -> fn actions + | Some (fd, s) -> + match get_dir_fd_opt fd with + | None -> Fmt.invalid_arg "cwd is not an OS directory!" + | Some dir_fd -> + with_dir dir_fd s @@ fun cwd -> + fn (Process.Fork_action.fchdir cwd :: actions) + in + with_actions cwd @@ fun actions -> + process (Process.spawn ~sw actions) +end + let domain_mgr ~run_event_loop = object inherit Eio.Domain_manager.t @@ -409,6 +455,7 @@ let stdenv ~run_event_loop = method stdout = stdout method stderr = stderr method net = net + method process_mgr = process_mgr method domain_mgr = domain_mgr ~run_event_loop method clock = clock method mono_clock = mono_clock diff --git a/lib_eio_posix/eio_posix.ml b/lib_eio_posix/eio_posix.ml index c9efe65e5..3a79320f3 100644 --- a/lib_eio_posix/eio_posix.ml +++ b/lib_eio_posix/eio_posix.ml @@ -33,6 +33,7 @@ let run main = method clock = Time.clock method mono_clock = Time.mono_clock method net = Net.v + method process_mgr = Process.v method domain_mgr = Domain_mgr.v method cwd = ((Fs.cwd, "") :> Eio.Fs.dir Eio.Path.t) method fs = ((Fs.fs, "") :> Eio.Fs.dir Eio.Path.t) diff --git a/lib_eio_posix/process.ml b/lib_eio_posix/process.ml new file mode 100644 index 000000000..94150731c --- /dev/null +++ b/lib_eio_posix/process.ml @@ -0,0 +1,38 @@ +open Eio.Std + +module Process = Low_level.Process + +let process proc : Eio.Process.t = object + method pid = Process.pid proc + + method await = + match Eio.Promise.await @@ Process.exit_status proc with + | Unix.WEXITED i -> `Exited i + | Unix.WSIGNALED i -> `Signaled i + | Unix.WSTOPPED _ -> assert false + + method signal i = Process.signal proc i +end + +let v = object + inherit Eio_unix.Process.mgr + + method spawn_unix ~sw ?cwd ~env ~fds ~executable args = + let actions = Process.Fork_action.[ + inherit_fds fds; + execve executable ~argv:(Array.of_list args) ~env + ] in + let with_actions cwd fn = match cwd with + | None -> fn actions + | Some ((dir, path) : Eio.Fs.dir Eio.Path.t) -> + match Eio.Generic.probe dir Fs.Posix_dir with + | None -> Fmt.invalid_arg "cwd is not an OS directory!" + | Some posix -> + posix#with_parent_dir path @@ fun dirfd s -> + Switch.run @@ fun launch_sw -> + let cwd = Low_level.openat ?dirfd ~sw:launch_sw ~mode:0 s Low_level.Open_flags.(rdonly + directory) in + fn (Process.Fork_action.fchdir cwd :: actions) + in + with_actions cwd @@ fun actions -> + process (Process.spawn ~sw actions) +end diff --git a/tests/process.md b/tests/process.md new file mode 100644 index 000000000..20ac63607 --- /dev/null +++ b/tests/process.md @@ -0,0 +1,159 @@ +# Setting up the environment + +```ocaml +# #require "eio_main";; +``` + +Creating some useful helper functions + +```ocaml +open Eio.Std + +module Flow = Eio.Flow +module Process = Eio.Process + +let ( / ) = Eio.Path.( / ) + +let run fn = + Eio_main.run @@ fun env -> + fn env#process_mgr env + +let check_signal expected = function + | `Signaled x when x = expected -> Ok () + | x -> Error x;; +``` + +Running a program as a subprocess: + +```ocaml +# run @@ fun mgr _env -> + Switch.run @@ fun sw -> + let t = Process.spawn ~sw mgr [ "echo"; "hello world" ] in + Process.await t;; +hello world +- : Process.exit_status = `Exited 0 +``` + +Stopping a subprocess works and checking the status waits and reports correctly: + +```ocaml +# run @@ fun mgr _env -> + Switch.run @@ fun sw -> + let t = Process.spawn ~sw mgr [ "sleep"; "10" ] in + Process.signal t Sys.sigkill; + Process.await t |> check_signal Sys.sigkill;; +- : (unit, Process.exit_status) result = Ok () +``` + +A switch will stop a process when it is released: + +```ocaml +# run @@ fun mgr env -> + let proc = Switch.run (fun sw -> Process.spawn ~sw mgr [ "sleep"; "10" ]) in + Process.await proc |> check_signal Sys.sigkill;; +- : (unit, Process.exit_status) result = Ok () +``` + +Passing in flows allows you to redirect the child process' stdout: + +```ocaml +# run @@ fun mgr env -> + let fs = Eio.Stdenv.fs env in + let filename = "process-test.txt" in + Eio.Path.(with_open_out ~create:(`Exclusive 0o600) (fs / filename)) @@ fun stdout -> + Process.run mgr ~stdout [ "echo"; "Hello" ]; + Eio.Path.(load (fs / filename));; +- : string = "Hello\n" +``` + +Piping data to and from the child: + +```ocaml +# run @@ fun mgr env -> + let stdin = Eio.Flow.string_source "one\ntwo\nthree\n" in + Process.parse_out mgr Eio.Buf_read.line ~stdin ["wc"; "-l"] |> String.trim;; +- : string = "3" +``` + +Spawning subprocesses in new domains works normally: + +```ocaml +# run @@ fun mgr env -> + Eio.Domain_manager.run env#domain_mgr @@ fun () -> + Process.run mgr [ "echo"; "Hello from another domain" ];; +Hello from another domain +- : unit = () +``` + +Calling `await_exit` multiple times on the same spawn just returns the status: + +```ocaml +# run @@ fun mgr env -> + Switch.run @@ fun sw -> + let t = Process.spawn ~sw mgr [ "echo"; "hello world" ] in + (Process.await t, Process.await t, Process.await t);; +hello world +- : Process.exit_status * Process.exit_status * Process.exit_status = +(`Exited 0, `Exited 0, `Exited 0) +``` + +Using a sink that is not backed by a file descriptor: + +```ocaml +# run @@ fun mgr env -> + let buf = Buffer.create 16 in + Eio.Process.run mgr ~stdout:(Flow.buffer_sink buf) [ "echo"; "Hello, world" ]; + Buffer.contents buf +- : string = "Hello, world\n" +``` + +Changing directory (unconfined): + +```ocaml +# run @@ fun mgr env -> + let root = env#fs / "/" in + Process.run mgr ~cwd:root [ "env"; "pwd" ];; +/ +- : unit = () +``` + +Changing directory (confined): + +```ocaml +# run @@ fun mgr env -> + let subdir = env#cwd / "proc-sub-dir" in + Eio.Path.mkdir subdir ~perm:0o700; + Eio.Path.with_open_dir subdir @@ fun subdir -> + Eio.Path.save (subdir / "test-cwd") "test-data" ~create:(`Exclusive 0o600); + Process.run mgr ~cwd:subdir [ "cat"; "test-cwd" ];; +test-data +- : unit = () +``` + +If a command fails, we get shown the arguments (quoted if necessary): + +```ocaml +# run @@ fun mgr env -> + Process.run mgr ["bash"; "-c"; "exit 3"; ""; "foo"];; +Exception: +Eio.Io Process Child_error Exited 3, + running command: bash -c "exit 3" "" foo +``` + +The default environment: + +```ocaml +# run @@ fun mgr env -> + Unix.putenv "DISPLAY" ":1"; + Process.parse_out mgr Eio.Buf_read.line ["sh"; "-c"; "echo $DISPLAY"];; +- : string = ":1" +``` + +A custom environment: + +```ocaml +# run @@ fun mgr env -> + let env = [| "DISPLAY=:2" |] in + Process.parse_out mgr Eio.Buf_read.line ["sh"; "-c"; "echo $DISPLAY"] ~env;; +- : string = ":2" +```