Stream to Seq

CHANGES.md

@@ -25,6 +25,8 @@
 
 - The function `to_file` now adds a newline at the end of the generated file. An
   optional argument allows to return to the original behaviour (#124, @panglesd)
+- The `stream_from_*` and `stream_to_*` functions now use a `Seq.t` instead of a
+  `Stream.t`, and they are renamed into `seq_from_*` and `seq_to_*` (@gasche, #131).
 
 ### Fix
 

bench/bench.ml

@@ -25,6 +25,13 @@ let large_string_list =
   in
   `List strings
 
+let streamable_string =
+  let buf = Buffer.create (large * 100) in
+  for i = 1 to large do
+    Printf.bprintf buf "%d\n" i
+  done;
+  Buffer.contents buf
+
 let main () =
   Command.run (Bench.make_command [
     Bench.Test.create ~name:"JSON reading" (fun () ->
@@ -46,6 +53,13 @@ let main () =
     Bench.Test.create ~name:"JSON writing assoc to channel" (fun () ->
       Out_channel.with_file "/dev/null" ~f:(fun oc ->
       ignore (Yojson.Safe.to_channel oc large_int_assoc)));
+    begin
+      let buf = Buffer.create 1000 in
+      Bench.Test.create ~name:"JSON seq roundtrip" (fun () ->
+        let stream = Yojson.Safe.seq_from_string ~buf streamable_string in
+        ignore (Yojson.Safe.seq_to_string ~buf stream)
+      )
+    end;
   ])
 
 let () =

bin/ydump.ml

@@ -46,7 +46,7 @@ let polycat write_one streaming in_file out_file =
   in
   try
     if streaming then
-      Stream.iter (write_one oc) (Yojson.Safe.stream_from_channel ~fname ic)
+      Seq.iter (write_one oc) (Yojson.Safe.seq_from_channel ~fname ic)
     else
       write_one oc (Yojson.Safe.from_channel ~fname ic);
     finally ();

lib/dune

@@ -34,4 +34,5 @@
  (public_name yojson)
  (modules yojson)
  (synopsis "JSON parsing and printing")
+ (libraries seq)
  (flags (-safe-string)))

lib/read.mli

@@ -71,82 +71,82 @@ val from_lexbuf :
       is false and indicates that only JSON whitespace can be found between
       the end of the JSON value and the end of the input. *)
 
-val stream_from_string :
+val seq_from_string :
   ?buf:Buffer.t ->
   ?fname:string ->
   ?lnum:int ->
-  string -> t Stream.t
+  string -> t Seq.t
   (** Input a sequence of JSON values from a string.
       Whitespace between JSON values is fine but not required.
       See [from_string] for the meaning of the optional arguments. *)
 
-val stream_from_channel :
+val seq_from_channel :
   ?buf:Buffer.t ->
   ?fin:(unit -> unit) ->
   ?fname:string ->
   ?lnum:int ->
-  in_channel -> t Stream.t
+  in_channel -> t Seq.t
   (** Input a sequence of JSON values from a channel.
       Whitespace between JSON values is fine but not required.
       @param fin finalization function executed once when the end of the
-      stream is reached either because there is no more input or because
+      sequence is reached either because there is no more input or because
       the input could not be parsed, raising an exception.
       @raise Finally When the parsing and the finalizer both raised, [Finally (exn, fin_exn)]
       is raised, [exn] being the parsing exception and [fin_exn] the finalizer one.
 
       See [from_string] for the meaning of the other optional arguments. *)
 
-val stream_from_file :
+val seq_from_file :
   ?buf:Buffer.t ->
   ?fname:string ->
   ?lnum:int ->
-  string -> t Stream.t
+  string -> t Seq.t
   (** Input a sequence of JSON values from a file.
       Whitespace between JSON values is fine but not required.
 
       See [from_string] for the meaning of the optional arguments. *)
 
-val stream_from_lexbuf :
+val seq_from_lexbuf :
   lexer_state ->
   ?fin:(unit -> unit) ->
-  Lexing.lexbuf -> t Stream.t
+  Lexing.lexbuf -> t Seq.t
   (** Input a sequence of JSON values from a lexbuf.
       A valid initial [lexer_state] can be created with [init_lexer].
       Whitespace between JSON values is fine but not required.
       @raise Finally When the parsing and the finalizer both raised, [Finally (exn, fin_exn)]
       is raised, [exn] being the parsing exception and [fin_exn] the finalizer one.
 
-      See [stream_from_channel] for the meaning of the optional [fin]
+      See [seq_from_channel] for the meaning of the optional [fin]
       argument. *)
 
 
 type json_line = [ `Json of t | `Exn of exn ]
     (** The type of values resulting from a parsing attempt of a JSON value. *)
 
-val linestream_from_channel :
+val lineseq_from_channel :
   ?buf:Buffer.t ->
   ?fin:(unit -> unit) ->
   ?fname:string ->
   ?lnum:int ->
-  in_channel -> json_line Stream.t
+  in_channel -> json_line Seq.t
   (** Input a sequence of JSON values, one per line, from a channel.
       Exceptions raised when reading malformed lines are caught
       and represented using [`Exn].
 
-      See [stream_from_channel] for the meaning of the optional [fin]
+      See [seq_from_channel] for the meaning of the optional [fin]
       argument.
       See [from_string] for the meaning of the other optional arguments. *)
 
-val linestream_from_file :
+val lineseq_from_file :
   ?buf:Buffer.t ->
   ?fname:string ->
   ?lnum:int ->
-  string -> json_line Stream.t
+  string -> json_line Seq.t
   (** Input a sequence of JSON values, one per line, from a file.
       Exceptions raised when reading malformed lines are caught
       and represented using [`Exn].
 
-      See [stream_from_channel] for the meaning of the optional [fin]
+      See [seq_from_channel] for the meaning of the optional [fin]
       argument.
       See [from_string] for the meaning of the other optional arguments. *)
 

lib/read.mll

@@ -1143,30 +1143,30 @@ and junk = parse
 
   exception Finally of exn * exn
 
-  let stream_from_lexbuf v ?(fin = fun () -> ()) lexbuf =
+  let seq_from_lexbuf v ?(fin = fun () -> ()) lexbuf =
     let stream = Some true in
-    let f i =
-      try Some (from_lexbuf v ?stream lexbuf)
+    let rec f () =
+      try Seq.Cons (from_lexbuf v ?stream lexbuf, f)
       with
           End_of_input ->
             fin ();
-            None
+            Seq.Nil
         | e ->
             (try fin () with fin_e -> raise (Finally (e, fin_e)));
             raise e
     in
-    Stream.from f
+    f
 
-  let stream_from_string ?buf ?fname ?lnum s =
+  let seq_from_string ?buf ?fname ?lnum s =
     let v = init_lexer ?buf ?fname ?lnum () in
-    stream_from_lexbuf v (Lexing.from_string s)
+    seq_from_lexbuf v (Lexing.from_string s)
 
-  let stream_from_channel ?buf ?fin ?fname ?lnum ic =
+  let seq_from_channel ?buf ?fin ?fname ?lnum ic =
     let lexbuf = Lexing.from_channel ic in
     let v = init_lexer ?buf ?fname ?lnum () in
-    stream_from_lexbuf v ?fin lexbuf
+    seq_from_lexbuf v ?fin lexbuf
 
-  let stream_from_file ?buf ?fname ?lnum file =
+  let seq_from_file ?buf ?fname ?lnum file =
     let ic = open_in file in
     let fin () = close_in ic in
     let fname =
@@ -1176,37 +1176,36 @@ and junk = parse
     in
     let lexbuf = Lexing.from_channel ic in
     let v = init_lexer ?buf ?fname ?lnum () in
-    stream_from_lexbuf v ~fin lexbuf
+    seq_from_lexbuf v ~fin lexbuf
 
   type json_line = [ `Json of t | `Exn of exn ]
 
-  let linestream_from_channel
+  let lineseq_from_channel
       ?buf ?(fin = fun () -> ()) ?fname ?lnum:(lnum0 = 1) ic =
     let buf =
       match buf with
           None -> Some (Buffer.create 256)
         | Some _ -> buf
     in
-    let f i =
+    let rec f lnum = fun () ->
       try
         let line = input_line ic in
-        let lnum = lnum0 + i in
-        Some (`Json (from_string ?buf ?fname ~lnum line))
+        Seq.Cons (`Json (from_string ?buf ?fname ~lnum line), f (lnum + 1))
       with
-          End_of_file -> fin (); None
-        | e -> Some (`Exn e)
+          End_of_file -> fin (); Seq.Nil
+        | e -> Seq.Cons (`Exn e, f (lnum + 1))
     in
-    Stream.from f
+    f lnum0
 
-  let linestream_from_file ?buf ?fname ?lnum file =
+  let lineseq_from_file ?buf ?fname ?lnum file =
     let ic = open_in file in
     let fin () = close_in ic in
     let fname =
       match fname with
           None -> Some file
         | x -> x
     in
-    linestream_from_channel ?buf ~fin ?fname ?lnum ic
+    lineseq_from_channel ?buf ~fin ?fname ?lnum ic
 
   let prettify ?std s =
     pretty_to_string ?std (from_string s)

lib/write.ml

@@ -467,38 +467,38 @@ let to_file ?len ?std ?(newline = true) file x =
     close_out_noerr oc;
     raise e
 
-let stream_to_buffer ?std ob st =
-  Stream.iter (to_buffer ?std ob) st
+let seq_to_buffer ?std ob st =
+  Seq.iter (to_buffer ?std ob) st
 
-let stream_to_string ?buf ?(len = 256) ?std st =
+let seq_to_string ?buf ?(len = 256) ?std st =
   let ob =
     match buf with
         None -> Buffer.create len
       | Some ob ->
           Buffer.clear ob;
           ob
   in
-  stream_to_buffer ?std ob st;
+  seq_to_buffer ?std ob st;
   let s = Buffer.contents ob in
   Buffer.clear ob;
   s
 
-let stream_to_channel ?buf ?(len=2096) ?std oc st =
+let seq_to_channel ?buf ?(len=2096) ?std oc seq =
   let ob =
     match buf with
         None -> Buffer.create len
       | Some ob -> ob
   in
-  Stream.iter (fun json ->
+  Seq.iter (fun json ->
     to_buffer ?std ob json;
     Buffer.output_buffer oc ob;
     Buffer.clear ob;
-  ) st
+  ) seq
 
-let stream_to_file ?len ?std file st =
+let seq_to_file ?len ?std file st =
   let oc = open_out file in
   try
-    stream_to_channel ?len ?std oc st;
+    seq_to_channel ?len ?std oc st;
     close_out oc
   with e ->
     close_out_noerr oc;

lib/write.mli

@@ -61,36 +61,36 @@ val to_buffer :
   (** Write a compact JSON value to an existing buffer.
       See [to_string] for the role of the optional argument. *)
 
-val stream_to_string :
+val seq_to_string :
   ?buf:Buffer.t ->
   ?len:int ->
   ?std:bool ->
-  t Stream.t -> string
+  t Seq.t -> string
   (** Write a newline-separated sequence of compact one-line JSON values to
       a string.
       See [to_string] for the role of the optional arguments. *)
 
-val stream_to_channel :
+val seq_to_channel :
   ?buf:Buffer.t ->
   ?len:int ->
   ?std:bool ->
-  out_channel -> t Stream.t -> unit
+  out_channel -> t Seq.t -> unit
   (** Write a newline-separated sequence of compact one-line JSON values to
       a channel.
       See [to_channel] for the role of the optional arguments. *)
 
-val stream_to_file :
+val seq_to_file :
   ?len:int ->
   ?std:bool ->
-  string -> t Stream.t -> unit
+  string -> t Seq.t -> unit
   (** Write a newline-separated sequence of compact one-line JSON values to
       a file.
       See [to_string] for the role of the optional arguments. *)
 
-val stream_to_buffer :
+val seq_to_buffer :
   ?std:bool ->
   Buffer.t ->
-  t Stream.t -> unit
+  t Seq.t -> unit
   (** Write a newline-separated sequence of compact one-line JSON values to
       an existing buffer.
       See [to_string] for the role of the optional arguments. *)

test/test_write.ml

@@ -29,24 +29,29 @@ let to_file () =
   test ~newline:true ();
   test ~newline:false ()
 
-let stream_to_file () =
-  let output_file = Filename.temp_file "test_yojson_stream_to_file" ".json" in
+(* List.to_seq is not available on old OCaml versions. *)
+let rec list_to_seq = function
+  | [] -> (fun () -> Seq.Nil)
+  | x :: xs -> (fun () -> Seq.Cons (x, list_to_seq xs))
+
+let seq_to_file () =
+  let output_file = Filename.temp_file "test_yojson_seq_to_file" ".json" in
   let data = [`String "foo"; `String "bar"] in
-  Yojson.Safe.stream_to_file output_file (Stream.of_list data);
+  Yojson.Safe.seq_to_file output_file (list_to_seq data);
   let read_data =
-    let stream = Yojson.Safe.stream_from_file output_file in
+    let seq = Yojson.Safe.seq_from_file output_file in
     let acc = ref [] in
-    Stream.iter (fun v -> acc := v :: !acc) stream;
+    Seq.iter (fun v -> acc := v :: !acc) seq;
     List.rev !acc
   in
   Sys.remove output_file;
   if data <> read_data then
     (* TODO: it would be nice to use Alcotest.check,
        but we don't have a 'testable' instance for JSON values. *)
-    Alcotest.fail "stream_{to,from}_file roundtrip failure"
+    Alcotest.fail "seq_{to,from}_file roundtrip failure"
 
 let single_json = [
   "to_string", `Quick, to_string;
   "to_file", `Quick, to_file;
-  "stream_to_file", `Quick, stream_to_file;
+  "seq_to_file", `Quick, seq_to_file;
 ]

yojson.opam

@@ -18,6 +18,7 @@ depends: [
   "cppo" {build}
   "alcotest" {with-test & >= "0.8.5"}
   "odoc" {with-doc}
+  "seq" {>= "0.2.2"}
 ]
 synopsis:
   "Yojson is an optimized parsing and printing library for the JSON format"