Streaming serialize and deserialize packets

[dtolnay]

From IRC:

<lu_zero> I have some multimedia format I want to write down in a sort of readable form, so I have some global information that can fit a json itself and then for each packet/frame I'd have 1 json each
<lu_zero> initially I wanted to assemble the array of packets/frames as they come but looks not so plausible
<lu_zero> so I'd just concatenate the multiple json (and that works)
<lu_zero> but then I couldn't find a way to tell serde on the deserializing side that's ok to have trailing data

[dtolnay]

Streaming serialization and deserialization of concatenated JSON objects

#[macro_use]
extern crate serde_derive;

extern crate serde;
extern crate serde_json;

use serde::{Serialize, Deserialize};

#[derive(Serialize, Deserialize, Debug)]
struct Index {
    len: usize,
    name: String,
}

#[derive(Serialize, Deserialize, Debug)]
struct Packet {
    name: String,
    data: Vec<u8>,
}

fn main() {
    try_main().unwrap();
}

fn try_main() -> Result<(), serde_json::Error> {
    let mut out = Vec::new();

    {
        let mut ser = serde_json::Serializer::new(&mut out);
        let npackets = 4;
        Index { len: npackets, name: "index".to_owned() }.serialize(&mut ser)?;
        for i in 0..npackets {
            Packet { name: "packet".to_owned(), data: vec![i as u8] }.serialize(&mut ser)?;
        }
    }

    println!("{}", String::from_utf8_lossy(&out));

    {
        let mut de = serde_json::Deserializer::from_slice(&out);
        let index = Index::deserialize(&mut de)?;
        println!("{:?}", index);
        for _ in 0..index.len {
            let packet = Packet::deserialize(&mut de)?;
            println!("{:?}", packet);
        }
    }

    Ok(())
}

[dtolnay]

Streaming serialization and deserialization of a single top-level array

#[macro_use]
extern crate serde_derive;

extern crate serde;
extern crate serde_json;

use serde::ser::{Serializer, SerializeSeq};
use serde::de::{self, Deserializer, Visitor, SeqAccess};

use std::fmt;
use std::io::{self, Write};

#[derive(Serialize, Deserialize, Debug)]
struct Index {
    len: usize,
    name: String,
}

#[derive(Serialize, Deserialize, Debug)]
struct Packet {
    name: String,
    data: Vec<u8>,
}

fn main() {
    try_main().unwrap();
}

fn try_main() -> Result<(), serde_json::Error> {
    let mut out = Vec::new();

    {
        let mut ser = serde_json::Serializer::pretty(&mut out);
        let npackets = 4;
        let mut seq = ser.serialize_seq(Some(1 + npackets as usize))?; // or None if unknown
        seq.serialize_element(&Index { len: npackets, name: "index".to_owned() })?;
        for i in 0..npackets {
            seq.serialize_element(&Packet { name: "packet".to_owned(), data: vec![i as u8] })?;
        }
        seq.end()?;
    }

    println!("{}", String::from_utf8_lossy(&out));

    {
        let mut de = serde_json::Deserializer::from_slice(&out);
        let action = DebugPackets { out: io::stdout() };
        de.deserialize_seq(action)?;
    }

    Ok(())
}

struct DebugPackets<W> {
    out: W,
}

impl<'de, W> Visitor<'de> for DebugPackets<W>
    where W: Write
{
    type Value = ();

    fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
        formatter.write_str("array of index + packets")
    }

    fn visit_seq<A>(mut self, mut seq: A) -> Result<Self::Value, A::Error>
        where A: SeqAccess<'de>
    {
        let index = match seq.next_element::<Index>()? {
            Some(index) => index,
            None => {
                return Err(de::Error::custom("index was missing"));
            }
        };
        let _ = writeln!(self.out, "{:?}", index);

        for _ in 0..index.len {
            match seq.next_element::<Packet>()? {
                Some(packet) => {
                    let _ = writeln!(self.out, "{:?}", packet);
                }
                None => {
                    return Err(de::Error::custom("fewer packets than expected"));
                }
            }
        }

        Ok(())
    }
}

[rofrol]

Same with string and without index:

#[macro_use]
extern crate serde_derive;

extern crate serde;
extern crate serde_json;

use serde::de::{self, Deserializer, SeqAccess, Visitor};

use std::fmt;
use std::io::{self, Write};

#[derive(Serialize, Deserialize, Debug)]
struct Packet {
    name: String,
    data: Vec<u8>,
}

fn main() -> std::io::Result<()> {
    let out = r#"
[
  {
    "name": "packet",
    "data": [
      0
    ]
  },
  {
    "name": "packet",
    "data": [
      1
    ]
  },
  {
    "name": "packet",
    "data": [
      2
    ]
  },
  {
    "name": "packet",
    "data": [
      3
    ]
  }
]
"#;

    {
        let mut de = serde_json::Deserializer::from_str(&out);
        let action = DebugPackets { out: io::stdout() };
        de.deserialize_seq(action)?;
    }

    Ok(())
}

struct DebugPackets<W> {
    out: W,
}

impl<'de, W> Visitor<'de> for DebugPackets<W>
where
    W: Write,
{
    type Value = ();

    fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
        formatter.write_str("array of packets")
    }

    fn visit_seq<A>(mut self, mut seq: A) -> Result<Self::Value, A::Error>
    where
        A: SeqAccess<'de>,
    {
        while let Some(packet) = seq.next_element::<Packet>()? {
            let _ = writeln!(self.out, "{:?}", packet);
        }
        Ok(())
    }
}