From 8e5b4f4ad28daeb2003262576bf81b8b7ddfa8f6 Mon Sep 17 00:00:00 2001 From: Vinnie Magro Date: Mon, 8 Jan 2024 16:16:47 -0800 Subject: [PATCH] [antlir2] import sendstream_parser crate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Summary: I wrote this upstream, now I need to actually use it in antlir2. I'm re-homing this crate here and will publish future revisions from the antlir2 github repository and will archive my own copy. Test Plan: ``` ❯ buck2 test fbcode//antlir/antlir2/sendstream_parser/... Buck UI: https://www.internalfb.com/buck2/bcf62163-969c-4806-9d97-511de66fd1ab Test UI: https://www.internalfb.com/intern/testinfra/testrun/14636698802964271 Note: Using experimental modern dice Network: Up: 2.9KiB Down: 5.7KiB (reSessionID-52f3c095-f2e3-41fe-a37e-2ad379a55a43) Jobs completed: 767. Time elapsed: 8.8s. Cache hits: 0%. Commands: 2 (cached: 0, remote: 0, local: 2) Tests finished: Pass 3. Fail 0. Fatal 0. Skip 0. Build failure 0 ``` Differential Revision: D52049042 fbshipit-source-id: aa414ab61cfd4ab8a036669cc1c8e709e2953409 --- antlir/antlir2/sendstream_parser/BUCK | 28 + antlir/antlir2/sendstream_parser/Cargo.toml | 32 + antlir/antlir2/sendstream_parser/README.md | 12 + antlir/antlir2/sendstream_parser/src/lib.rs | 692 ++++++++++++++++++ antlir/antlir2/sendstream_parser/src/ser.rs | 108 +++ .../antlir2/sendstream_parser/src/wire/cmd.rs | 391 ++++++++++ .../antlir2/sendstream_parser/src/wire/mod.rs | 43 ++ .../antlir2/sendstream_parser/src/wire/tlv.rs | 325 ++++++++ .../testdata/demo.sendstream | Bin 0 -> 320693 bytes .../sendstream_parser/testdata/demo.txt | 98 +++ .../testdata/make-sendstream.sh | 63 ++ 11 files changed, 1792 insertions(+) create mode 100644 antlir/antlir2/sendstream_parser/BUCK create mode 100644 antlir/antlir2/sendstream_parser/Cargo.toml create mode 100644 antlir/antlir2/sendstream_parser/README.md create mode 100644 antlir/antlir2/sendstream_parser/src/lib.rs create mode 100644 antlir/antlir2/sendstream_parser/src/ser.rs create mode 100644 antlir/antlir2/sendstream_parser/src/wire/cmd.rs create mode 100644 antlir/antlir2/sendstream_parser/src/wire/mod.rs create mode 100644 antlir/antlir2/sendstream_parser/src/wire/tlv.rs create mode 100644 antlir/antlir2/sendstream_parser/testdata/demo.sendstream create mode 100644 antlir/antlir2/sendstream_parser/testdata/demo.txt create mode 100755 antlir/antlir2/sendstream_parser/testdata/make-sendstream.sh diff --git a/antlir/antlir2/sendstream_parser/BUCK b/antlir/antlir2/sendstream_parser/BUCK new file mode 100644 index 00000000000..442d11fba96 --- /dev/null +++ b/antlir/antlir2/sendstream_parser/BUCK @@ -0,0 +1,28 @@ +load("//antlir/bzl:build_defs.bzl", "rust_library") + +oncall("antlir") + +rust_library( + name = "sendstream_parser", + srcs = glob(["src/**/*.rs"]), + autocargo = { + "cargo_target_config": { + "test": True, + }, + }, + features = ["serde"], + test_deps = [ + "similar-asserts", + ], + test_features = ["serde"], + test_srcs = glob(["testdata/*"]), + deps = [ + "derive_more", + "hex", + "nix", + "nom", + "serde", + "thiserror", + "uuid", + ], +) diff --git a/antlir/antlir2/sendstream_parser/Cargo.toml b/antlir/antlir2/sendstream_parser/Cargo.toml new file mode 100644 index 00000000000..894471975bb --- /dev/null +++ b/antlir/antlir2/sendstream_parser/Cargo.toml @@ -0,0 +1,32 @@ +# @generated by autocargo from //antlir/antlir2/sendstream_parser:[sendstream_parser,sendstream_parser-unittest] + +[package] +name = "sendstream_parser" +version = "0.0.0" +edition = "2021" +repository = "https://github.com/facebookincubator/antlir" +license = "MIT" + +[lib] +doctest = false + +[[test]] +name = "sendstream_parser" +path = "src/lib.rs" + +[dependencies] +derive_more = "0.99.17" +hex = "0.4.3" +nix = "0.25" +nom = "7.1" +serde = { version = "1.0.185", features = ["derive", "rc"], optional = true } +thiserror = "1.0.49" +uuid = { version = "1.2", features = ["serde", "v4", "v5", "v6", "v7", "v8"] } + +[dev-dependencies] +serde = { version = "1.0.185", features = ["derive", "rc"] } +similar-asserts = "1.4.2" + +[features] +default = ["serde", "serde"] +serde = [] diff --git a/antlir/antlir2/sendstream_parser/README.md b/antlir/antlir2/sendstream_parser/README.md new file mode 100644 index 00000000000..061d42483d3 --- /dev/null +++ b/antlir/antlir2/sendstream_parser/README.md @@ -0,0 +1,12 @@ +[![Workflow Status](https://github.com/vmagro/sendstream_utils/actions/workflows/main.yml/badge.svg)](https://github.com/vmagro/sendstream_utils/actions) +[![docs.rs](https://img.shields.io/docsrs/sendstream_parser)](https://docs.rs/sendstream_parser) +![Maintenance](https://img.shields.io/badge/maintenance-experimental-blue.svg) + +# sendstream_parser + +Rust parser for [BTRFS +Sendstreams](https://btrfs.readthedocs.io/en/latest/Send-receive.html) +which are created via +[btrfs-send](https://btrfs.readthedocs.io/en/latest/btrfs-send.html). + +License: MIT diff --git a/antlir/antlir2/sendstream_parser/src/lib.rs b/antlir/antlir2/sendstream_parser/src/lib.rs new file mode 100644 index 00000000000..4498a9c597c --- /dev/null +++ b/antlir/antlir2/sendstream_parser/src/lib.rs @@ -0,0 +1,692 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +//! Rust parser for [BTRFS +//! Sendstreams](https://btrfs.readthedocs.io/en/latest/Send-receive.html) +//! which are created via +//! [btrfs-send](https://btrfs.readthedocs.io/en/latest/btrfs-send.html). + +#![feature(macro_metavar_expr)] +#![deny(clippy::unwrap_used)] +#![deny(clippy::expect_used)] + +use std::borrow::Cow; +use std::ops::Deref; +use std::os::unix::prelude::PermissionsExt; +use std::path::Path; + +use derive_more::AsRef; +use derive_more::Deref; +use derive_more::From; +use nix::sys::stat::SFlag; +use nix::unistd::Gid; +use nix::unistd::Uid; +#[cfg(feature = "serde")] +use serde::Deserialize; +#[cfg(feature = "serde")] +use serde::Serialize; +use uuid::Uuid; + +#[cfg(feature = "serde")] +mod ser; +mod wire; + +#[derive(Debug, thiserror::Error)] +pub enum Error<'a> { + #[error("Parse error: {0:?}")] + Parse(nom::error::Error<&'a [u8]>), + #[error( + "Sendstream had unexpected trailing data. This probably means the parser is broken: {0:?}" + )] + TrailingData(Vec), + #[error("Sendstream is incomplete")] + Incomplete, +} + +impl<'a> From> for Error<'a> { + fn from(e: nom::error::Error<&'a [u8]>) -> Self { + Self::Parse(e) + } +} + +pub type Result<'a, R> = std::result::Result>; + +/// This is the main entrypoint of this crate. It provides access to the +/// sequence of [Command]s that make up this sendstream. +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr(feature = "serde", derive(Deserialize, Serialize))] +pub struct Sendstream<'a> { + #[cfg_attr(feature = "serde", serde(borrow))] + commands: Vec>, +} + +impl<'a> Sendstream<'a> { + pub fn commands(&self) -> &[Command<'a>] { + &self.commands + } + + pub fn into_commands(self) -> Vec> { + self.commands + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr(feature = "serde", derive(Deserialize, Serialize))] +#[cfg_attr(feature = "serde", serde(bound(deserialize = "'de: 'a")))] +#[cfg_attr(feature = "serde", serde(untagged))] +pub enum Command<'a> { + Chmod(Chmod<'a>), + Chown(Chown<'a>), + Clone(Clone<'a>), + End, + Link(Link<'a>), + Mkdir(Mkdir<'a>), + Mkfifo(Mkfifo<'a>), + Mkfile(Mkfile<'a>), + Mknod(Mknod<'a>), + Mksock(Mksock<'a>), + RemoveXattr(RemoveXattr<'a>), + Rename(Rename<'a>), + Rmdir(Rmdir<'a>), + SetXattr(SetXattr<'a>), + Snapshot(Snapshot<'a>), + Subvol(Subvol<'a>), + Symlink(Symlink<'a>), + Truncate(Truncate<'a>), + Unlink(Unlink<'a>), + UpdateExtent(UpdateExtent<'a>), + Utimes(Utimes<'a>), + Write(Write<'a>), +} + +impl<'a> Command<'a> { + /// Exposed for tests to ensure that the demo sendstream is exhaustive and + /// exercises all commands + #[cfg(test)] + pub(crate) fn command_type(&self) -> wire::cmd::CommandType { + match self { + Self::Chmod(_) => wire::cmd::CommandType::Chmod, + Self::Chown(_) => wire::cmd::CommandType::Chown, + Self::Clone(_) => wire::cmd::CommandType::Clone, + Self::End => wire::cmd::CommandType::End, + Self::Link(_) => wire::cmd::CommandType::Link, + Self::Mkdir(_) => wire::cmd::CommandType::Mkdir, + Self::Mkfifo(_) => wire::cmd::CommandType::Mkfifo, + Self::Mkfile(_) => wire::cmd::CommandType::Mkfile, + Self::Mknod(_) => wire::cmd::CommandType::Mknod, + Self::Mksock(_) => wire::cmd::CommandType::Mksock, + Self::RemoveXattr(_) => wire::cmd::CommandType::RemoveXattr, + Self::Rename(_) => wire::cmd::CommandType::Rename, + Self::Rmdir(_) => wire::cmd::CommandType::Rmdir, + Self::SetXattr(_) => wire::cmd::CommandType::SetXattr, + Self::Snapshot(_) => wire::cmd::CommandType::Snapshot, + Self::Subvol(_) => wire::cmd::CommandType::Subvol, + Self::Symlink(_) => wire::cmd::CommandType::Symlink, + Self::Truncate(_) => wire::cmd::CommandType::Truncate, + Self::Unlink(_) => wire::cmd::CommandType::Unlink, + Self::UpdateExtent(_) => wire::cmd::CommandType::UpdateExtent, + Self::Utimes(_) => wire::cmd::CommandType::Utimes, + Self::Write(_) => wire::cmd::CommandType::Write, + } + } +} + +macro_rules! from_cmd { + ($t:ident) => { + impl<'a> From<$t<'a>> for Command<'a> { + fn from(c: $t<'a>) -> Self { + Self::$t(c) + } + } + }; +} + +macro_rules! one_getter { + ($f:ident, $ft:ty, copy) => { + pub fn $f(&self) -> $ft { + self.$f + } + }; + ($f:ident, $ft:ty, borrow) => { + pub fn $f(&self) -> &$ft { + &self.$f + } + }; +} + +macro_rules! getters { + ($t:ident, [$(($f:ident, $ft:ident, $ref:tt)),+]) => { + impl<'a> $t<'a> { + $( + one_getter!($f, $ft, $ref); + )+ + } + }; +} + +/// Because the stream is emitted in inode order, not FS order, the destination +/// directory may not exist at the time that a creation command is emitted, so +/// it will end up with an opaque name that will end up getting renamed to the +/// final name later in the stream. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, AsRef)] +#[as_ref(forward)] +#[cfg_attr(feature = "serde", derive(Deserialize, Serialize))] +#[cfg_attr(feature = "serde", serde(transparent))] +pub struct TemporaryPath<'a>(#[cfg_attr(feature = "serde", serde(borrow))] pub(crate) &'a Path); + +impl<'a> TemporaryPath<'a> { + pub fn as_path(&self) -> &Path { + self.as_ref() + } +} + +impl<'a> Deref for TemporaryPath<'a> { + type Target = Path; + + #[inline] + fn deref(&self) -> &Self::Target { + self.0 + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord)] +#[cfg_attr(feature = "serde", derive(Deserialize, Serialize))] +#[cfg_attr(feature = "serde", serde(transparent))] +pub struct Ctransid(pub u64); + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr(feature = "serde", derive(Deserialize, Serialize))] +pub struct Subvol<'a> { + #[cfg_attr(feature = "serde", serde(borrow))] + pub(crate) path: &'a Path, + pub(crate) uuid: Uuid, + pub(crate) ctransid: Ctransid, +} +from_cmd!(Subvol); +getters! {Subvol, [(path, Path, borrow), (uuid, Uuid, copy), (ctransid, Ctransid, copy)]} + +#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, AsRef, Deref)] +#[cfg_attr(feature = "serde", derive(Deserialize, Serialize))] +#[cfg_attr(feature = "serde", serde(transparent))] +pub struct Mode(u32); + +impl Mode { + pub fn mode(self) -> nix::sys::stat::Mode { + nix::sys::stat::Mode::from_bits_truncate(self.0) + } + + pub fn permissions(self) -> std::fs::Permissions { + std::fs::Permissions::from_mode(self.0) + } + + pub fn file_type(self) -> SFlag { + SFlag::from_bits_truncate(self.0) + } +} + +impl std::fmt::Debug for Mode { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Mode") + .field("permissions", &self.permissions()) + .field("type", &self.file_type()) + .finish() + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr(feature = "serde", derive(Deserialize, Serialize))] +pub struct Chmod<'a> { + #[cfg_attr(feature = "serde", serde(borrow))] + pub(crate) path: &'a Path, + pub(crate) mode: Mode, +} +from_cmd!(Chmod); +getters! {Chmod, [(path, Path, borrow), (mode, Mode, copy)]} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr(feature = "serde", derive(Deserialize, Serialize))] +pub struct Chown<'a> { + #[cfg_attr(feature = "serde", serde(borrow))] + pub(crate) path: &'a Path, + #[cfg_attr(feature = "serde", serde(with = "crate::ser::uid"))] + pub(crate) uid: Uid, + #[cfg_attr(feature = "serde", serde(with = "crate::ser::gid"))] + pub(crate) gid: Gid, +} +from_cmd!(Chown); +getters! {Chown, [(path, Path, borrow), (uid, Uid, copy), (gid, Gid, copy)]} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, AsRef, Deref)] +#[cfg_attr(feature = "serde", derive(Deserialize, Serialize))] +#[cfg_attr(feature = "serde", serde(transparent))] +pub struct CloneLen(u64); + +impl CloneLen { + pub fn as_u64(self) -> u64 { + self.0 + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr(feature = "serde", derive(Deserialize, Serialize))] +pub struct Clone<'a> { + pub(crate) src_offset: FileOffset, + pub(crate) len: CloneLen, + #[cfg_attr(feature = "serde", serde(borrow))] + pub(crate) src_path: &'a Path, + pub(crate) uuid: Uuid, + pub(crate) ctransid: Ctransid, + #[cfg_attr(feature = "serde", serde(borrow))] + pub(crate) dst_path: &'a Path, + pub(crate) dst_offset: FileOffset, +} +from_cmd!(Clone); +getters! {Clone, [ + (src_offset, FileOffset, copy), + (len, CloneLen, copy), + (src_path, Path, borrow), + (uuid, Uuid, copy), + (ctransid, Ctransid, copy), + (dst_path, Path, borrow), + (dst_offset, FileOffset, copy) +]} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, AsRef)] +#[as_ref(forward)] +#[cfg_attr(feature = "serde", derive(Deserialize, Serialize))] +#[cfg_attr(feature = "serde", serde(transparent))] +pub struct LinkTarget<'a>(#[cfg_attr(feature = "serde", serde(borrow))] &'a Path); + +impl<'a> LinkTarget<'a> { + #[inline] + pub fn as_path(&self) -> &Path { + self.0 + } +} + +impl<'a> Deref for LinkTarget<'a> { + type Target = Path; + + #[inline] + fn deref(&self) -> &Self::Target { + self.0 + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr(feature = "serde", derive(Deserialize, Serialize))] +pub struct Link<'a> { + #[cfg_attr(feature = "serde", serde(borrow))] + pub(crate) link_name: &'a Path, + #[cfg_attr(feature = "serde", serde(borrow))] + pub(crate) target: LinkTarget<'a>, +} +from_cmd!(Link); +getters! {Link, [(link_name, Path, borrow), (target, LinkTarget, borrow)]} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr(feature = "serde", derive(Deserialize, Serialize))] +pub struct Mkdir<'a> { + #[cfg_attr(feature = "serde", serde(borrow))] + pub(crate) path: TemporaryPath<'a>, + pub(crate) ino: Ino, +} +from_cmd!(Mkdir); +getters! {Mkdir, [(path, TemporaryPath, borrow), (ino, Ino, copy)]} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, AsRef, Deref)] +#[cfg_attr(feature = "serde", derive(Deserialize, Serialize))] +#[cfg_attr(feature = "serde", serde(transparent))] +pub struct Rdev(u64); + +impl Rdev { + pub fn as_u64(self) -> u64 { + self.0 + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr(feature = "serde", derive(Deserialize, Serialize))] +pub struct Mkspecial<'a> { + #[cfg_attr(feature = "serde", serde(borrow))] + pub(crate) path: TemporaryPath<'a>, + pub(crate) ino: Ino, + pub(crate) rdev: Rdev, + pub(crate) mode: Mode, +} +getters! {Mkspecial, [ + (path, TemporaryPath, borrow), + (ino, Ino, copy), + (rdev, Rdev, copy), + (mode, Mode, copy) +]} + +macro_rules! special { + ($t:ident) => { + #[derive(Debug, Clone, PartialEq, Eq, AsRef, Deref)] + #[cfg_attr(feature = "serde", derive(Deserialize, Serialize))] + #[cfg_attr(feature = "serde", serde(transparent))] + #[repr(transparent)] + pub struct $t<'a>(#[cfg_attr(feature = "serde", serde(borrow))] Mkspecial<'a>); + from_cmd!($t); + }; +} +special!(Mkfifo); +special!(Mknod); +special!(Mksock); + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr(feature = "serde", derive(Deserialize, Serialize))] +pub struct Mkfile<'a> { + #[cfg_attr(feature = "serde", serde(borrow))] + pub(crate) path: TemporaryPath<'a>, + pub(crate) ino: Ino, +} +from_cmd!(Mkfile); +getters! {Mkfile, [(path, TemporaryPath, borrow), (ino, Ino, copy)]} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr(feature = "serde", derive(Deserialize, Serialize))] +pub struct RemoveXattr<'a> { + #[cfg_attr(feature = "serde", serde(borrow))] + pub(crate) path: &'a Path, + #[cfg_attr(feature = "serde", serde(borrow))] + pub(crate) name: XattrName<'a>, +} +from_cmd!(RemoveXattr); +getters! {RemoveXattr, [(path, Path, borrow), (name, XattrName, borrow)]} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr(feature = "serde", derive(Deserialize, Serialize))] +pub struct Rename<'a> { + #[cfg_attr(feature = "serde", serde(borrow))] + pub(crate) from: &'a Path, + #[cfg_attr(feature = "serde", serde(borrow))] + pub(crate) to: &'a Path, +} +from_cmd!(Rename); +getters! {Rename, [(from, Path, borrow), (to, Path, borrow)]} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr(feature = "serde", derive(Deserialize, Serialize))] +pub struct Rmdir<'a> { + #[cfg_attr(feature = "serde", serde(borrow))] + pub(crate) path: &'a Path, +} +from_cmd!(Rmdir); +getters! {Rmdir, [(path, Path, borrow)]} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr(feature = "serde", derive(Deserialize, Serialize))] +pub struct Symlink<'a> { + #[cfg_attr(feature = "serde", serde(borrow))] + pub(crate) link_name: &'a Path, + pub(crate) ino: Ino, + pub(crate) target: LinkTarget<'a>, +} +from_cmd!(Symlink); +getters! {Symlink, [(link_name, Path, borrow), (ino, Ino, copy), (target, LinkTarget, borrow)]} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, AsRef, From)] +#[as_ref(forward)] +#[from(forward)] +pub struct XattrName<'a>(&'a [u8]); + +impl<'a> XattrName<'a> { + #[inline] + pub fn as_slice(&self) -> &[u8] { + self.0 + } +} + +impl<'a> Deref for XattrName<'a> { + type Target = [u8]; + + #[inline] + fn deref(&self) -> &Self::Target { + self.0 + } +} + +#[derive(Debug, Clone, PartialEq, Eq, AsRef, From)] +#[as_ref(forward)] +#[from(forward)] +pub struct XattrData<'a>(&'a [u8]); + +impl<'a> XattrData<'a> { + #[inline] + pub fn as_slice(&self) -> &[u8] { + self.0 + } +} + +impl<'a> Deref for XattrData<'a> { + type Target = [u8]; + + #[inline] + fn deref(&self) -> &Self::Target { + self.0 + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr(feature = "serde", derive(Deserialize, Serialize))] +pub struct SetXattr<'a> { + #[cfg_attr(feature = "serde", serde(borrow))] + pub(crate) path: &'a Path, + pub(crate) name: XattrName<'a>, + pub(crate) data: XattrData<'a>, +} +from_cmd!(SetXattr); +getters! {SetXattr, [(path, Path, borrow), (name, XattrName, borrow), (data, XattrData, borrow)]} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr(feature = "serde", derive(Deserialize, Serialize))] +pub struct Snapshot<'a> { + #[cfg_attr(feature = "serde", serde(borrow))] + pub(crate) path: &'a Path, + pub(crate) uuid: Uuid, + pub(crate) ctransid: Ctransid, + pub(crate) clone_uuid: Uuid, + pub(crate) clone_ctransid: Ctransid, +} +from_cmd!(Snapshot); +getters! {Snapshot, [ + (path, Path, borrow), + (uuid, Uuid, copy), + (ctransid, Ctransid, copy), + (clone_uuid, Uuid, copy), + (clone_ctransid, Ctransid, copy) +]} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr(feature = "serde", derive(Deserialize, Serialize))] +pub struct Truncate<'a> { + #[cfg_attr(feature = "serde", serde(borrow))] + pub(crate) path: &'a Path, + pub(crate) size: u64, +} +from_cmd!(Truncate); +getters! {Truncate, [(path, Path, borrow), (size, u64, copy)]} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr(feature = "serde", derive(Deserialize, Serialize))] +pub struct Unlink<'a> { + #[cfg_attr(feature = "serde", serde(borrow))] + pub(crate) path: &'a Path, +} +from_cmd!(Unlink); +getters! {Unlink, [(path, Path, borrow)]} + +#[allow(clippy::len_without_is_empty)] +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr(feature = "serde", derive(Deserialize, Serialize))] +pub struct UpdateExtent<'a> { + #[cfg_attr(feature = "serde", serde(borrow))] + pub(crate) path: &'a Path, + pub(crate) offset: FileOffset, + pub(crate) len: u64, +} +from_cmd!(UpdateExtent); +getters! {UpdateExtent, [(path, Path, borrow), (offset, FileOffset, copy), (len, u64, copy)]} + +macro_rules! time_alias { + ($a:ident) => { + #[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, AsRef, Deref)] + #[cfg_attr(feature = "serde", derive(Deserialize, Serialize))] + #[cfg_attr(feature = "serde", serde(transparent))] + #[as_ref(forward)] + #[repr(transparent)] + pub struct $a(std::time::SystemTime); + }; +} + +time_alias!(Atime); +time_alias!(Ctime); +time_alias!(Mtime); + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr(feature = "serde", derive(Deserialize, Serialize))] +pub struct Utimes<'a> { + #[cfg_attr(feature = "serde", serde(borrow))] + pub(crate) path: &'a Path, + pub(crate) atime: Atime, + pub(crate) mtime: Mtime, + pub(crate) ctime: Ctime, +} +from_cmd!(Utimes); +getters! {Utimes, [(path, Path, borrow), (atime, Atime, copy), (mtime, Mtime,copy), (ctime, Ctime, copy)]} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, AsRef, Deref)] +#[cfg_attr(feature = "serde", derive(Deserialize, Serialize))] +#[cfg_attr(feature = "serde", serde(transparent))] +pub struct Ino(u64); + +#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, AsRef, Deref)] +#[cfg_attr(feature = "serde", derive(Deserialize, Serialize))] +#[cfg_attr(feature = "serde", serde(transparent))] +pub struct FileOffset(u64); + +impl FileOffset { + pub fn as_u64(self) -> u64 { + self.0 + } +} + +#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, AsRef, From)] +#[as_ref(forward)] +pub struct Data<'a>(&'a [u8]); + +impl<'a> Data<'a> { + #[inline] + pub fn as_slice(&self) -> &[u8] { + self.0 + } +} + +impl<'a> Deref for Data<'a> { + type Target = [u8]; + + #[inline] + fn deref(&self) -> &Self::Target { + self.0 + } +} + +impl<'a> std::fmt::Debug for Data<'a> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let s = match std::str::from_utf8(self.0) { + Ok(s) => Cow::Borrowed(s), + Err(_) => Cow::Owned(hex::encode(self.0)), + }; + if s.len() <= 128 { + write!(f, "{s:?}") + } else { + write!( + f, + "{:?} {:?}", + &s[..64], + s.len(), + &s[s.len() - 64..] + ) + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr(feature = "serde", derive(Deserialize, Serialize))] +pub struct Write<'a> { + #[cfg_attr(feature = "serde", serde(borrow))] + pub(crate) path: &'a Path, + pub(crate) offset: FileOffset, + pub(crate) data: Data<'a>, +} +from_cmd!(Write); +getters! {Write, [(path, Path, borrow), (offset, FileOffset, copy), (data, Data, borrow)]} + +#[cfg(test)] +mod tests { + use std::collections::BTreeSet; + use std::ffi::OsString; + use std::fmt::Write; + + use similar_asserts::SimpleDiff; + + use super::*; + + // serialize sendstream commands to diffable text + fn serialize_to_txt(sendstreams: &[Sendstream]) -> String { + let mut out = String::new(); + for (i, s) in sendstreams.iter().enumerate() { + writeln!(out, "BEGIN SENDSTREAM {i}").unwrap(); + for cmd in s.commands() { + writeln!(out, "{cmd:?}").unwrap(); + } + writeln!(out, "END SENDSTREAM {i}").unwrap(); + } + out + } + + #[test] + fn parse_demo() { + let sendstreams = Sendstream::parse_all(include_bytes!("../testdata/demo.sendstream")) + .expect("failed to parse demo.sendstream"); + let parsed_txt = serialize_to_txt(&sendstreams); + if let Some(dst) = std::env::var_os("UPDATE_DEMO_TXT") { + std::fs::write(dst, serialize_to_txt(&sendstreams)).unwrap(); + } else { + let good_txt = include_str!("../testdata/demo.txt"); + if parsed_txt != good_txt { + panic!( + "{}", + SimpleDiff::from_str(&parsed_txt, good_txt, "parsed", "good") + ) + } + } + } + + #[test] + fn sendstream_covers_all_commands() { + let all_cmds: BTreeSet<_> = wire::cmd::CommandType::iter() + .filter(|c| *c != wire::cmd::CommandType::Unspecified) + // update_extent is used for no-file-data sendstreams (`btrfs send + // --no-data`), so it's not super useful to cover here + .filter(|c| *c != wire::cmd::CommandType::UpdateExtent) + .collect(); + let sendstreams = Sendstream::parse_all(include_bytes!("../testdata/demo.sendstream")) + .expect("failed to parse demo.sendstream"); + let seen_cmds = sendstreams + .iter() + .flat_map(|s| s.commands.iter().map(|c| c.command_type())) + .collect(); + + if all_cmds != seen_cmds { + let missing: BTreeSet<_> = all_cmds.difference(&seen_cmds).collect(); + panic!("sendstream did not include some commands: {:?}", missing,); + } + } +} diff --git a/antlir/antlir2/sendstream_parser/src/ser.rs b/antlir/antlir2/sendstream_parser/src/ser.rs new file mode 100644 index 00000000000..75273949723 --- /dev/null +++ b/antlir/antlir2/sendstream_parser/src/ser.rs @@ -0,0 +1,108 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +use serde::Deserialize; +use serde::Deserializer; +use serde::Serialize; +use serde::Serializer; + +use crate::Data; +use crate::XattrData; +use crate::XattrName; + +pub(crate) mod uid { + use nix::unistd::Uid; + + use super::*; + + pub fn deserialize<'de, D>(d: D) -> Result + where + D: Deserializer<'de>, + { + u32::deserialize(d).map(Uid::from_raw) + } + + pub fn serialize(u: &Uid, s: S) -> Result + where + S: Serializer, + { + u.as_raw().serialize(s) + } +} + +pub(crate) mod gid { + use nix::unistd::Gid; + + use super::*; + + pub fn deserialize<'de, D>(d: D) -> Result + where + D: Deserializer<'de>, + { + u32::deserialize(d).map(Gid::from_raw) + } + + pub fn serialize(g: &Gid, s: S) -> Result + where + S: Serializer, + { + g.as_raw().serialize(s) + } +} + +pub(crate) mod utf8 { + use serde::ser::Error; + + use super::*; + + pub fn deserialize<'de, D, T>(d: D) -> Result + where + D: Deserializer<'de>, + T: From<&'de [u8]>, + { + <&str>::deserialize(d).map(|s| s.as_bytes()).map(T::from) + } + + pub fn serialize(t: T, s: S) -> Result + where + S: Serializer, + T: AsRef<[u8]>, + { + std::str::from_utf8(t.as_ref()) + .map_err(|_| S::Error::custom("not utf8 string")) + .and_then(|d| d.serialize(s)) + } +} + +macro_rules! utf8_serde { + ($t:ident) => { + impl<'a, 'de> Deserialize<'de> for $t<'a> + where + 'de: 'a, + { + fn deserialize(d: D) -> Result + where + D: Deserializer<'de>, + { + crate::ser::utf8::deserialize(d) + } + } + + impl<'a> Serialize for $t<'a> { + fn serialize(&self, s: S) -> Result + where + S: Serializer, + { + crate::ser::utf8::serialize(self, s) + } + } + }; +} + +utf8_serde!(Data); +utf8_serde!(XattrName); +utf8_serde!(XattrData); diff --git a/antlir/antlir2/sendstream_parser/src/wire/cmd.rs b/antlir/antlir2/sendstream_parser/src/wire/cmd.rs new file mode 100644 index 00000000000..8834fd5f470 --- /dev/null +++ b/antlir/antlir2/sendstream_parser/src/wire/cmd.rs @@ -0,0 +1,391 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +use nom::IResult; + +use crate::wire::tlv::attr_types; +use crate::wire::tlv::parse_tlv; +use crate::wire::tlv::parse_tlv_with_attr; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct CommandHeader { + /// Command size, excluing command header itself + pub(crate) len: usize, + /// Command type, Check btrfs_send_command in kernel send.h for all types + pub(crate) ty: CommandType, + /// CRC32 checksum, including the header, with checksum filled with 0. + pub(crate) crc32: u32, +} + +impl CommandHeader { + pub(crate) fn parse(input: &[u8]) -> IResult<&[u8], Self> { + let (input, len) = nom::number::complete::le_u32(input)?; + let (input, ty) = CommandType::parse(input)?; + let (input, crc32) = nom::number::complete::le_u32(input)?; + Ok(( + input, + Self { + len: len as usize, + ty, + crc32, + }, + )) + } +} + +macro_rules! command_type { + ($enm: ident, $($v:ident),+) => { + /// All of the btrfs sendstream commands. Copied from linux/fs/btrfs/send.h + /// WARNING: order is important! + #[derive( + Debug, + Copy, + Clone, + PartialEq, + Eq, + PartialOrd, + Ord, + )] + pub(crate) enum $enm { + $($v,)+ + /// Unknown command, maybe it's new? + Unknown(u16), + } + + impl $enm { + const fn from_u16(u: u16) -> Self { + match u { + $(${index()} => Self::$v,)+ + _ => Self::Unknown(u), + } + } + + #[cfg(test)] + pub(crate) fn iter() -> impl Iterator { + [$(Self::$v,)+].into_iter() + } + } + } +} + +command_type!( + CommandType, + // variants below + Unspecified, + Subvol, + Snapshot, + Mkfile, + Mkdir, + Mknod, + Mkfifo, + Mksock, + Symlink, + Rename, + Link, + Unlink, + Rmdir, + SetXattr, + RemoveXattr, + Write, + Clone, + Truncate, + Chmod, + Chown, + Utimes, + End, + UpdateExtent +); + +impl CommandType { + fn parse(input: &[u8]) -> IResult<&[u8], Self> { + let (input, ty) = nom::number::complete::le_u16(input)?; + Ok((input, Self::from_u16(ty))) + } +} + +macro_rules! parse_subtypes { + ($hdr: expr, $cmd_data:expr, $($t:ident),+) => { + match $hdr.ty { + $(CommandType::$t => { + let (remaining, cmd) = crate::$t::parse($cmd_data).expect(concat!("failed to parse ", stringify!($t))); + (remaining, cmd.into()) + }),+ + CommandType::End => ($cmd_data, crate::Command::End), + _ => { + unreachable!("all btrfs sendstream command types are covered, what is this? {:?}", $hdr) + } + } + } +} + +impl<'a> crate::Command<'a> { + pub(crate) fn parse(input: &'a [u8]) -> IResult<&'a [u8], Self> { + let (input, hdr) = CommandHeader::parse(input)?; + let (input, cmd_data) = nom::bytes::complete::take(hdr.len)(input)?; + let (cmd_remaining, cmd): (_, crate::Command) = parse_subtypes!( + hdr, + cmd_data, + Chmod, + Chown, + Clone, + Link, + Mkdir, + Mkfifo, + Mkfile, + Mknod, + Mksock, + RemoveXattr, + Rename, + Rmdir, + SetXattr, + Snapshot, + Subvol, + Symlink, + Truncate, + Unlink, + UpdateExtent, + Utimes, + Write + ); + + assert!(cmd_remaining.is_empty(), "command length is wrong",); + Ok((input, cmd)) + } +} + +impl<'a> crate::Subvol<'a> { + fn parse(input: &'a [u8]) -> IResult<&[u8], Self> { + let (input, path) = parse_tlv(input)?; + let (input, uuid) = parse_tlv(input)?; + let (input, ctransid) = parse_tlv(input)?; + Ok(( + input, + Self { + path, + uuid, + ctransid, + }, + )) + } +} + +impl<'a> crate::Chmod<'a> { + fn parse(input: &'a [u8]) -> IResult<&[u8], Self> { + let (input, path) = parse_tlv(input)?; + let (input, mode) = parse_tlv(input)?; + Ok((input, Self { path, mode })) + } +} + +impl<'a> crate::Chown<'a> { + fn parse(input: &'a [u8]) -> IResult<&[u8], Self> { + let (input, path) = parse_tlv(input)?; + let (input, uid) = parse_tlv(input)?; + let (input, gid) = parse_tlv(input)?; + Ok((input, Self { path, uid, gid })) + } +} + +impl<'a> crate::Clone<'a> { + fn parse(input: &'a [u8]) -> IResult<&[u8], Self> { + let (input, dst_offset) = parse_tlv(input)?; + let (input, len) = parse_tlv(input)?; + let (input, dst_path) = parse_tlv(input)?; + let (input, uuid) = parse_tlv_with_attr::<_, 16, attr_types::CloneUuid>(input)?; + let (input, ctransid) = parse_tlv_with_attr::<_, 8, attr_types::CloneCtransid>(input)?; + let (input, src_path) = parse_tlv_with_attr::<_, 0, attr_types::ClonePath>(input)?; + let (input, src_offset) = parse_tlv_with_attr::<_, 8, attr_types::CloneOffset>(input)?; + Ok(( + input, + Self { + src_offset, + len, + src_path, + uuid, + ctransid, + dst_path, + dst_offset, + }, + )) + } +} + +impl<'a> crate::Link<'a> { + fn parse(input: &'a [u8]) -> IResult<&[u8], Self> { + let (input, link_name) = parse_tlv(input)?; + let (input, target) = parse_tlv(input)?; + Ok((input, Self { target, link_name })) + } +} + +impl<'a> crate::Symlink<'a> { + fn parse(input: &'a [u8]) -> IResult<&[u8], Self> { + let (input, link_name) = parse_tlv(input)?; + let (input, ino) = parse_tlv(input)?; + let (input, target) = parse_tlv(input)?; + Ok(( + input, + Self { + target, + ino, + link_name, + }, + )) + } +} + +impl<'a> crate::Mkdir<'a> { + fn parse(input: &'a [u8]) -> IResult<&[u8], Self> { + let (input, path) = parse_tlv(input)?; + let (input, ino) = parse_tlv(input)?; + Ok((input, Self { path, ino })) + } +} + +impl<'a> crate::Mkfile<'a> { + fn parse(input: &'a [u8]) -> IResult<&[u8], Self> { + let (input, path) = parse_tlv(input)?; + let (input, ino) = parse_tlv(input)?; + Ok((input, Self { path, ino })) + } +} + +impl<'a> crate::Mkspecial<'a> { + fn parse(input: &'a [u8]) -> IResult<&[u8], Self> { + let (input, path) = parse_tlv(input)?; + let (input, ino) = parse_tlv(input)?; + let (input, rdev) = parse_tlv(input)?; + let (input, mode) = parse_tlv(input)?; + Ok(( + input, + Self { + path, + ino, + rdev, + mode, + }, + )) + } +} + +macro_rules! mkspecial { + ($t:ident) => { + impl<'a> crate::$t<'a> { + fn parse(input: &'a [u8]) -> IResult<&[u8], Self> { + crate::Mkspecial::parse(input).map(|(r, s)| (r, Self(s))) + } + } + }; +} + +mkspecial!(Mknod); +mkspecial!(Mkfifo); +mkspecial!(Mksock); + +impl<'a> crate::RemoveXattr<'a> { + fn parse(input: &'a [u8]) -> IResult<&[u8], Self> { + let (input, path) = parse_tlv(input)?; + let (input, name) = parse_tlv(input)?; + Ok((input, Self { path, name })) + } +} + +impl<'a> crate::Rename<'a> { + fn parse(input: &'a [u8]) -> IResult<&[u8], Self> { + let (input, from) = parse_tlv(input)?; + let (input, to) = parse_tlv_with_attr::<_, 0, attr_types::PathTo>(input)?; + Ok((input, Self { from, to })) + } +} + +impl<'a> crate::Rmdir<'a> { + fn parse(input: &'a [u8]) -> IResult<&[u8], Self> { + let (input, path) = parse_tlv(input)?; + Ok((input, Self { path })) + } +} + +impl<'a> crate::SetXattr<'a> { + fn parse(input: &'a [u8]) -> IResult<&[u8], Self> { + let (input, path) = parse_tlv(input)?; + let (input, name) = parse_tlv(input)?; + let (input, data) = parse_tlv(input)?; + Ok((input, Self { path, name, data })) + } +} + +impl<'a> crate::Truncate<'a> { + fn parse(input: &'a [u8]) -> IResult<&[u8], Self> { + let (input, path) = parse_tlv(input)?; + let (input, size) = parse_tlv(input)?; + Ok((input, Self { path, size })) + } +} + +impl<'a> crate::Snapshot<'a> { + fn parse(input: &'a [u8]) -> IResult<&[u8], Self> { + let (input, path) = parse_tlv(input)?; + let (input, uuid) = parse_tlv(input)?; + let (input, ctransid) = parse_tlv(input)?; + let (input, clone_uuid) = parse_tlv_with_attr::<_, 16, attr_types::CloneUuid>(input)?; + let (input, clone_ctransid) = + parse_tlv_with_attr::<_, 8, attr_types::CloneCtransid>(input)?; + Ok(( + input, + Self { + path, + uuid, + ctransid, + clone_uuid, + clone_ctransid, + }, + )) + } +} + +impl<'a> crate::Unlink<'a> { + fn parse(input: &'a [u8]) -> IResult<&[u8], Self> { + let (input, path) = parse_tlv(input)?; + Ok((input, Self { path })) + } +} + +impl<'a> crate::UpdateExtent<'a> { + fn parse(input: &'a [u8]) -> IResult<&[u8], Self> { + let (input, path) = parse_tlv(input)?; + let (input, offset) = parse_tlv(input)?; + let (input, len) = parse_tlv(input)?; + Ok((input, Self { path, offset, len })) + } +} + +impl<'a> crate::Utimes<'a> { + fn parse(input: &'a [u8]) -> IResult<&[u8], Self> { + let (input, path) = parse_tlv(input)?; + let (input, atime) = parse_tlv(input)?; + let (input, mtime) = parse_tlv(input)?; + let (input, ctime) = parse_tlv(input)?; + Ok(( + input, + Self { + path, + atime, + mtime, + ctime, + }, + )) + } +} + +impl<'a> crate::Write<'a> { + fn parse(input: &'a [u8]) -> IResult<&[u8], Self> { + let (input, path) = parse_tlv(input)?; + let (input, offset) = parse_tlv(input)?; + let (input, data) = parse_tlv(input)?; + Ok((input, Self { path, offset, data })) + } +} diff --git a/antlir/antlir2/sendstream_parser/src/wire/mod.rs b/antlir/antlir2/sendstream_parser/src/wire/mod.rs new file mode 100644 index 00000000000..6aba6fab819 --- /dev/null +++ b/antlir/antlir2/sendstream_parser/src/wire/mod.rs @@ -0,0 +1,43 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +use nom::IResult; + +use crate::Sendstream; + +static MAGIC_HEADER: &[u8] = b"btrfs-stream\0"; + +pub(crate) mod cmd; +mod tlv; +use crate::Error; +use crate::Result; + +impl<'a> Sendstream<'a> { + fn parse(input: &'a [u8]) -> IResult<&'a [u8], Self> { + let (input, _) = nom::bytes::complete::tag(MAGIC_HEADER)(input)?; + let (input, version) = nom::number::complete::le_u32(input)?; + assert_eq!(1, version); + let (input, commands) = nom::multi::many1(crate::Command::parse)(input)?; + Ok((input, Self { commands })) + } + + pub fn parse_all(input: &'a [u8]) -> Result> { + match nom::combinator::complete(nom::multi::many1(Sendstream::parse))(input) { + Ok((left, sendstreams)) => { + if !left.is_empty() { + Err(Error::TrailingData(left.to_vec())) + } else { + Ok(sendstreams) + } + } + Err(e) => match e { + nom::Err::Error(e) | nom::Err::Failure(e) => Err(e.into()), + nom::Err::Incomplete(_) => Err(Error::Incomplete), + }, + } + } +} diff --git a/antlir/antlir2/sendstream_parser/src/wire/tlv.rs b/antlir/antlir2/sendstream_parser/src/wire/tlv.rs new file mode 100644 index 00000000000..96a1b49c3b1 --- /dev/null +++ b/antlir/antlir2/sendstream_parser/src/wire/tlv.rs @@ -0,0 +1,325 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +use std::ffi::OsStr; +use std::os::unix::ffi::OsStrExt; +use std::path::Path; +use std::time::Duration; +use std::time::SystemTime; + +use nix::unistd::Gid; +use nix::unistd::Uid; +use nom::IResult; +use uuid::Uuid; + +/// Parse the TLV with the output type's primary attribute tag +pub(crate) fn parse_tlv<'i, T, const L: usize, Attr>(input: &'i [u8]) -> IResult<&'i [u8], T> +where + T: Tlv<'i, L, Attr = Attr>, + Attr: AttrTypeParam, +{ + parse_tlv_with_attr::<'i, T, L, T::Attr>(input) +} + +/// Parse the TLV with an explicit attribute tag. This allows for parsing +/// identical data types from the different Attrs. +pub(crate) fn parse_tlv_with_attr<'i, T, const L: usize, Attr>( + input: &'i [u8], +) -> IResult<&'i [u8], T> +where + T: Tlv<'i, L>, + // Ensure that T can be parsed from this attribute + T: ParsesFromAttr, + Attr: AttrTypeParam, +{ + // guarantee that the tlv type is what we expected + let (input, _) = nom::bytes::complete::tag(Attr::attr().tag())(input)?; + match L { + 0 => { + let (input, len) = nom::number::complete::le_u16(input)?; + let (input, data) = nom::bytes::complete::take(len)(input)?; + Ok((input, T::parse(data))) + } + _ => { + // this will cause the parser to fail if the length is not + // exactly L bytes + let (input, _) = nom::bytes::complete::tag((L as u16).to_le_bytes())(input)?; + let (input, data) = nom::bytes::complete::take(L)(input)?; + Ok(( + input, + #[allow(clippy::expect_used)] + T::parse_exact(data.try_into().expect("length is already checked")), + )) + } + } +} + +/// Type-length-value struct. If L is not 0, the parser will automatically +/// ensure that the data is exactly L bytes long, and will call parse_exact +/// instead of parse. +pub(crate) trait Tlv<'i, const L: usize>: ParsesFromAttr { + /// Default attribute when calling parse_tlv + type Attr: AttrTypeParam; + + /// Parse the data into whatever the inner type is + fn parse(_data: &'i [u8]) -> Self + where + Self: Sized, + { + unimplemented!() + } + + fn parse_exact(_data: [u8; L]) -> Self + where + Self: Sized, + { + unimplemented!() + } +} + +pub(crate) trait ParsesFromAttr +where + Attr: AttrTypeParam, +{ +} + +macro_rules! tlv_impl { + ($lt: lifetime, $ty: ty, $default_attr: ident, $parse: expr) => { + impl<$lt> Tlv<$lt, 0> for $ty { + type Attr = attr_types::$default_attr; + + fn parse(data: &$lt [u8]) -> Self { + $parse(data) + } + } + + impl<$lt> ParsesFromAttr for $ty {} + }; + ($lt: lifetime, $ty: ty, $default_attr: ident, $parse:expr, $($attr:ident),+) => { + tlv_impl!($lt, $ty, $default_attr, $parse); + $(impl<$lt> ParsesFromAttr for $ty {})+ + }; + ($ty: ty, $len: literal, $default_attr: ident, $parse: expr) => { + impl<'i> Tlv<'i, $len> for $ty { + type Attr = attr_types::$default_attr; + + fn parse_exact(data: [u8; $len]) -> Self { + $parse(data) + } + } + + impl ParsesFromAttr for $ty {} + }; + ($ty: ty, $len: literal, $default_attr: ident, $parse:expr, $($attr:ident),+) => { + tlv_impl!($ty, $len, $default_attr, $parse); + $(impl ParsesFromAttr for $ty {})+ + }; +} + +tlv_impl!( + 'i, + &'i Path, + Path, + |data: &'i [u8]| -> &'i Path { Path::new(OsStr::from_bytes(data)) }, + PathTo, + ClonePath +); + +tlv_impl!( + 'i, + crate::TemporaryPath<'i>, + Path, + |data: &'i [u8]| -> crate::TemporaryPath<'i> { crate::TemporaryPath(Path::new(OsStr::from_bytes(data))) } +); + +tlv_impl!( + Uuid, + 16, + Uuid, + |data: [u8; 16]| -> Uuid { Uuid::from_u128_le(u128::from_le_bytes(data)) }, + CloneUuid +); + +tlv_impl!( + crate::Ctransid, + 8, + Ctransid, + |data: [u8; 8]| -> crate::Ctransid { crate::Ctransid(u64::from_le_bytes(data)) }, + CloneCtransid +); + +tlv_impl!(Uid, 8, Uid, |data: [u8; 8]| -> Uid { + Uid::from_raw(u64::from_le_bytes(data) as u32) +}); + +tlv_impl!(Gid, 8, Gid, |data: [u8; 8]| -> Gid { + Gid::from_raw(u64::from_le_bytes(data) as u32) +}); + +tlv_impl!(crate::Mode, 8, Mode, |data: [u8; 8]| -> crate::Mode { + crate::Mode(u64::from_le_bytes(data) as u32) +}); + +tlv_impl!(crate::Ino, 8, Ino, |data: [u8; 8]| -> crate::Ino { + crate::Ino(u64::from_le_bytes(data)) +}); + +tlv_impl!( + 'i, + crate::XattrName<'i>, + XattrName, + |data: &'i [u8]| -> crate::XattrName<'i> { + crate::XattrName(data) + } +); + +tlv_impl!( + 'i, + crate::XattrData<'i>, + XattrData, + |data: &'i [u8]| -> crate::XattrData<'i> { + crate::XattrData(data) + } +); + +tlv_impl!( + crate::FileOffset, + 8, + FileOffset, + |data: [u8; 8]| -> crate::FileOffset { crate::FileOffset(u64::from_le_bytes(data)) }, + CloneOffset +); + +tlv_impl!( + 'i, + crate::Data<'i>, + Data, + |data: &'i [u8]| -> crate::Data<'i> { + crate::Data(data) + } +); + +tlv_impl!( + 'i, + crate::LinkTarget<'i>, + Link, + |data: &'i [u8]| -> crate::LinkTarget<'i> { + crate::LinkTarget(Path::new(OsStr::from_bytes(data))) + } +); + +tlv_impl!(crate::Rdev, 8, Rdev, |data: [u8; 8]| -> crate::Rdev { + crate::Rdev(u64::from_le_bytes(data)) +}); + +tlv_impl!( + crate::CloneLen, + 8, + CloneLen, + |data: [u8; 8]| -> crate::CloneLen { crate::CloneLen(u64::from_le_bytes(data)) } +); + +tlv_impl!(u64, 8, Size, |data: [u8; 8]| -> u64 { + u64::from_le_bytes(data) +}); + +fn parse_time(data: [u8; 12]) -> SystemTime { + #[allow(clippy::expect_used)] + let secs = u64::from_le_bytes(data[..8].try_into().expect("right size")); + #[allow(clippy::expect_used)] + let nanos = u32::from_le_bytes(data[8..].try_into().expect("right size")); + SystemTime::UNIX_EPOCH + Duration::from_secs(secs) + Duration::from_nanos(nanos.into()) +} + +macro_rules! time_tlv { + ($i:ident) => { + tlv_impl!(crate::$i, 12, $i, |data: [u8; 12]| -> crate::$i { + crate::$i(parse_time(data)) + }); + }; +} + +time_tlv!(Atime); +time_tlv!(Mtime); +time_tlv!(Ctime); + +pub(crate) trait AttrTypeParam { + fn attr() -> Attr; +} + +macro_rules! gen_attrs_code { + ($enm: ident, $($v:ident),+) => { + #[derive( + Debug, + Copy, + Clone, + PartialEq, + Eq, + PartialOrd, + Ord, + )] + pub(crate) enum $enm { + $($v,)+ + } + + impl $enm { + const fn as_u16(self) -> u16 { + match self { + $(Self::$v => ${index()},)+ + } + } + } + + pub(crate) mod attr_types { + /// Empty type used as type parameter for parse_tlv + $( + pub(crate) struct $v(); + impl super::AttrTypeParam for $v { + fn attr() -> super::Attr { + super::Attr::$v + } + } + )+ + } + } +} + +gen_attrs_code!( + Attr, + // variants go below, order is important and must match send.h + Unspecified, + Uuid, + Ctransid, + Ino, + Size, + Mode, + Uid, + Gid, + Rdev, + Ctime, + Mtime, + Atime, + Otime, + XattrName, + XattrData, + Path, + PathTo, + Link, + FileOffset, + Data, + CloneUuid, + CloneCtransid, + ClonePath, + CloneOffset, + CloneLen +); + +impl Attr { + fn tag(self) -> [u8; 2] { + self.as_u16().to_le_bytes() + } +} diff --git a/antlir/antlir2/sendstream_parser/testdata/demo.sendstream b/antlir/antlir2/sendstream_parser/testdata/demo.sendstream new file mode 100644 index 0000000000000000000000000000000000000000..5b7e9e9b3715eb371ec2b9fae0f5e4ee48d3e704 GIT binary patch literal 320693 zcmeI*eT-bydB^d`Ucc-MX4boF%wmj>X)rI|*Sj-jLl8kJJYTR{n+qEv+#hpIv$P&HHmR}pEGHvG{9+>8t;pH>Gb&5BHUFfc&sOL=Cmt(n z&xiFtM`X|LYR0!(uJ63z7pIR)^=C{jy8EoV{X0W)nmn-akM&4=9hVD0sfo;)@_ z@UGYEwcRrL*Xp{L8nQ~VEi>Di+qO(k&rCO``*a~ox(hi=n+?c=Fe@`Qn0R!}`X2uS_My>;`Q& zDF1c-c-4^8-}&$6T#!er>kZ$7Reuj=u}xQZm8{)-t^TrWTd_(%$W>wU$sjFbmrwc+ z_v!kJ_}z2oH}u@g%(f0aEpE3EuJ+FPMr^)~IS@55Y`e)&( z?|*6SjLv3-T>Q-EAMO5YS^K7?mI1v;H3%Z_jbDemg_kj)?v99}mCNc7Aocn{k=GKPET7@%rI+-mmxA zd7rqg=iB37*L%#r_P_pqd1}v!+d7-Pb-Jg=mUS1^^U3$yFd%<9{Km!w(+ulJf%Oa_E+xy55Kscxi8jpJLY@lWB>HIa~4ce@1Pt0^uC@B@~`V1&lr)NCX(`){9z1RF}zr{nm?gKl?h z>FKo<-Co<*%OXTuq&jy}a&oy;vqw?6Y@yi+~oB3kBw{gDrzPfJT z?vqaDd+@~H_p7w&nB2c1{K`q2)_ZaLp67dd(Z8;MW8y5g@5Q%q!%_eD;!BTQ^Jk6TdH4RskfXc&0*z}IRbrw{ z7jFtznjo>|gbA~>uwiM-DmOvo+O7y)I%{m~^v%LXI)SmSDv~TRrA^v8u}ec2xu{A@ zQ#$po=Oe%}4z#gQ~Q~<&~~e zXIoura@)@BoJ}K}>xS!p9L(aXt&6j|I=ZFC7KJsTi{leLq#IP3RyFH_(xiTcOgqT+ zMU_vQs}6;>U0Ut1!f795VHns@+l5uzMS;#$H!C&mJadsveJ}c+(uIaq+z$M9o2=Dx zp$kl8^}xWtn`E(Hjlg%1)9xs`{aPiHgZ%dR?eX2?yT_iBJtu$nIBMajg=0>RIY|s8 z2J#+heLmxp*0C0>1u4ThaZZ#PrKZL}je!ag^*!o)RA;HqQd_6C&Y2OM*W%CoLCTOa zqzoxT%8)Xo3@Jm(kTRqUDMQMTGNcSCL&}gcqzozJ`aQ1S)9X69`jV?Jx%!f;FS+`X zt1r3wlB+Md`jV?Jx%!f;FS+`Xt1r3wlB+Md`jR_cai=SP2Q;J%DMQMTGNcSCL&}gc zqzoxT%8)Xo3@Jm(kTRqUDMQMTGVbWlcnplD!Dt$cg~C`UjAX+&M2thkC{m0f#Ta0W z0mcY!jK9bDdyJmQ=!uMd>75WML&}gcqzoxT%8)Xo3@Jm(kTRqUDMQMTGNcSCL&}gc zq)bOOF-RFwhLj;?NEuRwlp$qE8B&InA!SGzQihZvWk?xPhLj;?nXwKjL&}gcqzoxT z%8)Xo3@Jm(kTRqUDMQMTGNcSCL&}gcq^yeL04YPtkTRqUDMQMTGNcSCL&}gcqzoxT z%HBUxw&AEqLyoMtV^u?jWX{HMHl@O0lM(3`(Kr2LRG%?m5d#)6U=afrF<=n`7BOHE z0~Rr0kuQZYwD&H5uxMPns1h^3Uoe0uAPR^AqJStM3Wx%tfG8mP{sB>(6X(P^aZa2Q z=fpX2PMj0xWVk7AqRXv8xZr^c9=PCv3m&NNQQxD!M}3d_9`!xyd(`)+?@`~QzQ?UW zxHSm32I1Bq+!`c^OoEglWk?xPhLj;?NEuRwlp$qE8B&InA!SGzQihZvWk?xPX2L2) z%8)Xo3@Jm(kTRqUDMQMTGNcSCL&}gcqzoxT%8)Xo3@JNMxi(UUlp$qE8B&InA!SGz zQihZvWk?xPhLj;?NEuRwlp$qEng3gWlp$qE8B&InA!SGzQihZvWk?xPhLj;?NEuRw zlp$qE8B&&-gBenWlp$qE8B&InA!SGzQihZvWk?xPhLj;?NEuRwlp$qEnTx6vDMQMT zGNcSCL&}gcqzoxT%8)Xo3@Jm(kTRqUDMQMTGNi2XmmsSaW?>lEFer7;*mhA+>a#RU zO*_wAWYcoO6y6kF%AS)wCworzoa{N-bEa8QQe&XTK#hSK12qO}3?v3>4AdB?G5q{$ z3>$70X~^X_l)D=;By%>7vnl<3*knZdMQ&Xp{xPc0{KwQ4CUu1~RcV6Q5$lQd#Cl>q zv7T6ObC2Cg7ROHiHl!{wE;6<#P0Zm9hc`dJ!yArTs$_C-mp`C0u3c1#i878_IBMaj zg`*aZS~zMc?17461jPu75fmdRMo^3(t&`SC>s;`_1rMCIOMMS1L&}gcqzoxT%8;^B zcLYcoQihZvWk?xPhLj;?NEuRwlp$qE8B&InA!SGzQihZvWhQhnQihZvWk?xPhLj;? zNEuRwlp$qE8B&InA!SGzQihZvWk{LcoWdYwNEuRwlp$qE8B&InA!SGzQihZvWk?xP zhLj;?NEuRwlrhX=90v(fhLj;?NEuRwlp$qE8B&InA!SGzQihZvWk?xPhLj;?NEzer z+3t8}E;YGr=XTDfkkezWY5W-lRc-6^pA(J zt}2o&GNn!0S+0LKLKk5zSj#V)wQM*l(vX)&cO7iVkj&XQ&Zg9zZ89SLBC=F}L`Fr9 zwgaOgZ?aa)g)T6WEkGGi29zOM5G{xnL<^z?(Sm3ZlqTi$B~D-B^d(MT;`Ako5s(_B z_RB_UNSU#Rj15efB}(j>{+l9YNEuRwlp$qE8B&InA!SHe>JlAz+9b|DIcQTCBon^I zTlu=`##z2jM8Ur(th%ty5d4woYvwDMQMTGNcSCL&}gcqzoxT z%8)XoEa?tKL2aF3H%e2LX8sCYYi!y1U|5p`vuV)zq6Cv>w~k1{o|8Q%drtP83|Pdl z8@63qY7EpEs4-AupvFLrff@rf25Jn{7^pE&W1z->lp$qE8B&InA!SGz_r_)1Sw_>a zT(@2n7AZr@kTRqUDMQMTGNcSCL&}gcqzoxT%8)Xo3@Jm(kTQ#uMVa2nhpSM0Lex(th%ty5b^ z%8)Xo3@Jm(kTRqUDMQMTGNcSCL(24=f(j`^%8)Xo3@Jm(kTRqUDMQMTGNcSCL&}gc zr0f?#%HBL8(vX**xu&lnLo#RMIGgepmLwz6FLHFL_{XTcd?bim+ZCZpXN`@WztDnU zNH8QA5)2831Ve%$!H{4`FeDff3<-w62*HrN7gycmIvw@1GOcRXk@v`ZmibX!j7v>!+qs>yX=HN}1BrpeKw=;@Bo6jxdJkw@k{Cz~BnA=#iGjpGVjwY)7)T5x1`-2_ z!CZAHy-PHi12L&}gcqzoxT%8)Xo>{O-fd=VLyM<3jCTSJCq&c<;z z6>Cv0St0%UR_C#@_I#cG*N7bW)Y)&nYl7w4=rrp6AyN)?QC-X`MOnuGj0eU7uX{-5q-yvP!ZoTQ6;H+cG^pGu@o-)8+Mdmv@mi z8<6jO`{DB!G#i#R_04MLwiUbdMjLI{X~tx7)4gRwmUo)e*Y4UoujyI4vaEX6&cBXz z7Q-^q7ri)A>!CB}pD)$-L>h}NeM~28ys&xS@!Oxc`mP6EXt_PQk|tbOLdm&EK-J)A!SGzQihZvWk?xPhLj;? zNEuRwlp$r@Z$8$0?;~YM8B&InA!SGzQihZvWk?xPhLj;?NEuRwlp$qE8B&InG0sX* znh+^N%8)Xo3@Jm(kTRqUDMQMTGNcSCL&}gcqzoxT%8)XojQfqr?= zhLj;?NEuRwlp$qE8B&InA!SIJvHI6w!W1^r`&Sw3szQ$tOlgxgiGjpGVjwY)7)T5x z1`-2_fy6*!ATf{_jI9i{b!zL>)~T&iTc@^;lp$qE8B&InA!SGzQihZvWk?xPhLj;? zRWNCgGNcSCL&}gcqzoxT%8)Xo3@Jm(kTRqUDMQMTGNcSCL&|LGuf(d-(zsN|2#ec+ zIZ!#1wOS+w5(9~W#6V&oF_0KY3?v2;!}~8W96ut`kn1nF|Bo87M&@iBXH#*O+oYM> zR_xLnZ8ReNB1e~se~ik@M}o+;T@ku;mS6b$?-$N*FRl{3uZI7S^V=if6Yx#y3QmRL zR2WW$;Zzt-h2c~fU>n#5wt;P68`uW6fo)(L=S0eoGNcSC)54HJ%8)Xo3@Jm(kTRqU zDMQMTGNcSCL&}gcqzoxT%8)Xo3@OX)oJGozGNcSCL&}gcqzoxT%8)Xo3@Jm(kTRqU zDMQMTGNcSCD{az7%8)Xo3@Jm(kTRqUDMQMTGNcSCL&}gcqzoxT%8)Xo3@IzCd=e=` z%8)Xo3@Jm(kTRqUDMQMTGNcSCL&}gcqzoxT%8)Xo%%(0t%8)Xo3@Jm(kTRqUDMQMT zGNcSCL&}gcqzoxT%8)Xo3@I~NtA&&yWk?xPhLj;?NEuRwlp$qE8B&InA!SGzQihZv zWk?xPX2L8%%8)Xo3@Jm(kTRqUDMQMTGNcSCL&}gcqzoxT%8)Xo3@Nkagmyw&*s!#^ zPcT8`+O7y)I!j{s|C1Ol7Lif;{V&|oXvi9wvvHhF#hD(-G;`aEU3#O9R!F}#>O5A~ zo}Z`xH6l~5t^LDACta@1A79d$bf^9^Cf|GEitjJld`M1{2R1$u`U%RiI&-l0`s;uF z`7;OJ^?JRw>ywwRpZl$btdeZY)|uwEEz{F8)6MBVUGhM8$sf>W1M=W!Zr-<`*|0QZ zu9~$?9f+sAso8R|y;NI|$whacb$3Ib)mQ1854rLBD*fomW8?Kz@~=1OG^^z)Gk#8E z;jEW+dv9DDZPuSrd4A@@+fLGOu`c&>ACa%keDdLwOjPgeE4AU6eB;2s{P#&3*0-!c_4Jf~UGFLX+V|8-x!{w(eRg+KxApYYP_3u5<^031z#D(pdG5ba19ITbb=NMK zd{~C1$if?K*-W#@PWtDk)@u{Guu&O)@}4IeCzhsy6Knyh0Xhp-~PnackMW~ z_x16ioon>{r@QZWdiT|>KYi|J?FVkX@3g7{pXwc9kx) zDr`O(q-E^#Nx!z6v}r@`NRq$kX*#0oa^f+n^Z4~Fi}v4SHxBym(`tF<2^(_j4Obv+y5U$55L49T;vZ2zkNJeFB=zimb~ iwxx|ut#{Vwj8Bs_&m2D7&`pV4-h2;d?)#wgQTbomIg20w literal 0 HcmV?d00001 diff --git a/antlir/antlir2/sendstream_parser/testdata/demo.txt b/antlir/antlir2/sendstream_parser/testdata/demo.txt new file mode 100644 index 00000000000..0f4ba00e043 --- /dev/null +++ b/antlir/antlir2/sendstream_parser/testdata/demo.txt @@ -0,0 +1,98 @@ +BEGIN SENDSTREAM 0 +Subvol(Subvol { path: "demo", uuid: 0fbf2b5f-ff82-a748-8b41-e35aec190b49, ctransid: Ctransid(720050) }) +Chown(Chown { path: "", uid: Uid(0), gid: Gid(0) }) +Chmod(Chmod { path: "", mode: Mode { permissions: Permissions(FilePermissions { mode: 493 }), type: (empty) } }) +Utimes(Utimes { path: "", atime: Atime(SystemTime { tv_sec: 1671045523, tv_nsec: 426350787 }), mtime: Mtime(SystemTime { tv_sec: 1671045523, tv_nsec: 434350827 }), ctime: Ctime(SystemTime { tv_sec: 1671045523, tv_nsec: 434350827 }) }) +Mkdir(Mkdir { path: TemporaryPath("o257-720050-0"), ino: Ino(257) }) +Rename(Rename { from: "o257-720050-0", to: "hello" }) +Utimes(Utimes { path: "", atime: Atime(SystemTime { tv_sec: 1671045523, tv_nsec: 426350787 }), mtime: Mtime(SystemTime { tv_sec: 1671045523, tv_nsec: 434350827 }), ctime: Ctime(SystemTime { tv_sec: 1671045523, tv_nsec: 434350827 }) }) +Chown(Chown { path: "hello", uid: Uid(0), gid: Gid(0) }) +Chmod(Chmod { path: "hello", mode: Mode { permissions: Permissions(FilePermissions { mode: 493 }), type: (empty) } }) +Utimes(Utimes { path: "hello", atime: Atime(SystemTime { tv_sec: 1671045523, tv_nsec: 391350615 }), mtime: Mtime(SystemTime { tv_sec: 1671045523, tv_nsec: 410350708 }), ctime: Ctime(SystemTime { tv_sec: 1671045523, tv_nsec: 410350708 }) }) +Mkfile(Mkfile { path: TemporaryPath("o258-720050-0"), ino: Ino(258) }) +Rename(Rename { from: "o258-720050-0", to: "hello/msg" }) +Link(Link { link_name: "hello/msg-hard", target: LinkTarget("hello/msg") }) +Utimes(Utimes { path: "hello", atime: Atime(SystemTime { tv_sec: 1671045523, tv_nsec: 391350615 }), mtime: Mtime(SystemTime { tv_sec: 1671045523, tv_nsec: 410350708 }), ctime: Ctime(SystemTime { tv_sec: 1671045523, tv_nsec: 410350708 }) }) +Utimes(Utimes { path: "hello", atime: Atime(SystemTime { tv_sec: 1671045523, tv_nsec: 391350615 }), mtime: Mtime(SystemTime { tv_sec: 1671045523, tv_nsec: 410350708 }), ctime: Ctime(SystemTime { tv_sec: 1671045523, tv_nsec: 410350708 }) }) +SetXattr(SetXattr { path: "hello/msg", name: XattrName([117, 115, 101, 114, 46, 97, 110, 116, 108, 105, 114, 46, 100, 101, 109, 111]), data: XattrData([123, 34, 104, 101, 108, 108, 111, 34, 58, 32, 34, 119, 111, 114, 108, 100, 34, 125]) }) +Write(Write { path: "hello/msg", offset: FileOffset(0), data: "Hello world!\n" }) +Chown(Chown { path: "hello/msg", uid: Uid(0), gid: Gid(0) }) +Chmod(Chmod { path: "hello/msg", mode: Mode { permissions: Permissions(FilePermissions { mode: 256 }), type: (empty) } }) +Utimes(Utimes { path: "hello/msg", atime: Atime(SystemTime { tv_sec: 1671045523, tv_nsec: 391350615 }), mtime: Mtime(SystemTime { tv_sec: 1671045523, tv_nsec: 391350615 }), ctime: Ctime(SystemTime { tv_sec: 1671045523, tv_nsec: 396350639 }) }) +Mkfifo(Mkfifo(Mkspecial { path: TemporaryPath("o259-720050-0"), ino: Ino(259), rdev: Rdev(0), mode: Mode { permissions: Permissions(FilePermissions { mode: 4516 }), type: S_IFIFO } })) +Rename(Rename { from: "o259-720050-0", to: "myfifo" }) +Utimes(Utimes { path: "", atime: Atime(SystemTime { tv_sec: 1671045523, tv_nsec: 426350787 }), mtime: Mtime(SystemTime { tv_sec: 1671045523, tv_nsec: 434350827 }), ctime: Ctime(SystemTime { tv_sec: 1671045523, tv_nsec: 434350827 }) }) +Chown(Chown { path: "myfifo", uid: Uid(0), gid: Gid(0) }) +Chmod(Chmod { path: "myfifo", mode: Mode { permissions: Permissions(FilePermissions { mode: 420 }), type: (empty) } }) +Utimes(Utimes { path: "myfifo", atime: Atime(SystemTime { tv_sec: 1671045523, tv_nsec: 394350629 }), mtime: Mtime(SystemTime { tv_sec: 1671045523, tv_nsec: 394350629 }), ctime: Ctime(SystemTime { tv_sec: 1671045523, tv_nsec: 394350629 }) }) +Symlink(Symlink { link_name: "o260-720050-0", ino: Ino(260), target: LinkTarget("hello/msg") }) +Rename(Rename { from: "o260-720050-0", to: "hello/msg-sym" }) +Utimes(Utimes { path: "hello", atime: Atime(SystemTime { tv_sec: 1671045523, tv_nsec: 391350615 }), mtime: Mtime(SystemTime { tv_sec: 1671045523, tv_nsec: 410350708 }), ctime: Ctime(SystemTime { tv_sec: 1671045523, tv_nsec: 410350708 }) }) +Chown(Chown { path: "hello/msg-sym", uid: Uid(0), gid: Gid(0) }) +Utimes(Utimes { path: "hello/msg-sym", atime: Atime(SystemTime { tv_sec: 1671045523, tv_nsec: 395350634 }), mtime: Mtime(SystemTime { tv_sec: 1671045523, tv_nsec: 395350634 }), ctime: Ctime(SystemTime { tv_sec: 1671045523, tv_nsec: 395350634 }) }) +Mkfile(Mkfile { path: TemporaryPath("o261-720050-0"), ino: Ino(261) }) +Rename(Rename { from: "o261-720050-0", to: "to-be-deleted" }) +Utimes(Utimes { path: "", atime: Atime(SystemTime { tv_sec: 1671045523, tv_nsec: 426350787 }), mtime: Mtime(SystemTime { tv_sec: 1671045523, tv_nsec: 434350827 }), ctime: Ctime(SystemTime { tv_sec: 1671045523, tv_nsec: 434350827 }) }) +Chown(Chown { path: "to-be-deleted", uid: Uid(0), gid: Gid(0) }) +Chmod(Chmod { path: "to-be-deleted", mode: Mode { permissions: Permissions(FilePermissions { mode: 420 }), type: (empty) } }) +Utimes(Utimes { path: "to-be-deleted", atime: Atime(SystemTime { tv_sec: 1671045523, tv_nsec: 397350644 }), mtime: Mtime(SystemTime { tv_sec: 1671045523, tv_nsec: 397350644 }), ctime: Ctime(SystemTime { tv_sec: 1671045523, tv_nsec: 397350644 }) }) +Mkdir(Mkdir { path: TemporaryPath("o262-720050-0"), ino: Ino(262) }) +Rename(Rename { from: "o262-720050-0", to: "dir-to-be-deleted" }) +Utimes(Utimes { path: "", atime: Atime(SystemTime { tv_sec: 1671045523, tv_nsec: 426350787 }), mtime: Mtime(SystemTime { tv_sec: 1671045523, tv_nsec: 434350827 }), ctime: Ctime(SystemTime { tv_sec: 1671045523, tv_nsec: 434350827 }) }) +Chown(Chown { path: "dir-to-be-deleted", uid: Uid(0), gid: Gid(0) }) +Chmod(Chmod { path: "dir-to-be-deleted", mode: Mode { permissions: Permissions(FilePermissions { mode: 493 }), type: (empty) } }) +Utimes(Utimes { path: "dir-to-be-deleted", atime: Atime(SystemTime { tv_sec: 1671045523, tv_nsec: 398350649 }), mtime: Mtime(SystemTime { tv_sec: 1671045523, tv_nsec: 398350649 }), ctime: Ctime(SystemTime { tv_sec: 1671045523, tv_nsec: 398350649 }) }) +Mkfile(Mkfile { path: TemporaryPath("o263-720050-0"), ino: Ino(263) }) +Rename(Rename { from: "o263-720050-0", to: "hello/lorem" }) +Utimes(Utimes { path: "hello", atime: Atime(SystemTime { tv_sec: 1671045523, tv_nsec: 391350615 }), mtime: Mtime(SystemTime { tv_sec: 1671045523, tv_nsec: 410350708 }), ctime: Ctime(SystemTime { tv_sec: 1671045523, tv_nsec: 410350708 }) }) +Write(Write { path: "hello/lorem", offset: FileOffset(0), data: "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do " "consectetur adipiscing elit, sed do eiusmod tempor incididunt ut" }) +Write(Write { path: "hello/lorem", offset: FileOffset(49152), data: " labore et dolore magna aliqua. Ut enim ad minim veniam, quis no" "ua. Ut enim ad minim veniam, quis nostrud exercitation ullamco l" }) +Write(Write { path: "hello/lorem", offset: FileOffset(98304), data: "aboris nisi ut aliquip ex ea commodo consequat. Duis aute irure " "tur. Excepteur sint occaecat cupidatat non proident, sunt in cul" }) +Write(Write { path: "hello/lorem", offset: FileOffset(131072), data: "pa qui officia deserunt mollit anim id est laborum.\nLorem ipsum " "it anim id est laborum.\nLorem ipsum dolor sit amet, consectetur " }) +Write(Write { path: "hello/lorem", offset: FileOffset(180224), data: "adipiscing elit, sed do eiusmod tempor incididunt ut labore et d" " sunt in culpa qui officia deserunt mollit anim id est laborum.\n" }) +Chown(Chown { path: "hello/lorem", uid: Uid(0), gid: Gid(0) }) +Chmod(Chmod { path: "hello/lorem", mode: Mode { permissions: Permissions(FilePermissions { mode: 420 }), type: (empty) } }) +Utimes(Utimes { path: "hello/lorem", atime: Atime(SystemTime { tv_sec: 1671045523, tv_nsec: 398350649 }), mtime: Mtime(SystemTime { tv_sec: 1671045523, tv_nsec: 409350703 }), ctime: Ctime(SystemTime { tv_sec: 1671045523, tv_nsec: 409350703 }) }) +Mkfile(Mkfile { path: TemporaryPath("o264-720050-0"), ino: Ino(264) }) +Rename(Rename { from: "o264-720050-0", to: "hello/lorem-reflinked" }) +Utimes(Utimes { path: "hello", atime: Atime(SystemTime { tv_sec: 1671045523, tv_nsec: 391350615 }), mtime: Mtime(SystemTime { tv_sec: 1671045523, tv_nsec: 410350708 }), ctime: Ctime(SystemTime { tv_sec: 1671045523, tv_nsec: 410350708 }) }) +Clone(Clone { src_offset: FileOffset(0), len: CloneLen(131072), src_path: "hello/lorem", uuid: 0fbf2b5f-ff82-a748-8b41-e35aec190b49, ctransid: Ctransid(720050), dst_path: "hello/lorem-reflinked", dst_offset: FileOffset(0) }) +Write(Write { path: "hello/lorem-reflinked", offset: FileOffset(131072), data: "pa qui officia deserunt mollit anim id est laborum.\nLorem ipsum " "it anim id est laborum.\nLorem ipsum dolor sit amet, consectetur " }) +Write(Write { path: "hello/lorem-reflinked", offset: FileOffset(180224), data: "adipiscing elit, sed do eiusmod tempor incididunt ut labore et d" " sunt in culpa qui officia deserunt mollit anim id est laborum.\n" }) +Chown(Chown { path: "hello/lorem-reflinked", uid: Uid(0), gid: Gid(0) }) +Chmod(Chmod { path: "hello/lorem-reflinked", mode: Mode { permissions: Permissions(FilePermissions { mode: 420 }), type: (empty) } }) +Utimes(Utimes { path: "hello/lorem-reflinked", atime: Atime(SystemTime { tv_sec: 1671045523, tv_nsec: 410350708 }), mtime: Mtime(SystemTime { tv_sec: 1671045523, tv_nsec: 411350713 }), ctime: Ctime(SystemTime { tv_sec: 1671045523, tv_nsec: 411350713 }) }) +Mkfile(Mkfile { path: TemporaryPath("o265-720050-0"), ino: Ino(265) }) +Rename(Rename { from: "o265-720050-0", to: "huge-empty-file" }) +Utimes(Utimes { path: "", atime: Atime(SystemTime { tv_sec: 1671045523, tv_nsec: 426350787 }), mtime: Mtime(SystemTime { tv_sec: 1671045523, tv_nsec: 434350827 }), ctime: Ctime(SystemTime { tv_sec: 1671045523, tv_nsec: 434350827 }) }) +Truncate(Truncate { path: "huge-empty-file", size: 107374182400 }) +Chown(Chown { path: "huge-empty-file", uid: Uid(0), gid: Gid(0) }) +Chmod(Chmod { path: "huge-empty-file", mode: Mode { permissions: Permissions(FilePermissions { mode: 420 }), type: (empty) } }) +Utimes(Utimes { path: "huge-empty-file", atime: Atime(SystemTime { tv_sec: 1671045523, tv_nsec: 412350718 }), mtime: Mtime(SystemTime { tv_sec: 1671045523, tv_nsec: 412350718 }), ctime: Ctime(SystemTime { tv_sec: 1671045523, tv_nsec: 412350718 }) }) +Mknod(Mknod(Mkspecial { path: TemporaryPath("o266-720050-0"), ino: Ino(266), rdev: Rdev(259), mode: Mode { permissions: Permissions(FilePermissions { mode: 8612 }), type: S_IFCHR } })) +Rename(Rename { from: "o266-720050-0", to: "null" }) +Utimes(Utimes { path: "", atime: Atime(SystemTime { tv_sec: 1671045523, tv_nsec: 426350787 }), mtime: Mtime(SystemTime { tv_sec: 1671045523, tv_nsec: 434350827 }), ctime: Ctime(SystemTime { tv_sec: 1671045523, tv_nsec: 434350827 }) }) +Chown(Chown { path: "null", uid: Uid(0), gid: Gid(0) }) +Chmod(Chmod { path: "null", mode: Mode { permissions: Permissions(FilePermissions { mode: 420 }), type: (empty) } }) +Utimes(Utimes { path: "null", atime: Atime(SystemTime { tv_sec: 1671045523, tv_nsec: 413350723 }), mtime: Mtime(SystemTime { tv_sec: 1671045523, tv_nsec: 413350723 }), ctime: Ctime(SystemTime { tv_sec: 1671045523, tv_nsec: 413350723 }) }) +Mksock(Mksock(Mkspecial { path: TemporaryPath("o267-720050-0"), ino: Ino(267), rdev: Rdev(0), mode: Mode { permissions: Permissions(FilePermissions { mode: 49645 }), type: S_IFDIR | S_IFREG | S_IFSOCK } })) +Rename(Rename { from: "o267-720050-0", to: "socket-node.sock" }) +Utimes(Utimes { path: "", atime: Atime(SystemTime { tv_sec: 1671045523, tv_nsec: 426350787 }), mtime: Mtime(SystemTime { tv_sec: 1671045523, tv_nsec: 434350827 }), ctime: Ctime(SystemTime { tv_sec: 1671045523, tv_nsec: 434350827 }) }) +Chown(Chown { path: "socket-node.sock", uid: Uid(0), gid: Gid(0) }) +Chmod(Chmod { path: "socket-node.sock", mode: Mode { permissions: Permissions(FilePermissions { mode: 493 }), type: (empty) } }) +Utimes(Utimes { path: "socket-node.sock", atime: Atime(SystemTime { tv_sec: 1671045523, tv_nsec: 434350827 }), mtime: Mtime(SystemTime { tv_sec: 1671045523, tv_nsec: 434350827 }), ctime: Ctime(SystemTime { tv_sec: 1671045523, tv_nsec: 434350827 }) }) +End +END SENDSTREAM 0 +BEGIN SENDSTREAM 1 +Snapshot(Snapshot { path: "demo-undo", uuid: ed2c87d3-12e3-c549-a699-635de66d6f35, ctransid: Ctransid(720053), clone_uuid: 0fbf2b5f-ff82-a748-8b41-e35aec190b49, clone_ctransid: Ctransid(720050) }) +Utimes(Utimes { path: "", atime: Atime(SystemTime { tv_sec: 1671045523, tv_nsec: 426350787 }), mtime: Mtime(SystemTime { tv_sec: 1671045523, tv_nsec: 789352576 }), ctime: Ctime(SystemTime { tv_sec: 1671045523, tv_nsec: 789352576 }) }) +RemoveXattr(RemoveXattr { path: "hello/msg", name: XattrName([117, 115, 101, 114, 46, 97, 110, 116, 108, 105, 114, 46, 100, 101, 109, 111]) }) +Write(Write { path: "hello/msg", offset: FileOffset(0), data: "Goodbye!\n" }) +Truncate(Truncate { path: "hello/msg", size: 9 }) +Utimes(Utimes { path: "hello/msg", atime: Atime(SystemTime { tv_sec: 1671045523, tv_nsec: 391350615 }), mtime: Mtime(SystemTime { tv_sec: 1671045523, tv_nsec: 790352581 }), ctime: Ctime(SystemTime { tv_sec: 1671045523, tv_nsec: 790352581 }) }) +Unlink(Unlink { path: "to-be-deleted" }) +Utimes(Utimes { path: "", atime: Atime(SystemTime { tv_sec: 1671045523, tv_nsec: 426350787 }), mtime: Mtime(SystemTime { tv_sec: 1671045523, tv_nsec: 789352576 }), ctime: Ctime(SystemTime { tv_sec: 1671045523, tv_nsec: 789352576 }) }) +Rmdir(Rmdir { path: "dir-to-be-deleted" }) +Utimes(Utimes { path: "", atime: Atime(SystemTime { tv_sec: 1671045523, tv_nsec: 426350787 }), mtime: Mtime(SystemTime { tv_sec: 1671045523, tv_nsec: 789352576 }), ctime: Ctime(SystemTime { tv_sec: 1671045523, tv_nsec: 789352576 }) }) +End +END SENDSTREAM 1 diff --git a/antlir/antlir2/sendstream_parser/testdata/make-sendstream.sh b/antlir/antlir2/sendstream_parser/testdata/make-sendstream.sh new file mode 100755 index 00000000000..fa02ca4e3b9 --- /dev/null +++ b/antlir/antlir2/sendstream_parser/testdata/make-sendstream.sh @@ -0,0 +1,63 @@ +#!/bin/bash +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +set -ex + +out="$(realpath "$2")" + +pushd "$1" + +btrfs subvolume delete demo demo-undo || true + +btrfs subvolume create demo +pushd demo +mkdir hello +echo "Hello world!" > hello/msg +chmod 0400 hello/msg +setfattr -n user.antlir.demo -v 'lorem ipsum' hello/msg +chown root:root hello/msg +mkfifo myfifo +ln -s hello/msg hello/msg-sym +ln hello/msg hello/msg-hard +touch to-be-deleted +mkdir dir-to-be-deleted +# larger example file that will not fit in the inode struct so will exercise +# reflinking +lorem="Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum." +echo "$lorem" > hello/lorem +set +x +for _i in {1..500} +do + echo "$lorem" >> hello/lorem +done +set -x +cp --reflink=always hello/lorem hello/lorem-reflinked +truncate -s100G huge-empty-file +mknod null c 1 3 +python3 -c "import socket as s; sock = s.socket(s.AF_UNIX); sock.bind('socket-node.sock')" + +popd + +btrfs subvolume snapshot demo demo-undo +btrfs property set demo ro true + +pushd demo-undo + +rm to-be-deleted +rmdir dir-to-be-deleted +setfattr --remove user.antlir.demo hello/msg +echo "Goodbye!" > hello/msg + +popd + +btrfs property set demo-undo ro true + +btrfs send demo -f "$out.1" +btrfs send -p demo demo-undo -f "$out.2" +cat "$out.1" "$out.2" > "$out" +rm "$out.1" "$out.2" + +popd