docs: improve documentation of protobuf to YAML serialization options

Also make the YAML formatting option more compact.

docs/Module Developer's Guide.md

@@ -41,11 +41,11 @@ directory. As a starting point we can use the following file:
 ```protobuf
 syntax = "proto2";
 
-import "YARA.proto";
+import "yara.proto";
 
 package text;
 
-option (YARA.module_options) = {
+option (yara.module_options) = {
   name : "text"
   root_message: "text.Text"
   rust_module: "text"
@@ -146,27 +146,6 @@ actually field tags (i.e: a unique number identifying each field in a message).
 This may be confusing if you are not familiar with protobuf's  syntax, so again:
 explore the protobuf's [documentation](https://developers.google.com/protocol-buffers). 
 
-One thing that can be done with integer fields is to represent them in some other way.
-This optional representation is shown in `yr dump` crate output. This crate provides
-two output formats: JSON and YAML. Both can be shown in colored output via `-c|--color` option.
-The last mentioned also provides custom representation for integer numbers. Let's say
-for some fields it makes sense to show them as hexadecimal numbers. This can be done by
-adding `[(yara.field_options).yaml_fmt = "<format>"];` descriptor to the field. 
-Currently supported formats are: hexadecimal number and human-readable timestamp.
-For example:
-
-```
-message Macho {
-  optional uint32 magic = 1 [(yara.field_options).yml_fmt= "x"];
-}
-```
-
-This will mark magic field as a hexadecimal number and it will be shown as 
-`magic: 0xfeedfacf` instead of `4277009103`. Other format that is supported right now is
-for timestamps. If you want to show some integer field as a timestamp you can do it by
-setting `[(yara.field_options).yml_fmt = "t"];` descriptor to the field and
-human readable timestamps will be shown in YAML comment after its integer value.
-
 Also notice that we are defining our fields as `optional`. In `proto2` fields
 must be either `optional` or `required`, while in `proto3` they are always
 optional and can't be forced to be required. We are going to discuss this topic
@@ -237,6 +216,41 @@ The bottom line is that with `proto3` you won't be able to have fields with
 is very useful in such cases, as you don't need to explicitly initialize all
 the fields in your structure.
 
+## Tweaking the module's YAML output
+
+The `yr dump` command outputs the structure generated by one or more YARA 
+modules, presenting the information in either JSON or YAML format. The default 
+output format is YAML, because of its inherent human-friendly nature. 
+Nevertheless, you can help YARA to further enhance the quality of produced
+YAML outputs.
+
+Certain integer fields find a more intuitive representation in hexadecimal 
+rather than decimal format. To communicate this preference to YARA, a dedicated 
+configuration option can be employed. Consider the following illustrative 
+example:
+
+```
+message Macho {
+  optional uint32 magic = 1 [(yaml.field).fmt = "x"];
+}
+```
+
+Here, `[(yaml.field).fmt = "x"]` instructs YARA to portray the magic field in 
+hexadecimal format (i.e., "x"). Consequently, the output will display 
+`magic: 0xfeedfacf` instead of the less readable `magic: 4277009103`.
+
+Supported format options also includes `"t"` for timestamps. For example:
+
+```
+optional uint32 my_timestamp = 1 [(yaml.field).fmt = "t"];
+```
+
+In this scenario, the output would be rendered as follows:
+
+```yaml
+my_timestamp: 999999999 # 2001-09-09 01:46:39 UTC
+```
+
 ## Implementing the module's main function
 
 Once you have a `.proto` file that describes the structure of your module you

yara-x-proto-yaml/src/lib.rs

@@ -1,3 +1,54 @@
+/*! Serializes a Protocol Buffer (protobuf) message to YAML.
+
+This crates serializes arbitrary protobuf messages to YAML format, producing
+YAML that is user-friendly, customizable and colorful. Some aspects of the
+produced YAML can be customized by using specific options in your `.proto`
+files. Let's use the following protobuf message definition as an example:
+
+```protobuf
+import "yaml.proto";
+
+message MyMessage {
+  optional int32 some_field = 1 [(yaml.field).fmt = "x"];
+}
+```
+
+The first think to note is the `import "yaml.proto"` statement before the
+message definition. The `yaml.proto` file defines the existing YAML formatting
+options, so you must include it in your own `.proto` file in order to be able
+to use the such options.
+
+The `[(yaml.field).fmt = "x"]` modifier, when applied to some field, indicates
+that values of that field must be rendered in hexadecimal form. The list of
+supported format modifiers is:
+
+- `x`: Serializes the value an hexadecimal number. Only valid for integer
+       fields.
+- `t`: Serializes the field as a timestamp. The value itself is rendered as a
+       decimal integer, but a comment is added with the timestamp in a
+       human-friendly format. Only valid for integer fields.
+
+# Examples
+
+Protobuf definition:
+
+```protobuf
+import "yaml.proto";
+
+message MyMessage {
+  optional int32 some_field = 1 [(yaml.field).fmt = "x"];
+  optional int64 some_timestamp = 2 [(yaml.field).fmt = "x"];
+}
+```
+
+YAML output:
+
+```yaml
+some_field: 0x8b1;
+timestamp: 999999999 # 2001-09-09 01:46:39 UTC
+```
+ */
+
 use chrono::prelude::{DateTime, NaiveDateTime, Utc};
 use itertools::Itertools;
 use protobuf::MessageDyn;
@@ -12,7 +63,7 @@ use protobuf::reflect::ReflectFieldRef::{Map, Optional, Repeated};
 use protobuf::reflect::ReflectValueRef;
 use protobuf::reflect::{FieldDescriptor, MessageRef};
 
-use crate::yaml::exts::field_options;
+use crate::yaml::exts::field as field_options;
 
 #[cfg(test)]
 mod tests;
@@ -31,7 +82,7 @@ impl ColorsConfig {
     const COMMENT: Color = Color::RGB(222, 184, 135); // Brown
 }
 
-// A struct that represents options for a field values
+/// A struct that represents options for a field values
 #[derive(Debug, Default, Clone)]
 struct ValueOptions {
     is_hex: bool,
@@ -69,8 +120,8 @@ impl<W: Write> Serializer<W> {
             .get(&field_descriptor.options)
             .map(|options| ValueOptions {
                 // Default for boolean is false
-                is_hex: options.yaml_fmt() == "x",
-                is_timestamp: options.yaml_fmt() == "t",
+                is_hex: options.fmt() == "x",
+                is_timestamp: options.fmt() == "t",
             })
             .unwrap_or_default()
     }

yara-x-proto-yaml/src/tests/test.proto

@@ -11,8 +11,8 @@ message SubMessage {
 }
 
 message Message {
-  optional int32 int32_hex = 1 [(yaml.field_options).yaml_fmt = "x"];
-  optional int64 timestamp = 2 [(yaml.field_options).yaml_fmt = "t"];
+  optional int32 int32_hex = 1 [(yaml.field).fmt = "x"];
+  optional int64 timestamp = 2 [(yaml.field).fmt = "t"];
   optional int32 int32_dec = 3;
   optional string str = 4;
   repeated SubMessage repeated_msg = 5;

yara-x-proto-yaml/src/yaml.proto

@@ -5,9 +5,9 @@ package yaml;
 import "google/protobuf/descriptor.proto";
 
 message FieldOptions {
-    optional string yaml_fmt = 3;
-  }
+    optional string fmt = 3;
+}
 
 extend google.protobuf.FieldOptions {
-    optional FieldOptions field_options = 51504;
-  }
+    optional FieldOptions field = 51504;
+}

yara-x-proto/src/yara.proto

@@ -16,7 +16,6 @@ message ModuleOptions {
 message FieldOptions {
   optional string name = 1;
   optional bool ignore = 2;
-  optional string yaml_fmt = 3;
 }
 
 message MessageOptions {

yara-x/build.rs

@@ -21,10 +21,15 @@ fn main() {
         .cargo_out_dir("protos")
         //.out_dir("src/modules/protos")
         .include("../yara-x-proto/src")
+        .include("../yara-x-proto-yaml/src")
         .include("src/modules/protos")
-        .input("../yara-x-proto/src/yara.proto");
+        .input("../yara-x-proto/src/yara.proto")
+        .input("../yara-x-proto-yaml/src/yaml.proto");
 
-    proto_parser.include("../yara-x-proto/src").include("src/modules/protos");
+    proto_parser
+        .include("../yara-x-proto/src")
+        .include("../yara-x-proto-yaml/src")
+        .include("src/modules/protos");
 
     for entry in fs::read_dir("src/modules/protos").unwrap() {
         let entry = entry.unwrap();

yara-x/src/modules/protos/macho.proto

@@ -1,5 +1,6 @@
 syntax = "proto2";
 import "yara.proto";
+import "yaml.proto";
 
 package macho;
 
@@ -11,7 +12,7 @@ option (yara.module_options) = {
 
 message Dylib {
   optional string name = 1;
-  optional uint32 timestamp = 2 [(yara.field_options).yaml_fmt = "t"];
+  optional uint32 timestamp = 2 [(yaml.field).fmt = "t"];
   optional string compatibility_version = 3;
   optional string current_version = 4;
 }
@@ -25,13 +26,13 @@ message RPath {
 message Section {
   optional string segname = 1;
   optional string sectname = 2;
-  optional uint64 addr = 3 [(yara.field_options).yaml_fmt = "x"];
-  optional uint64 size = 4 [(yara.field_options).yaml_fmt = "x"];
+  optional uint64 addr = 3 [(yaml.field).fmt = "x"];
+  optional uint64 size = 4 [(yaml.field).fmt = "x"];
   optional uint32 offset = 5;
   optional uint32 align = 6;
   optional uint32 reloff = 7;
   optional uint32 nreloc = 8;
-  optional uint32 flags = 9 [(yara.field_options).yaml_fmt = "x"];
+  optional uint32 flags = 9 [(yaml.field).fmt = "x"];
   optional uint32 reserved1 = 10;
   optional uint32 reserved2 = 11;
   optional uint32 reserved3 = 12;
@@ -41,14 +42,14 @@ message Segment {
   optional uint32 cmd = 1;
   optional uint32 cmdsize = 2;
   optional string segname = 3;
-  optional uint64 vmaddr = 4 [(yara.field_options).yaml_fmt = "x"];
-  optional uint64 vmsize = 5 [(yara.field_options).yaml_fmt = "x"];
+  optional uint64 vmaddr = 4 [(yaml.field).fmt = "x"];
+  optional uint64 vmsize = 5 [(yaml.field).fmt = "x"];
   optional uint64 fileoff = 6;
   optional uint64 filesize = 7;
-  optional uint32 maxprot = 8 [(yara.field_options).yaml_fmt = "x"];
-  optional uint32 initprot = 9 [(yara.field_options).yaml_fmt = "x"];
+  optional uint32 maxprot = 8 [(yaml.field).fmt = "x"];
+  optional uint32 initprot = 9 [(yaml.field).fmt = "x"];
   optional uint32 nsects = 10;
-  optional uint32 flags = 11 [(yara.field_options).yaml_fmt = "x"];
+  optional uint32 flags = 11 [(yaml.field).fmt = "x"];
   repeated Section sections = 12;
 }
 
@@ -62,13 +63,13 @@ message FatArch {
 }
 
 message File {
-  optional uint32 magic = 1 [(yara.field_options).yaml_fmt = "x"];
+  optional uint32 magic = 1 [(yaml.field).fmt = "x"];
   optional uint32 cputype = 2;
   optional uint32 cpusubtype = 3;
   optional uint32 filetype = 4;
   optional uint32 ncmds = 5;
   optional uint32 sizeofcmds = 6;
-  optional uint32 flags = 7 [(yara.field_options).yaml_fmt = "x"];
+  optional uint32 flags = 7 [(yaml.field).fmt = "x"];
   optional uint32 reserved = 8;
   optional uint64 number_of_segments = 9;
   repeated Segment segments = 10;
@@ -80,13 +81,13 @@ message File {
 
 message Macho {
   // Set Mach-O header and basic fields
-  optional uint32 magic = 1 [(yara.field_options).yaml_fmt = "x"];
+  optional uint32 magic = 1 [(yaml.field).fmt = "x"];
   optional uint32 cputype = 2;
   optional uint32 cpusubtype = 3;
   optional uint32 filetype = 4;
   optional uint32 ncmds = 5;
   optional uint32 sizeofcmds = 6;
-  optional uint32 flags = 7 [(yara.field_options).yaml_fmt = "x"];
+  optional uint32 flags = 7 [(yaml.field).fmt = "x"];
   optional uint32 reserved = 8;
   optional uint64 number_of_segments = 9;
   repeated Segment segments = 10;
@@ -96,7 +97,7 @@ message Macho {
   optional uint64 stack_size = 14;
 
   // Add fields for Mach-O fat binary header
-  optional uint32 fat_magic = 15 [(yara.field_options).yaml_fmt = "x"];
+  optional uint32 fat_magic = 15 [(yaml.field).fmt = "x"];
   optional uint32 nfat_arch = 16;
   repeated FatArch fat_arch = 17;