Support MessagePack (in addition to CBOR)

With improvements to ARD XML encoding

ard/README.md

@@ -3,6 +3,8 @@ Agnostic Raw Data (ARD)
 
 This library is [also implemented in Python](https://github.com/tliron/python-ard).
 
+And check out the [ardconv](https://github.com/tliron/ardconv) ARD conversion tool.
+
 What is "agnostic raw data"?
 
 ### Agnostic
@@ -36,6 +38,9 @@ Data validation is out of scope for ARD. There's no schema. The idea is to suppo
 data of any structure and size. Once the ARD is made available other layers can validate its
 structure and otherwise process the values.
 
+This library does support such schema validation via conversion to Go structs using a
+[reflector](reflection.go).
+
 ### Data
 
 This is about *data* as opposed to the *representation of data*. What's the difference? ARD does
@@ -46,9 +51,10 @@ endiannes or precision of integers and floats, and also not concerned with chara
 ARD and Representation Formats
 ------------------------------
 
-### CBOR
+### CBOR and MessagePack
 
-[CBOR](https://cbor.io/) supports everything! The only caveat is that it is not human-readable.
+[CBOR](https://cbor.io/) and [MessagePack](https://msgpack.org/) support everything! Though note
+that they are not human-readable.
 
 ### YAML
 
@@ -70,10 +76,10 @@ support `!!omap` by default, so this use case may become less and less common.
 ### JSON
 
 JSON can be read into ARD. However, because JSON has fewer types and more limitations than YAML
-(no integers, only floats; map keys can only be strings), ARD will lose quite a bit of type
-information when translated into JSON.
+(no signed and unsigned integers, only floats; map keys can only be strings), ARD will lose quite a
+bit of type information when translated into JSON.
 
-We can overcome this challenge by extending JSON with some conventions for encoding extra types.
+We overcome this challenge by extending JSON with some conventions for encoding extra types.
 See [our conventions here](cjson.go) or
 [in the Python ARD library](https://github.com/tliron/python-ard/blob/main/ard/cjson.py).
 

ard/canonicalize.go

@@ -1,14 +1,14 @@
 package ard
 
+// TODO: not very efficient
+
 func Canonicalize(value Value) (Value, error) {
-	// Try CBOR first (faster), then YAML, and finally Compatible JSON
+	// Try CBOR first (fastest), then Compatible JSON, and finally YAML
 	if value, err := RoundtripCBOR(value); err == nil {
 		return value, nil
+	} else if value, err := RoundtripCompatibleJSON(value); err == nil {
+		return value, nil
 	} else {
-		if value, err := RoundtripYAML(value); err == nil {
-			return value, nil
-		} else {
-			return RoundtripCompatibleJSON(value)
-		}
+		return RoundtripYAML(value)
 	}
 }

ard/copy.go

@@ -35,7 +35,7 @@ func CopyToARD(value Value) (Value, error) {
 
 		default:
 			// TODO: not very efficient
-			return RoundtripCBOR(value)
+			return Roundtrip(value, "")
 		}
 	}
 }
@@ -58,7 +58,7 @@ func NormalizeStringMapsCopyToARD(value Value) (Value, error) {
 	}
 }
 
-// Will leave non-ARD types as is
+// Will leave primitive and non-ARD types as is
 func SimpleCopy(value Value) Value {
 	switch value_ := value.(type) {
 	case Map:

ard/decode.go

@@ -1,10 +1,10 @@
 package ard
 
 import (
+	"bytes"
 	"fmt"
 	"strings"
 
-	"github.com/fxamacker/cbor/v2"
 	"github.com/tliron/kutil/util"
 )
 
@@ -23,7 +23,10 @@ func Decode(code string, format string, locate bool) (Value, Locator, error) {
 		return DecodeCompatibleXML(code, locate)
 
 	case "cbor":
-		return DecodeCBOR(code, locate)
+		return DecodeCBOR(code)
+
+	case "messagepack":
+		return DecodeMessagePack(code)
 
 	default:
 		return nil, nil, fmt.Errorf("unsupported format: %q", format)
@@ -47,14 +50,18 @@ func DecodeCompatibleXML(code string, locate bool) (Value, Locator, error) {
 }
 
 // The code should be in Base64
-func DecodeCBOR(code string, locate bool) (Value, Locator, error) {
-	var value Value
-	if bytes, err := util.FromBase64(code); err == nil {
-		if err := cbor.Unmarshal(bytes, &value); err == nil {
-			return value, nil, nil
-		} else {
-			return nil, nil, err
-		}
+func DecodeCBOR(code string) (Value, Locator, error) {
+	if bytes_, err := util.FromBase64(code); err == nil {
+		return ReadCBOR(bytes.NewReader(bytes_))
+	} else {
+		return nil, nil, err
+	}
+}
+
+// The code should be in Base64
+func DecodeMessagePack(code string) (Value, Locator, error) {
+	if bytes_, err := util.FromBase64(code); err == nil {
+		return ReadMessagePack(bytes.NewReader(bytes_))
 	} else {
 		return nil, nil, err
 	}

ard/node.go

@@ -1,6 +1,8 @@
 package ard
 
-import "github.com/tliron/yamlkeys"
+import (
+	"github.com/tliron/yamlkeys"
+)
 
 //
 // Node

ard/read.go

@@ -2,12 +2,12 @@ package ard
 
 import (
 	"encoding/json"
-	"errors"
 	"fmt"
 	"io"
 
 	"github.com/beevik/etree"
 	"github.com/fxamacker/cbor/v2"
+	"github.com/tliron/kutil/util"
 	"github.com/tliron/yamlkeys"
 	"gopkg.in/yaml.v3"
 )
@@ -27,7 +27,10 @@ func Read(reader io.Reader, format string, locate bool) (Value, Locator, error)
 		return ReadCompatibleXML(reader, locate)
 
 	case "cbor":
-		return ReadCBOR(reader, locate)
+		return ReadCBOR(reader)
+
+	case "messagepack":
+		return ReadMessagePack(reader)
 
 	default:
 		return nil, nil, fmt.Errorf("unsupported format: %q", format)
@@ -83,18 +86,18 @@ func ReadCompatibleXML(reader io.Reader, locate bool) (Value, Locator, error) {
 	document := etree.NewDocument()
 	if _, err := document.ReadFrom(reader); err == nil {
 		elements := document.ChildElements()
-		if len(elements) == 1 {
+		if length := len(elements); length == 1 {
 			value, err := FromCompatibleXML(elements[0])
 			return value, nil, err
 		} else {
-			return nil, nil, errors.New("unsupported XML")
+			return nil, nil, fmt.Errorf("unsupported XML: %d documents", length)
 		}
 	} else {
 		return nil, nil, err
 	}
 }
 
-func ReadCBOR(reader io.Reader, locate bool) (Value, Locator, error) {
+func ReadCBOR(reader io.Reader) (Value, Locator, error) {
 	var value Value
 	decoder := cbor.NewDecoder(reader)
 	if err := decoder.Decode(&value); err == nil {
@@ -103,3 +106,15 @@ func ReadCBOR(reader io.Reader, locate bool) (Value, Locator, error) {
 		return nil, nil, err
 	}
 }
+
+func ReadMessagePack(reader io.Reader) (Value, Locator, error) {
+	var value Value
+	decoder := util.NewMessagePackDecoder(reader)
+	if err := decoder.Decode(&value); err == nil {
+		// The MessagePack decoder uses StringMaps, not Maps
+		value, _ := NormalizeMaps(value)
+		return value, nil, nil
+	} else {
+		return nil, nil, err
+	}
+}

ard/roundtrip.go

@@ -1,46 +1,90 @@
 package ard
 
 import (
+	"bytes"
 	"encoding/json"
+	"encoding/xml"
+	"fmt"
 	"strings"
 
 	"github.com/fxamacker/cbor/v2"
-	"github.com/tliron/yamlkeys"
+	"github.com/tliron/kutil/util"
 	"gopkg.in/yaml.v3"
 )
 
-func RoundtripCBOR(value Value) (Value, error) {
-	if bytes, err := cbor.Marshal(value); err == nil {
-		var value Value
-		if err := cbor.Unmarshal(bytes, &value); err == nil {
-			return value, nil
-		} else {
-			return nil, err
-		}
-	} else {
-		return nil, err
+// Default is CBOR
+func Roundtrip(value Value, format string) (Value, error) {
+	switch format {
+	case "yaml":
+		return RoundtripYAML(value)
+
+	case "cjson":
+		return RoundtripCompatibleJSON(value)
+
+	case "xml":
+		return RoundtripCompatibleXML(value)
+
+	case "cbor", "":
+		return RoundtripCBOR(value)
+
+	case "messagepack":
+		return RoundtripMessagePack(value)
+
+	default:
+		return nil, fmt.Errorf("unsupported format: %q", format)
 	}
 }
 
 func RoundtripYAML(value Value) (Value, error) {
 	var writer strings.Builder
 	encoder := yaml.NewEncoder(&writer)
 	if err := encoder.Encode(value); err == nil {
-		return yamlkeys.Decode(strings.NewReader(writer.String()))
+		value_, _, err := ReadYAML(strings.NewReader(writer.String()), false)
+		return value_, err
 	} else {
 		return nil, err
 	}
 }
 
 func RoundtripCompatibleJSON(value Value) (Value, error) {
+	value = EnsureCompatibleJSON(value)
 	var writer strings.Builder
 	encoder := json.NewEncoder(&writer)
-	value = EnsureCompatibleJSON(value)
 	if err := encoder.Encode(value); err == nil {
+		value_, _, err := ReadCompatibleJSON(strings.NewReader(writer.String()), false)
+		return value_, err
+	} else {
+		return nil, err
+	}
+}
+
+func RoundtripCompatibleXML(value Value) (Value, error) {
+	// Because we don't provide explicit marshalling for XML in the codebase (as we do for
+	// JSON and YAML) we must canonicalize the data before encoding it
+	if value, err := Canonicalize(value); err == nil {
+		value = ToCompatibleXML(value)
+		var writer strings.Builder
+		if _, err := writer.WriteString(xml.Header); err == nil {
+			encoder := xml.NewEncoder(&writer)
+			encoder.Indent("", "")
+			if err := encoder.Encode(value); err == nil {
+				value_, _, err := ReadCompatibleXML(strings.NewReader(writer.String()), false)
+				return value_, err
+			} else {
+				return nil, err
+			}
+		} else {
+			return nil, err
+		}
+	} else {
+		return nil, err
+	}
+}
+
+func RoundtripCBOR(value Value) (Value, error) {
+	if bytes, err := cbor.Marshal(value); err == nil {
 		var value_ Value
-		decoder := json.NewDecoder(strings.NewReader(writer.String()))
-		if err := decoder.Decode(&value_); err == nil {
-			value_, _ = FromCompatibleJSON(value_)
+		if err := cbor.Unmarshal(bytes, &value_); err == nil {
 			return value_, nil
 		} else {
 			return nil, err
@@ -49,3 +93,14 @@ func RoundtripCompatibleJSON(value Value) (Value, error) {
 		return nil, err
 	}
 }
+
+func RoundtripMessagePack(value Value) (Value, error) {
+	var buffer bytes.Buffer
+	encoder := util.NewMessagePackEncoder(&buffer)
+	if err := encoder.Encode(value); err == nil {
+		value_, _, err := ReadMessagePack(&buffer)
+		return value_, err
+	} else {
+		return nil, err
+	}
+}

ard/string.go

@@ -1,38 +1,39 @@
 package ard
 
 import (
-	"fmt"
 	"strconv"
 	"time"
+
+	"github.com/tliron/kutil/util"
 )
 
-func ValueToString(data Value) string {
-	switch data_ := data.(type) {
+func ValueToString(value Value) string {
+	switch value_ := value.(type) {
 	case bool:
-		return strconv.FormatBool(data_)
+		return strconv.FormatBool(value_)
 	case int64:
-		return strconv.FormatInt(data_, 10)
+		return strconv.FormatInt(value_, 10)
 	case int32:
-		return strconv.FormatInt(int64(data_), 10)
+		return strconv.FormatInt(int64(value_), 10)
 	case int8:
-		return strconv.FormatInt(int64(data_), 10)
+		return strconv.FormatInt(int64(value_), 10)
 	case int:
-		return strconv.FormatInt(int64(data_), 10)
+		return strconv.FormatInt(int64(value_), 10)
 	case uint64:
-		return strconv.FormatUint(data_, 10)
+		return strconv.FormatUint(value_, 10)
 	case uint32:
-		return strconv.FormatUint(uint64(data_), 10)
+		return strconv.FormatUint(uint64(value_), 10)
 	case uint8:
-		return strconv.FormatUint(uint64(data_), 10)
+		return strconv.FormatUint(uint64(value_), 10)
 	case uint:
-		return strconv.FormatUint(uint64(data_), 10)
+		return strconv.FormatUint(uint64(value_), 10)
 	case float64:
-		return strconv.FormatFloat(data_, 'g', -1, 64)
+		return strconv.FormatFloat(value_, 'g', -1, 64)
 	case float32:
-		return strconv.FormatFloat(float64(data_), 'g', -1, 32)
+		return strconv.FormatFloat(float64(value_), 'g', -1, 32)
 	case time.Time:
-		return data_.String()
+		return value_.String()
 	default:
-		return fmt.Sprintf("%s", data_)
+		return util.ToString(value)
 	}
 }

ard/validate.go

@@ -1,5 +1,6 @@
 package ard
 
 func IsValid(value Value) bool {
+	// TODO
 	return true
 }