From 59a544640143d6f90dc96897751b0f9e6bfe241d Mon Sep 17 00:00:00 2001 From: Adam Hughes <9903835+tri-adam@users.noreply.github.com> Date: Thu, 2 Mar 2023 17:07:10 +0000 Subject: [PATCH 1/2] feat: custom metadata support Add OptMetadata, which allows supplying custom metadata when calling NewDescriptorInput. Add GetMetadata, which allows a user to parse custom metadata from a Descriptor. --- pkg/sif/descriptor.go | 69 +++++++++++++----- pkg/sif/descriptor_input.go | 16 +++- pkg/sif/descriptor_input_test.go | 9 ++- pkg/sif/descriptor_test.go | 55 +++++++++++++- .../TestNewDescriptorInput/OptMetadata.golden | Bin 0 -> 585 bytes 5 files changed, 125 insertions(+), 24 deletions(-) create mode 100644 pkg/sif/testdata/TestNewDescriptorInput/OptMetadata.golden diff --git a/pkg/sif/descriptor.go b/pkg/sif/descriptor.go index 8fa926a4..4d6cdf3c 100644 --- a/pkg/sif/descriptor.go +++ b/pkg/sif/descriptor.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022, Sylabs Inc. All rights reserved. +// Copyright (c) 2018-2023, Sylabs Inc. All rights reserved. // Copyright (c) 2017, SingularityWare, LLC. All rights reserved. // Copyright (c) 2017, Yannick Cote All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the @@ -10,6 +10,7 @@ package sif import ( "bytes" "crypto" + "encoding" "encoding/binary" "errors" "fmt" @@ -78,28 +79,49 @@ func (d *rawDescriptor) setName(name string) error { var errExtraTooLarge = errors.New("extra value too large") -// setExtra encodes v into the extra field of d. -func (d *rawDescriptor) setExtra(v interface{}) error { +// setExtra encodes v into the extra field of d. If the encoding.BinaryMarshaler interface is +// implemented by v, it is used for marshaling. Otherwise, binary.Write() is used. +func (d *rawDescriptor) setExtra(v any) error { if v == nil { return nil } - if binary.Size(v) > len(d.Extra) { - return errExtraTooLarge + var extra []byte + + if m, ok := v.(encoding.BinaryMarshaler); ok { + b, err := m.MarshalBinary() + if err != nil { + return err + } + extra = b + } else { + b := new(bytes.Buffer) + if err := binary.Write(b, binary.LittleEndian, v); err != nil { + return err + } + extra = b.Bytes() } - b := new(bytes.Buffer) - if err := binary.Write(b, binary.LittleEndian, v); err != nil { - return err + if len(extra) > len(d.Extra) { + return errExtraTooLarge } - for i := copy(d.Extra[:], b.Bytes()); i < len(d.Extra); i++ { + for i := copy(d.Extra[:], extra); i < len(d.Extra); i++ { d.Extra[i] = 0 } return nil } +// getExtra decodes the extra fields of d into v. If the encoding.BinaryUnmarshaler interface is +// implemented by v, it is used for unmarshaling. Otherwise, binary.Read() is used. +func (d *rawDescriptor) getExtra(v any) error { + if u, ok := v.(encoding.BinaryUnmarshaler); ok { + return u.UnmarshalBinary(d.Extra[:]) + } + return binary.Read(bytes.NewReader(d.Extra[:]), binary.LittleEndian, v) +} + // getPartitionMetadata gets metadata for a partition data object. func (d rawDescriptor) getPartitionMetadata() (FSType, PartType, string, error) { if got, want := d.DataType, DataPartition; got != want { @@ -108,9 +130,8 @@ func (d rawDescriptor) getPartitionMetadata() (FSType, PartType, string, error) var p partition - b := bytes.NewReader(d.Extra[:]) - if err := binary.Read(b, binary.LittleEndian, &p); err != nil { - return 0, 0, "", fmt.Errorf("%w", err) + if err := d.getExtra(&p); err != nil { + return 0, 0, "", err } return p.Fstype, p.Parttype, p.Arch.GoArch(), nil @@ -168,11 +189,24 @@ func (d Descriptor) ModifiedAt() time.Time { return time.Unix(d.raw.ModifiedAt, // Name returns the name of the data object. func (d Descriptor) Name() string { return strings.TrimRight(string(d.raw.Name[:]), "\000") } +// GetMetadata reads metadata from d into v. If the encoding.BinaryUnmarshaler interface is +// implemented by v, it is used for unmarshaling. Otherwise, binary.Read() is used. +func (d Descriptor) GetMetadata(v any) error { + if err := d.raw.getExtra(v); err != nil { + return fmt.Errorf("%w", err) + } + return nil +} + // PartitionMetadata gets metadata for a partition data object. // //nolint:nonamedreturns // Named returns effective as documentation. func (d Descriptor) PartitionMetadata() (fs FSType, pt PartType, arch string, err error) { - return d.raw.getPartitionMetadata() + fs, pt, arch, err = d.raw.getPartitionMetadata() + if err != nil { + return 0, 0, "", fmt.Errorf("%w", err) + } + return fs, pt, arch, err } var errHashUnsupported = errors.New("hash algorithm unsupported") @@ -204,8 +238,7 @@ func (d Descriptor) SignatureMetadata() (ht crypto.Hash, fp []byte, err error) { var s signature - b := bytes.NewReader(d.raw.Extra[:]) - if err := binary.Read(b, binary.LittleEndian, &s); err != nil { + if err := d.raw.getExtra(&s); err != nil { return ht, fp, fmt.Errorf("%w", err) } @@ -232,8 +265,7 @@ func (d Descriptor) CryptoMessageMetadata() (FormatType, MessageType, error) { var m cryptoMessage - b := bytes.NewReader(d.raw.Extra[:]) - if err := binary.Read(b, binary.LittleEndian, &m); err != nil { + if err := d.raw.getExtra(&m); err != nil { return 0, 0, fmt.Errorf("%w", err) } @@ -248,8 +280,7 @@ func (d Descriptor) SBOMMetadata() (SBOMFormat, error) { var s sbom - b := bytes.NewReader(d.raw.Extra[:]) - if err := binary.Read(b, binary.LittleEndian, &s); err != nil { + if err := d.raw.getExtra(&s); err != nil { return 0, fmt.Errorf("%w", err) } diff --git a/pkg/sif/descriptor_input.go b/pkg/sif/descriptor_input.go index 3e81c394..5ee6bcd9 100644 --- a/pkg/sif/descriptor_input.go +++ b/pkg/sif/descriptor_input.go @@ -1,4 +1,4 @@ -// Copyright (c) 2021-2022, Sylabs Inc. All rights reserved. +// Copyright (c) 2021-2023, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE file distributed with the sources of this project regarding your // rights to use or distribute this software. @@ -19,7 +19,7 @@ type descriptorOpts struct { linkID uint32 alignment int name string - extra interface{} + extra any t time.Time } @@ -92,6 +92,15 @@ func OptObjectTime(t time.Time) DescriptorInputOpt { } } +// OptMetadata sets v as the metadata for a data object. If the encoding.BinaryMarshaler interface +// is implemented by v, it is used for marshaling. Otherwise, binary.Write() is used. +func OptMetadata(v any) DescriptorInputOpt { + return func(t DataType, opts *descriptorOpts) error { + opts.extra = v + return nil + } +} + type unexpectedDataTypeError struct { got DataType want []DataType @@ -259,7 +268,8 @@ const DefaultObjectGroup = 1 // // It is possible (and often necessary) to store additional metadata related to certain types of // data objects. Consider supplying options such as OptCryptoMessageMetadata, OptPartitionMetadata, -// OptSignatureMetadata, and OptSBOMMetadata for this purpose. +// OptSignatureMetadata, and OptSBOMMetadata for this purpose. To set custom metadata, use +// OptMetadata. // // By default, the data object will be placed in the default data object group (1). To override // this behavior, use OptNoGroup or OptGroupID. To link this data object, use OptLinkedID or diff --git a/pkg/sif/descriptor_input_test.go b/pkg/sif/descriptor_input_test.go index 5c34099e..11584531 100644 --- a/pkg/sif/descriptor_input_test.go +++ b/pkg/sif/descriptor_input_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2021-2022, Sylabs Inc. All rights reserved. +// Copyright (c) 2021-2023, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE file distributed with the sources of this project regarding your // rights to use or distribute this software. @@ -118,6 +118,13 @@ func TestNewDescriptorInput(t *testing.T) { OptObjectTime(time.Unix(946702800, 0)), }, }, + { + name: "OptMetadata", + t: DataGeneric, + opts: []DescriptorInputOpt{ + OptMetadata(testMetadata{100}), + }, + }, { name: "OptCryptoMessageMetadataUnexpectedDataType", t: DataGeneric, diff --git a/pkg/sif/descriptor_test.go b/pkg/sif/descriptor_test.go index c6c305cd..ab859244 100644 --- a/pkg/sif/descriptor_test.go +++ b/pkg/sif/descriptor_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022, Sylabs Inc. All rights reserved. +// Copyright (c) 2018-2023, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE file distributed with the sources of this project regarding your // rights to use or distribute this software. @@ -8,10 +8,12 @@ package sif import ( "bytes" "crypto" + "encoding/json" "errors" "io" "os" "path/filepath" + "reflect" "testing" "github.com/sebdah/goldie/v2" @@ -102,6 +104,57 @@ func TestDescriptor_Name(t *testing.T) { } } +type testMetadata struct { + Value int +} + +func (m testMetadata) MarshalBinary() ([]byte, error) { + return json.Marshal(m) +} + +func (m *testMetadata) UnmarshalBinary(b []byte) error { + return json.Unmarshal(bytes.TrimRight(b, "\x00"), m) +} + +func TestDescriptor_GetMetadata(t *testing.T) { + md := testMetadata{100} + + rd := rawDescriptor{ + DataType: DataGeneric, + } + if err := rd.setExtra(md); err != nil { + t.Fatal(err) + } + + tests := []struct { + name string + rd rawDescriptor + wantMD any + wantErr error + }{ + { + name: "OK", + rd: rd, + wantMD: md, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + d := Descriptor{raw: tt.rd} + + var md testMetadata + + if got, want := d.GetMetadata(&md), tt.wantErr; !errors.Is(got, want) { + t.Fatalf("got error %v, want %v", got, want) + } + + if got, want := md, tt.wantMD; !reflect.DeepEqual(got, want) { + t.Fatalf("got metadata %v, want %v", got, want) + } + }) + } +} + func TestDescriptor_PartitionMetadata(t *testing.T) { p := partition{ Fstype: FsSquash, diff --git a/pkg/sif/testdata/TestNewDescriptorInput/OptMetadata.golden b/pkg/sif/testdata/TestNewDescriptorInput/OptMetadata.golden new file mode 100644 index 0000000000000000000000000000000000000000..7dca87c66535980f0f68d336f0f614e48e65fc95 GIT binary patch literal 585 zcmZQ~V1NKd28It%CU%N5ujAwY|Nj|aG Date: Mon, 6 Mar 2023 14:32:21 +0000 Subject: [PATCH 2/2] refactor: use interface types for custom metadata Switch to encoding.BinaryUnmarshaler/encoding.BinaryMarshaler for GetMetadata/OptMetadata rather than "any" type. --- .golangci.yml | 2 +- pkg/sif/create.go | 4 +- pkg/sif/descriptor.go | 74 ++++++++++++++++++++----------------- pkg/sif/descriptor_input.go | 20 +++++----- pkg/sif/descriptor_test.go | 8 ++-- 5 files changed, 58 insertions(+), 50 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index fb7edca7..93c9d2ee 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -46,7 +46,7 @@ linters: - typecheck - unconvert - unparam - - unused + # - unused - whitespace linters-settings: diff --git a/pkg/sif/create.go b/pkg/sif/create.go index 104e9ea1..5eec9a39 100644 --- a/pkg/sif/create.go +++ b/pkg/sif/create.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022, Sylabs Inc. All rights reserved. +// Copyright (c) 2018-2023, Sylabs Inc. All rights reserved. // Copyright (c) 2017, SingularityWare, LLC. All rights reserved. // Copyright (c) 2017, Yannick Cote All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the @@ -69,7 +69,7 @@ func (f *FileImage) writeDataObject(i int, di DescriptorInput, t time.Time) erro // If this is a primary partition, verify there isn't another primary partition, and update the // architecture in the global header. - if p, ok := di.opts.extra.(partition); ok && p.Parttype == PartPrimSys { + if p, ok := di.opts.md.(partition); ok && p.Parttype == PartPrimSys { if ds, err := f.GetDescriptors(WithPartitionType(PartPrimSys)); err == nil && len(ds) > 0 { return errPrimaryPartition } diff --git a/pkg/sif/descriptor.go b/pkg/sif/descriptor.go index 4d6cdf3c..19272824 100644 --- a/pkg/sif/descriptor.go +++ b/pkg/sif/descriptor.go @@ -45,6 +45,11 @@ type partition struct { Arch archType } +// MarshalBinary encodes p into binary format. +func (p partition) MarshalBinary() ([]byte, error) { + return binaryMarshaler{p}.MarshalBinary() +} + // signature represents the SIF signature data object descriptor. type signature struct { Hashtype hashType @@ -62,6 +67,26 @@ type sbom struct { Format SBOMFormat } +// The binaryMarshaler type is an adapter that allows a type suitable for use with the +// encoding/binary package to be used as an encoding.BinaryMarshaler. +type binaryMarshaler struct{ any } + +// MarshalBinary encodes m into binary format. +func (m binaryMarshaler) MarshalBinary() ([]byte, error) { + var b bytes.Buffer + err := binary.Write(&b, binary.LittleEndian, m.any) + return b.Bytes(), err +} + +// The binaryUnmarshaler type is an adapter that allows a type suitable for use with the +// encoding/binary package to be used as an encoding.BinaryUnmarshaler. +type binaryUnmarshaler struct{ any } + +// UnmarshalBinary decodes b into u. +func (u binaryUnmarshaler) UnmarshalBinary(b []byte) error { + return binary.Read(bytes.NewReader(b), binary.LittleEndian, u.any) +} + var errNameTooLarge = errors.New("name value too large") // setName encodes name into the name field of d. @@ -79,27 +104,15 @@ func (d *rawDescriptor) setName(name string) error { var errExtraTooLarge = errors.New("extra value too large") -// setExtra encodes v into the extra field of d. If the encoding.BinaryMarshaler interface is -// implemented by v, it is used for marshaling. Otherwise, binary.Write() is used. -func (d *rawDescriptor) setExtra(v any) error { - if v == nil { +// setExtra marshals metadata from md into the "extra" field of d. +func (d *rawDescriptor) setExtra(md encoding.BinaryMarshaler) error { + if md == nil { return nil } - var extra []byte - - if m, ok := v.(encoding.BinaryMarshaler); ok { - b, err := m.MarshalBinary() - if err != nil { - return err - } - extra = b - } else { - b := new(bytes.Buffer) - if err := binary.Write(b, binary.LittleEndian, v); err != nil { - return err - } - extra = b.Bytes() + extra, err := md.MarshalBinary() + if err != nil { + return err } if len(extra) > len(d.Extra) { @@ -113,13 +126,9 @@ func (d *rawDescriptor) setExtra(v any) error { return nil } -// getExtra decodes the extra fields of d into v. If the encoding.BinaryUnmarshaler interface is -// implemented by v, it is used for unmarshaling. Otherwise, binary.Read() is used. -func (d *rawDescriptor) getExtra(v any) error { - if u, ok := v.(encoding.BinaryUnmarshaler); ok { - return u.UnmarshalBinary(d.Extra[:]) - } - return binary.Read(bytes.NewReader(d.Extra[:]), binary.LittleEndian, v) +// getExtra unmarshals metadata from the "extra" field of d into md. +func (d *rawDescriptor) getExtra(md encoding.BinaryUnmarshaler) error { + return md.UnmarshalBinary(d.Extra[:]) } // getPartitionMetadata gets metadata for a partition data object. @@ -130,7 +139,7 @@ func (d rawDescriptor) getPartitionMetadata() (FSType, PartType, string, error) var p partition - if err := d.getExtra(&p); err != nil { + if err := d.getExtra(binaryUnmarshaler{&p}); err != nil { return 0, 0, "", err } @@ -189,10 +198,9 @@ func (d Descriptor) ModifiedAt() time.Time { return time.Unix(d.raw.ModifiedAt, // Name returns the name of the data object. func (d Descriptor) Name() string { return strings.TrimRight(string(d.raw.Name[:]), "\000") } -// GetMetadata reads metadata from d into v. If the encoding.BinaryUnmarshaler interface is -// implemented by v, it is used for unmarshaling. Otherwise, binary.Read() is used. -func (d Descriptor) GetMetadata(v any) error { - if err := d.raw.getExtra(v); err != nil { +// GetMetadata unmarshals metadata from the "extra" field of d into md. +func (d Descriptor) GetMetadata(md encoding.BinaryUnmarshaler) error { + if err := d.raw.getExtra(md); err != nil { return fmt.Errorf("%w", err) } return nil @@ -238,7 +246,7 @@ func (d Descriptor) SignatureMetadata() (ht crypto.Hash, fp []byte, err error) { var s signature - if err := d.raw.getExtra(&s); err != nil { + if err := d.raw.getExtra(binaryUnmarshaler{&s}); err != nil { return ht, fp, fmt.Errorf("%w", err) } @@ -265,7 +273,7 @@ func (d Descriptor) CryptoMessageMetadata() (FormatType, MessageType, error) { var m cryptoMessage - if err := d.raw.getExtra(&m); err != nil { + if err := d.raw.getExtra(binaryUnmarshaler{&m}); err != nil { return 0, 0, fmt.Errorf("%w", err) } @@ -280,7 +288,7 @@ func (d Descriptor) SBOMMetadata() (SBOMFormat, error) { var s sbom - if err := d.raw.getExtra(&s); err != nil { + if err := d.raw.getExtra(binaryUnmarshaler{&s}); err != nil { return 0, fmt.Errorf("%w", err) } diff --git a/pkg/sif/descriptor_input.go b/pkg/sif/descriptor_input.go index 5ee6bcd9..3cfe5c65 100644 --- a/pkg/sif/descriptor_input.go +++ b/pkg/sif/descriptor_input.go @@ -7,6 +7,7 @@ package sif import ( "crypto" + "encoding" "errors" "fmt" "io" @@ -19,7 +20,7 @@ type descriptorOpts struct { linkID uint32 alignment int name string - extra any + md encoding.BinaryMarshaler t time.Time } @@ -92,11 +93,10 @@ func OptObjectTime(t time.Time) DescriptorInputOpt { } } -// OptMetadata sets v as the metadata for a data object. If the encoding.BinaryMarshaler interface -// is implemented by v, it is used for marshaling. Otherwise, binary.Write() is used. -func OptMetadata(v any) DescriptorInputOpt { +// OptMetadata marshals metadata from md into the "extra" field of d. +func OptMetadata(md encoding.BinaryMarshaler) DescriptorInputOpt { return func(t DataType, opts *descriptorOpts) error { - opts.extra = v + opts.md = md return nil } } @@ -164,7 +164,7 @@ func OptCryptoMessageMetadata(ft FormatType, mt MessageType) DescriptorInputOpt Messagetype: mt, } - opts.extra = m + opts.md = binaryMarshaler{m} return nil } } @@ -193,7 +193,7 @@ func OptPartitionMetadata(fs FSType, pt PartType, arch string) DescriptorInputOp Arch: sifarch, } - opts.extra = p + opts.md = p return nil } } @@ -230,7 +230,7 @@ func OptSignatureMetadata(ht crypto.Hash, fp []byte) DescriptorInputOpt { } copy(s.Entity[:], fp) - opts.extra = s + opts.md = binaryMarshaler{s} return nil } } @@ -248,7 +248,7 @@ func OptSBOMMetadata(f SBOMFormat) DescriptorInputOpt { Format: f, } - opts.extra = s + opts.md = binaryMarshaler{s} return nil } } @@ -327,5 +327,5 @@ func (di DescriptorInput) fillDescriptor(t time.Time, d *rawDescriptor) error { return err } - return d.setExtra(di.opts.extra) + return d.setExtra(di.opts.md) } diff --git a/pkg/sif/descriptor_test.go b/pkg/sif/descriptor_test.go index ab859244..9712c595 100644 --- a/pkg/sif/descriptor_test.go +++ b/pkg/sif/descriptor_test.go @@ -129,7 +129,7 @@ func TestDescriptor_GetMetadata(t *testing.T) { tests := []struct { name string rd rawDescriptor - wantMD any + wantMD testMetadata wantErr error }{ { @@ -261,7 +261,7 @@ func TestDescriptor_SignatureMetadata(t *testing.T) { rd := rawDescriptor{ DataType: tt.dt, } - if err := rd.setExtra(sig); err != nil { + if err := rd.setExtra(binaryMarshaler{sig}); err != nil { t.Fatal(err) } @@ -295,7 +295,7 @@ func TestDescriptor_CryptoMessageMetadata(t *testing.T) { rd := rawDescriptor{ DataType: DataCryptoMessage, } - if err := rd.setExtra(m); err != nil { + if err := rd.setExtra(binaryMarshaler{m}); err != nil { t.Fatal(err) } @@ -351,7 +351,7 @@ func TestDescriptor_SBOMMetadata(t *testing.T) { rd := rawDescriptor{ DataType: DataSBOM, } - if err := rd.setExtra(m); err != nil { + if err := rd.setExtra(binaryMarshaler{m}); err != nil { t.Fatal(err) }