Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat!: encoder compress timestamp with multiple local message type #439

Merged
merged 1 commit into from
Sep 16, 2024
Merged

feat!: encoder compress timestamp with multiple local message type #439

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 8 additions & 8 deletions cmd/fitactivity/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -416,10 +416,10 @@ loop:
}

headerInfo := fmt.Sprintf("interleave: %d", interleave)
headerOption := encoder.WithNormalHeader(byte(interleave))
headerOption := encoder.WithHeaderOption(encoder.HeaderOptionNormal, byte(interleave))
if compress {
headerInfo = "compress"
headerOption = encoder.WithCompressedTimestampHeader()
headerOption = encoder.WithHeaderOption(encoder.HeaderOptionCompressedTimestamp, 0)
}

select {
Expand Down Expand Up @@ -511,10 +511,10 @@ func conceal(ctx context.Context, fs *flag.FlagSet, args []string) (err error) {
}

headerInfo := fmt.Sprintf("interleave: %d", interleave)
headerOption := encoder.WithNormalHeader(byte(interleave))
headerOption := encoder.WithHeaderOption(encoder.HeaderOptionNormal, byte(interleave))
if compress {
headerInfo = "compress"
headerOption = encoder.WithCompressedTimestampHeader()
headerOption = encoder.WithHeaderOption(encoder.HeaderOptionCompressedTimestamp, 0)
}

fmt.Fprintf(os.Stderr, "- Concealing %d file(s) [first: %s m; last: %s m]\n",
Expand Down Expand Up @@ -699,10 +699,10 @@ func reduce(ctx context.Context, fs *flag.FlagSet, args []string) (err error) {
}

headerInfo := fmt.Sprintf("interleave: %d", interleave)
headerOption := encoder.WithNormalHeader(byte(interleave))
headerOption := encoder.WithHeaderOption(encoder.HeaderOptionNormal, byte(interleave))
if compress {
headerInfo = "compress"
headerOption = encoder.WithCompressedTimestampHeader()
headerOption = encoder.WithHeaderOption(encoder.HeaderOptionCompressedTimestamp, 0)
}

fmt.Fprintf(os.Stderr, "- Reducing %d file(s) [%s]\n",
Expand Down Expand Up @@ -876,10 +876,10 @@ func remove(ctx context.Context, fs *flag.FlagSet, args []string) (err error) {
}

headerInfo := fmt.Sprintf("interleave: %d", interleave)
headerOption := encoder.WithNormalHeader(byte(interleave))
headerOption := encoder.WithHeaderOption(encoder.HeaderOptionNormal, byte(interleave))
if compress {
headerInfo = "compress"
headerOption = encoder.WithCompressedTimestampHeader()
headerOption = encoder.WithHeaderOption(encoder.HeaderOptionCompressedTimestamp, 0)
}

var nameSuffix string
Expand Down
2 changes: 1 addition & 1 deletion cmd/fitconv/fitcsv/csv_to_fit.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ type CSVToFITConv struct {
func NewCSVToFITConv(fitWriter io.Writer, csvReader io.Reader) *CSVToFITConv {
enc := encoder.New(fitWriter,
encoder.WithProtocolVersion(proto.V2),
encoder.WithNormalHeader(15),
encoder.WithHeaderOption(encoder.HeaderOptionNormal, 15),
)

csvr := csv.NewReader(csvReader)
Expand Down
89 changes: 42 additions & 47 deletions encoder/encoder.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,22 +31,18 @@ const (
ErrWriterAtOrWriteSeekerIsExpected = errorString("io.WriterAt or io.WriteSeeker is expected")
)

// headerOption is header option.
type headerOption byte
// HeaderOption is header option for encoding message's Header.
type HeaderOption byte

const (
// headerOptionNormal is the default header option.
// This option has two sub-option to select from:
// 1. LocalMessageTypeZero [Default]
// Optimized for all devices. It only use LocalMesgNum 0.
// 2. MultipleLocalMessageTypes
// Using multiple local message types optimizes file size by avoiding the need to interleave different
// message definition. The number of multiple local message type can be specified between 0-15.
headerOptionNormal headerOption = 0

// Optimize file size by compressing timestamp's field in a message into its message header.
// When this enabled, LocalMesgNum 0 is automatically used since the 5 lsb is used for the timestamp.
headerOptionCompressedTimestamp headerOption = 1
// HeaderOptionNormal is the default header option. This option allow us to use local message type 0-15
// as the maximum number of allowed message definition interleave.
HeaderOptionNormal HeaderOption = 0

// HeaderOptionCompressedTimestamp optimizes file size by compressing timestamp's field in a message into
// its message header. Saves 7 bytes per message when its timestamp is compressed: 3 bytes for field definition
// and 4 bytes for the uint32 timestamp value. When this option is selected, only local messages type 0-3 is available.
HeaderOptionCompressedTimestamp HeaderOption = 1
)

// Encoder is FIT file encoder. See New() for details.
Expand Down Expand Up @@ -75,18 +71,18 @@ type Encoder struct {
}

type options struct {
messageValidator MessageValidator
writeBufferSize int
protocolVersion proto.Version
endianness byte
headerOption headerOption
multipleLocalMessageType byte
messageValidator MessageValidator
writeBufferSize int
protocolVersion proto.Version
endianness byte
headerOption HeaderOption
localMessageType byte
}

func defaultOptions() options {
return options{
endianness: proto.LittleEndian,
headerOption: headerOptionNormal,
headerOption: HeaderOptionNormal,
writeBufferSize: defaultWriteBufferSize,
}
}
Expand Down Expand Up @@ -125,28 +121,31 @@ func WithBigEndian() Option {
return func(o *options) { o.endianness = proto.BigEndian }
}

// WithCompressedTimestampHeader directs the Encoder to compress timestamp in header to reduce file size.
// Saves 7 bytes per message: 3 bytes for field definition and 4 bytes for the uint32 timestamp value.
func WithCompressedTimestampHeader() Option {
return func(o *options) { o.headerOption = headerOptionCompressedTimestamp }
}

// WithNormalHeader directs the Encoder to use NormalHeader for encoding the message using multiple local message types.
// By default, the Encoder uses local message type 0. This option allows users to specify values between 0-15 (while
// entering zero is equivalent to using the default option, nothing is changed). Using multiple local message types
// optimizes file size by avoiding the need to interleave different message definition.
// WithHeaderOption direct the Encoder to use this option instead of default HeaderOptionNormal and local message type zero.
// - If HeaderOptionNormal is selected, valid local message type value is 0-15; invalid values will be treated as 15.
// - If HeaderOptionCompressedTimestamp is selected, valid local message type value is 0-3; invalid values will be treated as 3.
// - Otherwise, no change will be made and the Encoder will use default values.
//
// Note: To minimize the required RAM for decoding, it's recommended to use a minimal number of local message types.
// NOTE: To minimize the required RAM for decoding, it's recommended to use a minimal number of local message type.
// For instance, embedded devices may only support decoding data from local message type 0. Additionally,
// multiple local message types should be avoided in file types like settings, where messages of the same type
// can be grouped together.
func WithNormalHeader(multipleLocalMessageType byte) Option {
if multipleLocalMessageType > proto.LocalMesgNumMask {
multipleLocalMessageType = proto.LocalMesgNumMask
}
func WithHeaderOption(headerOption HeaderOption, localMessageType byte) Option {
return func(o *options) {
o.headerOption = headerOptionNormal
o.multipleLocalMessageType = multipleLocalMessageType
switch headerOption {
case HeaderOptionNormal:
if localMessageType > 15 {
localMessageType = 15
}
case HeaderOptionCompressedTimestamp:
if localMessageType > 3 {
localMessageType = 3
}
default:
return
}
o.headerOption = headerOption
o.localMessageType = localMessageType
}
}

Expand Down Expand Up @@ -218,11 +217,7 @@ func (e *Encoder) Reset(w io.Writer, opts ...Option) {

e.reset()

var lruSize byte = 1
if e.options.headerOption == headerOptionNormal && e.options.multipleLocalMessageType > 0 {
lruSize = e.options.multipleLocalMessageType + 1
}
e.localMesgNumLRU.ResetWithNewSize(lruSize)
e.localMesgNumLRU.ResetWithNewSize(e.options.localMessageType + 1)
}

// reset resets the encoder's data that is being used for encoding,
Expand Down Expand Up @@ -440,7 +435,7 @@ func (e *Encoder) encodeMessage(mesg *proto.Message) (err error) {
}

var compressed bool
if e.options.headerOption == headerOptionCompressedTimestamp {
if e.options.headerOption == HeaderOptionCompressedTimestamp {
if e.w == io.Discard {
// NOTE: Only for calculating data size (Early Check Strategy)
var timestampField proto.Field
Expand All @@ -453,7 +448,7 @@ func (e *Encoder) encodeMessage(mesg *proto.Message) (err error) {
}
prevLen := len(mesg.Fields)
compressed = e.compressTimestampIntoHeader(mesg)
if prevLen > len(mesg.Fields) {
if compressed {
defer func() { // Revert: put timestamp field back at original index
mesg.Fields = mesg.Fields[:prevLen]
copy(mesg.Fields[i+1:], mesg.Fields[i:])
Expand All @@ -475,7 +470,7 @@ func (e *Encoder) encodeMessage(mesg *proto.Message) (err error) {

b[0] |= localMesgNum // Update the message definition header.
if compressed {
// TODO: implement compressed timestamp with multiple local messages type.
mesg.Header |= (localMesgNum << proto.CompressedBitShift)
} else {
mesg.Header |= localMesgNum
}
Expand Down Expand Up @@ -524,7 +519,7 @@ func (e *Encoder) compressTimestampIntoHeader(mesg *proto.Message) (ok bool) {
}

timeOffset := byte(timestamp & proto.CompressedTimeMask)
mesg.Header |= proto.MesgCompressedHeaderMask | timeOffset
mesg.Header = proto.MesgCompressedHeaderMask | timeOffset
mesg.RemoveFieldByNum(proto.FieldNumTimestamp)
return true
}
Expand Down
16 changes: 8 additions & 8 deletions encoder/encoder_bench_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -134,20 +134,20 @@ func BenchmarkEncode(b *testing.B) {
})
b.Run("normal header 15", func(b *testing.B) {
b.StopTimer()
enc := encoder.New(io.Discard, encoder.WithNormalHeader(15))
enc := encoder.New(io.Discard, encoder.WithHeaderOption(encoder.HeaderOptionNormal, 15))
b.StartTimer()
for i := 0; i < b.N; i++ {
_ = enc.Encode(fit)
enc.Reset(io.Discard, encoder.WithNormalHeader(15))
enc.Reset(io.Discard, encoder.WithHeaderOption(encoder.HeaderOptionNormal, 15))
}
})
b.Run("compressed timestamp header", func(b *testing.B) {
b.StopTimer()
enc := encoder.New(io.Discard, encoder.WithCompressedTimestampHeader())
enc := encoder.New(io.Discard, encoder.WithHeaderOption(encoder.HeaderOptionCompressedTimestamp, 0))
b.StartTimer()
for i := 0; i < b.N; i++ {
_ = enc.Encode(fit)
enc.Reset(io.Discard, encoder.WithCompressedTimestampHeader())
enc.Reset(io.Discard, encoder.WithHeaderOption(encoder.HeaderOptionCompressedTimestamp, 0))
}
})
}
Expand All @@ -168,20 +168,20 @@ func BenchmarkEncodeWriterAt(b *testing.B) {
})
b.Run("normal header 15", func(b *testing.B) {
b.StopTimer()
enc := encoder.New(DiscardAt, encoder.WithNormalHeader(15))
enc := encoder.New(DiscardAt, encoder.WithHeaderOption(encoder.HeaderOptionNormal, 15))
b.StartTimer()
for i := 0; i < b.N; i++ {
_ = enc.Encode(fit)
enc.Reset(DiscardAt, encoder.WithNormalHeader(15))
enc.Reset(DiscardAt, encoder.WithHeaderOption(encoder.HeaderOptionNormal, 15))
}
})
b.Run("compressed timestamp header", func(b *testing.B) {
b.StopTimer()
enc := encoder.New(DiscardAt, encoder.WithCompressedTimestampHeader())
enc := encoder.New(DiscardAt, encoder.WithHeaderOption(encoder.HeaderOptionCompressedTimestamp, 0))
b.StartTimer()
for i := 0; i < b.N; i++ {
_ = enc.Encode(fit)
enc.Reset(DiscardAt, encoder.WithCompressedTimestampHeader())
enc.Reset(DiscardAt, encoder.WithHeaderOption(encoder.HeaderOptionCompressedTimestamp, 0))
}
})
}
Expand Down
Loading