Skip to content

Commit

Permalink
groot/internal/httpio: add a pool of http.Request
Browse files Browse the repository at this point in the history
This CL adds a pool of http.Request to alleviate somewhat the memory
pressure of cloning those when requesting new chunks of data from the
HTTP server.

old: reading remote file after having downloaded to a local one
new: reading remote file w/ httpio + sync.Pool

```
  name                    old time/op    new time/op    delta
  DumpHTTP/delay-0s-8        279ms ± 1%     287ms ± 2%   +3.09%  (p=0.000 n=27+29)
  DumpHTTP/delay-5ms-8       279ms ± 2%     288ms ± 2%   +3.06%  (p=0.000 n=29+29)
  DumpHTTP/delay-10ms-8      281ms ± 3%     287ms ± 1%   +2.24%  (p=0.000 n=26+30)
  DumpHTTP/delay-100ms-8     286ms ± 3%     288ms ± 1%   +0.45%  (p=0.032 n=25+29)

  name                    old alloc/op   new alloc/op   delta
  DumpHTTP/delay-0s-8        156MB ± 0%     159MB ± 0%   +1.93%  (p=0.000 n=29+30)
  DumpHTTP/delay-5ms-8       156MB ± 0%     159MB ± 0%   +1.95%  (p=0.000 n=30+30)
  DumpHTTP/delay-10ms-8      155MB ± 0%     159MB ± 0%   +1.95%  (p=0.000 n=30+30)
  DumpHTTP/delay-100ms-8     155MB ± 0%     159MB ± 0%   +1.99%  (p=0.000 n=30+28)

  name                    old allocs/op  new allocs/op  delta
  DumpHTTP/delay-0s-8        81.4k ± 0%    117.6k ± 0%  +44.46%  (p=0.000 n=30+30)
  DumpHTTP/delay-5ms-8       81.4k ± 0%    117.6k ± 0%  +44.48%  (p=0.000 n=30+30)
  DumpHTTP/delay-10ms-8      81.4k ± 0%    117.5k ± 0%  +44.48%  (p=0.000 n=29+30)
  DumpHTTP/delay-100ms-8     81.4k ± 0%    117.6k ± 0%  +44.51%  (p=0.000 n=30+29)
```

Fixes #142.
  • Loading branch information
sbinet committed Mar 11, 2022
1 parent 0e0b993 commit 7909cd7
Showing 1 changed file with 19 additions and 2 deletions.
21 changes: 19 additions & 2 deletions groot/internal/httpio/reader.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
"io"
"net/http"
"strconv"
"sync"
)

// Reader presents an HTTP resource as an io.Reader and io.ReaderAt.
Expand All @@ -19,6 +20,8 @@ type Reader struct {
ctx context.Context
cancel context.CancelFunc

pool sync.Pool

r *io.SectionReader
len int64
etag string
Expand Down Expand Up @@ -69,6 +72,13 @@ func Open(uri string, opts ...Option) (r *Reader, err error) {
r.etag = hdr.Header.Get("Etag")
r.r = io.NewSectionReader(r, 0, r.len)

r.req.Header.Set("Range", "")
r.pool = sync.Pool{
New: func() interface{} {
return r.req.Clone(r.ctx)
},
}

return r, nil
}

Expand Down Expand Up @@ -107,8 +117,8 @@ func (r *Reader) ReadAt(p []byte, off int64) (int, error) {
}

rng := rng(off, off+int64(len(p))-1)
req := r.req.Clone(r.ctx)
req.Header.Set("Range", rng)
req := r.getReq(rng)
defer r.pool.Put(req)

resp, err := r.cli.Do(req)
if err != nil {
Expand Down Expand Up @@ -138,6 +148,13 @@ func (r *Reader) ReadAt(p []byte, off int64) (int, error) {
return n, nil
}

func (r *Reader) getReq(rng string) *http.Request {
o := r.pool.Get().(*http.Request)
o.Header = r.req.Header.Clone()
o.Header["Range"][0] = rng
return o
}

func rng(beg, end int64) string {
return "bytes=" + strconv.Itoa(int(beg)) + "-" + strconv.Itoa(int(end))
}
Expand Down

0 comments on commit 7909cd7

Please sign in to comment.